diff --git a/.env.example b/.env.example index e0e98c103..7a67045cd 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY=7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr +SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr" # SECURITY WARNING: don't run with debug turned on in production! DEBUG=true @@ -25,7 +25,7 @@ POSTGRES_HOST=db CELERY_BROKER=redis://redis:6379/0 CELERY_RESULT_BACKEND=redis://redis:6379/0 -EMAIL_HOST='smtp.mailgun.org' +EMAIL_HOST="smtp.mailgun.org" EMAIL_PORT=587 EMAIL_HOST_USER=mail@your.domain.here EMAIL_HOST_PASSWORD=emailpassword123 diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml new file mode 100644 index 000000000..3ce368ecd --- /dev/null +++ b/.github/workflows/django-tests.yml @@ -0,0 +1,68 @@ +name: Run Python Tests +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-20.04 + strategy: + max-parallel: 4 + matrix: + db: [postgres] + python-version: [3.9] + include: + - db: postgres + db_port: 5432 + + services: + postgres: + image: postgres:10 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: hunter2 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Run Tests + env: + DB: ${{ matrix.db }} + DB_HOST: 127.0.0.1 + DB_PORT: ${{ matrix.db_port }} + DB_PASSWORD: hunter2 + SECRET_KEY: beepbeep + DEBUG: true + DOMAIN: your.domain.here + OL_URL: https://openlibrary.org + BOOKWYRM_DATABASE_BACKEND: postgres + MEDIA_ROOT: images/ + POSTGRES_PASSWORD: hunter2 + POSTGRES_USER: postgres + POSTGRES_DB: github_actions + POSTGRES_HOST: 127.0.0.1 + CELERY_BROKER: "" + CELERY_RESULT_BACKEND: "" + EMAIL_HOST: "smtp.mailgun.org" + EMAIL_PORT: 587 + EMAIL_HOST_USER: "" + EMAIL_HOST_PASSWORD: "" + EMAIL_USE_TLS: true + run: | + python manage.py test diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py index 931af0a0c..852db345c 100644 --- a/bookwyrm/activitypub/__init__.py +++ b/bookwyrm/activitypub/__init__.py @@ -5,6 +5,8 @@ import sys from .base_activity import ActivityEncoder, Image, PublicKey, Signature from .base_activity import Link, Mention from .base_activity import ActivitySerializerError +from .base_activity import tag_formatter +from .base_activity import image_formatter, image_attachments_formatter from .note import Note, GeneratedNote, Article, Comment, Review, Quotation from .note import Tombstone from .interaction import Boost, Like diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 6ae7883ef..54c2baea9 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -1,12 +1,17 @@ ''' basics for an activitypub serializer ''' from dataclasses import dataclass, fields, MISSING from json import JSONEncoder +from uuid import uuid4 + +from django.core.files.base import ContentFile +from django.db.models.fields.related_descriptors \ + import ForwardManyToOneDescriptor, ManyToManyDescriptor, \ + ReverseManyToOneDescriptor +from django.db.models.fields.files import ImageFileDescriptor +import requests from bookwyrm import books_manager, models -from django.db.models.fields.related_descriptors \ - import ForwardManyToOneDescriptor - class ActivitySerializerError(ValueError): ''' routine problems serializing activitypub json ''' @@ -69,7 +74,8 @@ class ActivityObject: try: value = kwargs[field.name] except KeyError: - if field.default == MISSING: + if field.default == MISSING and \ + field.default_factory == MISSING: raise ActivitySerializerError(\ 'Missing required field: %s' % field.name) value = field.default @@ -90,6 +96,9 @@ class ActivityObject: model_fields = [m.name for m in model._meta.get_fields()] mapped_fields = {} + many_to_many_fields = {} + one_to_many_fields = {} + image_fields = {} for mapping in model.activity_mappings: if mapping.model_key not in model_fields: @@ -101,23 +110,56 @@ class ActivityObject: value = getattr(self, mapping.activity_key) model_field = getattr(model, mapping.model_key) - # remote_id -> foreign key resolver - if isinstance(model_field, ForwardManyToOneDescriptor) and value: + formatted_value = mapping.model_formatter(value) + if isinstance(model_field, ForwardManyToOneDescriptor) and \ + formatted_value: + # foreign key remote id reolver fk_model = model_field.field.related_model - value = resolve_foreign_key(fk_model, value) + reference = resolve_foreign_key(fk_model, formatted_value) + mapped_fields[mapping.model_key] = reference + elif isinstance(model_field, ManyToManyDescriptor): + many_to_many_fields[mapping.model_key] = formatted_value + elif isinstance(model_field, ReverseManyToOneDescriptor): + # attachments on statuses, for example + one_to_many_fields[mapping.model_key] = formatted_value + elif isinstance(model_field, ImageFileDescriptor): + # image fields need custom handling + image_fields[mapping.model_key] = formatted_value + else: + mapped_fields[mapping.model_key] = formatted_value - mapped_fields[mapping.model_key] = mapping.model_formatter(value) - - - # updating an existing model isntance if instance: + # updating an existing model isntance for k, v in mapped_fields.items(): setattr(instance, k, v) instance.save() - return instance + else: + # creating a new model instance + instance = model.objects.create(**mapped_fields) - # creating a new model instance - return model.objects.create(**mapped_fields) + # add many-to-many fields + for (model_key, values) in many_to_many_fields.items(): + getattr(instance, model_key).set(values) + instance.save() + + # add images + for (model_key, value) in image_fields.items(): + if not value: + continue + getattr(instance, model_key).save(*value, save=True) + + # add one to many fields + for (model_key, values) in one_to_many_fields.items(): + items = [] + for item in values: + # the reference id wasn't available at creation time + setattr(item, instance.__class__.__name__.lower(), instance) + item.save() + items.append(item) + if items: + getattr(instance, model_key).set(items) + instance.save() + return instance def serialize(self): @@ -129,7 +171,7 @@ class ActivityObject: def resolve_foreign_key(model, remote_id): ''' look up the remote_id on an activity json field ''' - if model in [models.Edition, models.Work]: + if model in [models.Edition, models.Work, models.Book]: return books_manager.get_or_create_book(remote_id) result = model.objects @@ -145,3 +187,62 @@ def resolve_foreign_key(model, remote_id): 'Could not resolve remote_id in %s model: %s' % \ (model.__name__, remote_id)) return result + + +def tag_formatter(tags, tag_type): + ''' helper function to extract foreign keys from tag activity json ''' + if not isinstance(tags, list): + return [] + items = [] + types = { + 'Book': models.Book, + 'Mention': models.User, + } + for tag in [t for t in tags if t.get('type') == tag_type]: + if not tag_type in types: + continue + remote_id = tag.get('href') + try: + item = resolve_foreign_key(types[tag_type], remote_id) + except ActivitySerializerError: + continue + items.append(item) + return items + + +def image_formatter(image_json): + ''' helper function to load images and format them for a model ''' + if isinstance(image_json, list): + try: + image_json = image_json[0] + except IndexError: + return None + + if not image_json or not hasattr(image_json, 'url'): + return None + url = image_json.get('url') + + try: + response = requests.get(url) + except ConnectionError: + return None + if not response.ok: + return None + + image_name = str(uuid4()) + '.' + url.split('.')[-1] + image_content = ContentFile(response.content) + return [image_name, image_content] + + +def image_attachments_formatter(images_json): + ''' deserialize a list of images ''' + attachments = [] + for image in images_json: + caption = image.get('name') + attachment = models.Attachment(caption=caption) + image_field = image_formatter(image) + if not image_field: + continue + attachment.image.save(*image_field, save=False) + attachments.append(attachment) + return attachments diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py index 60d36bd02..2bfafba70 100644 --- a/bookwyrm/activitypub/book.py +++ b/bookwyrm/activitypub/book.py @@ -25,7 +25,7 @@ class Book(ActivityObject): librarything_key: str goodreads_key: str - attachment: List[Image] = field(default=lambda: []) + attachment: List[Image] = field(default_factory=lambda: []) type: str = 'Book' @@ -56,10 +56,10 @@ class Work(Book): class Author(ActivityObject): ''' author of a book ''' name: str - born: str - died: str - aliases: str - bio: str - openlibrary_key: str - wikipedia_link: str + born: str = '' + died: str = '' + aliases: str = '' + bio: str = '' + openlibraryKey: str = '' + wikipediaLink: str = '' type: str = 'Person' diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py index ebc0cf3ce..9eab952d3 100644 --- a/bookwyrm/activitypub/note.py +++ b/bookwyrm/activitypub/note.py @@ -24,8 +24,8 @@ class Note(ActivityObject): cc: List[str] content: str replies: Dict - tag: List[Link] = field(default=lambda: []) - attachment: List[Image] = field(default=lambda: []) + tag: List[Link] = field(default_factory=lambda: []) + attachment: List[Image] = field(default_factory=lambda: []) sensitive: bool = False type: str = 'Note' diff --git a/bookwyrm/activitypub/person.py b/bookwyrm/activitypub/person.py index 118774a27..324d68e32 100644 --- a/bookwyrm/activitypub/person.py +++ b/bookwyrm/activitypub/person.py @@ -15,7 +15,7 @@ class Person(ActivityObject): summary: str publicKey: PublicKey endpoints: Dict - icon: Image = field(default=lambda: {}) + icon: Image = field(default_factory=lambda: {}) bookwyrmUser: bool = False manuallyApprovesFollowers: str = False discoverable: str = True diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 7fc4596b3..d709b075a 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -157,7 +157,7 @@ class AbstractConnector(ABC): def update_book_from_data(self, book, data, update_cover=True): ''' for creating a new book or syncing with data ''' - book = update_from_mappings(book, data, self.book_mappings) + book = self.update_from_mappings(book, data, self.book_mappings) author_text = [] for author in self.get_authors_from_data(data): @@ -262,23 +262,23 @@ class AbstractConnector(ABC): ''' get more info on a book ''' -def update_from_mappings(obj, data, mappings): - ''' assign data to model with mappings ''' - for mapping in mappings: - # check if this field is present in the data - value = data.get(mapping.remote_field) - if not value: - continue + def update_from_mappings(self, obj, data, mappings): + ''' assign data to model with mappings ''' + for mapping in mappings: + # check if this field is present in the data + value = data.get(mapping.remote_field) + if not value: + continue - # extract the value in the right format - try: - value = mapping.formatter(value) - except: - continue + # extract the value in the right format + try: + value = mapping.formatter(value) + except: + continue - # assign the formatted value to the model - obj.__setattr__(mapping.local_field, value) - return obj + # assign the formatted value to the model + obj.__setattr__(mapping.local_field, value) + return obj def get_date(date_string): diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index 6ed9dda1d..1bc81450d 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -1,55 +1,21 @@ ''' using another bookwyrm instance as a source of book data ''' -from uuid import uuid4 - -from django.core.exceptions import ObjectDoesNotExist -from django.core.files.base import ContentFile from django.db import transaction -import requests -from bookwyrm import models -from .abstract_connector import AbstractConnector, SearchResult, Mapping -from .abstract_connector import update_from_mappings, get_date, get_data +from bookwyrm import activitypub, models +from .abstract_connector import AbstractConnector, SearchResult +from .abstract_connector import get_data class Connector(AbstractConnector): ''' interact with other instances ''' - def __init__(self, identifier): - super().__init__(identifier) - self.key_mappings = [ - Mapping('isbn_13', model=models.Edition), - Mapping('isbn_10', model=models.Edition), - Mapping('lccn', model=models.Work), - Mapping('oclc_number', model=models.Edition), - Mapping('openlibrary_key'), - Mapping('goodreads_key'), - Mapping('asin'), - ] - self.book_mappings = self.key_mappings + [ - Mapping('sort_title'), - Mapping('subtitle'), - Mapping('description'), - Mapping('languages'), - Mapping('series'), - Mapping('series_number'), - Mapping('subjects'), - Mapping('subject_places'), - Mapping('first_published_date'), - Mapping('published_date'), - Mapping('pages'), - Mapping('physical_format'), - Mapping('publishers'), - ] - - self.author_mappings = [ - Mapping('name'), - Mapping('bio'), - Mapping('openlibrary_key'), - Mapping('wikipedia_link'), - Mapping('aliases'), - Mapping('born', formatter=get_date), - Mapping('died', formatter=get_date), - ] + def update_from_mappings(self, obj, data, mappings): + ''' serialize book data into a model ''' + if self.is_work_data(data): + work_data = activitypub.Work(**data) + return work_data.to_model(models.Work, instance=obj) + edition_data = activitypub.Edition(**data) + return edition_data.to_model(models.Edition, instance=obj) def get_remote_id_from_data(self, data): @@ -57,7 +23,7 @@ class Connector(AbstractConnector): def is_work_data(self, data): - return data['type'] == 'Work' + return data.get('type') == 'Work' def get_edition_from_work_data(self, data): @@ -71,46 +37,20 @@ class Connector(AbstractConnector): def get_authors_from_data(self, data): - for author_url in data.get('authors', []): - yield self.get_or_create_author(author_url) + ''' load author data ''' + for author_id in data.get('authors', []): + try: + yield models.Author.objects.get(origin_id=author_id) + except models.Author.DoesNotExist: + pass + data = get_data(author_id) + author_data = activitypub.Author(**data) + author = author_data.to_model(models.Author) + yield author def get_cover_from_data(self, data): - cover_data = data.get('attachment') - if not cover_data: - return None - try: - cover_url = cover_data[0].get('url') - except IndexError: - return None - try: - response = requests.get(cover_url) - except ConnectionError: - return None - - if not response.ok: - return None - - image_name = str(uuid4()) + '.' + cover_url.split('.')[-1] - image_content = ContentFile(response.content) - return [image_name, image_content] - - - def get_or_create_author(self, remote_id): - ''' load that author ''' - try: - return models.Author.objects.get(origin_id=remote_id) - except ObjectDoesNotExist: - pass - - data = get_data(remote_id) - - # ingest a new author - author = models.Author(origin_id=remote_id) - author = update_from_mappings(author, data, self.author_mappings) - author.save() - - return author + pass def parse_search_data(self, data): diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 5c26ad45b..5e18616d5 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -7,7 +7,6 @@ from django.core.files.base import ContentFile from bookwyrm import models from .abstract_connector import AbstractConnector, SearchResult, Mapping from .abstract_connector import ConnectorException -from .abstract_connector import update_from_mappings from .abstract_connector import get_date, get_data from .openlibrary_languages import languages @@ -185,7 +184,7 @@ class Connector(AbstractConnector): data = get_data(url) author = models.Author(openlibrary_key=olkey) - author = update_from_mappings(author, data, self.author_mappings) + author = self.update_from_mappings(author, data, self.author_mappings) name = data.get('name') # TODO this is making some BOLD assumption if name: diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index c2c223bd7..0e7c18567 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -253,7 +253,6 @@ def handle_delete_status(activity): status_builder.delete_status(status) - @app.task def handle_favorite(activity): ''' approval of your good good post ''' diff --git a/bookwyrm/migrations/0012_attachment.py b/bookwyrm/migrations/0012_attachment.py new file mode 100644 index 000000000..495538517 --- /dev/null +++ b/bookwyrm/migrations/0012_attachment.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.7 on 2020-11-24 19:39 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0011_auto_20201113_1727'), + ] + + operations = [ + migrations.CreateModel( + name='Attachment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_date', models.DateTimeField(auto_now_add=True)), + ('updated_date', models.DateTimeField(auto_now=True)), + ('remote_id', models.CharField(max_length=255, null=True)), + ('image', models.ImageField(blank=True, null=True, upload_to='status/')), + ('caption', models.TextField(blank=True, null=True)), + ('status', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='attachments', to='bookwyrm.Status')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/bookwyrm/migrations/0013_book_origin_id.py b/bookwyrm/migrations/0013_book_origin_id.py new file mode 100644 index 000000000..581a2406e --- /dev/null +++ b/bookwyrm/migrations/0013_book_origin_id.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.7 on 2020-11-24 21:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0012_attachment'), + ] + + operations = [ + migrations.AlterField( + model_name='book', + name='origin_id', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py index f16864cc1..35a34a257 100644 --- a/bookwyrm/models/__init__.py +++ b/bookwyrm/models/__init__.py @@ -2,12 +2,13 @@ import inspect import sys -from .book import Book, Work, Edition, Author +from .book import Book, Work, Edition +from .author import Author from .connector import Connector from .relationship import UserFollows, UserFollowRequest, UserBlocks from .shelf import Shelf, ShelfBook from .status import Status, GeneratedNote, Review, Comment, Quotation -from .status import Favorite, Boost, Notification, ReadThrough, ProgressMode, ProgressUpdate +from .status import Attachment, Favorite, Boost, Notification, ReadThrough, ProgressMode, ProgressUpdate from .tag import Tag from .user import User from .federated_server import FederatedServer diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py new file mode 100644 index 000000000..1d7017974 --- /dev/null +++ b/bookwyrm/models/author.py @@ -0,0 +1,50 @@ +''' database schema for info about authors ''' +from django.db import models +from django.utils import timezone + +from bookwyrm import activitypub +from bookwyrm.utils.fields import ArrayField + +from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel + + +class Author(ActivitypubMixin, BookWyrmModel): + ''' basic biographic info ''' + origin_id = models.CharField(max_length=255, null=True) + ''' copy of an author from OL ''' + openlibrary_key = models.CharField(max_length=255, blank=True, null=True) + sync = models.BooleanField(default=True) + last_sync_date = models.DateTimeField(default=timezone.now) + wikipedia_link = models.CharField(max_length=255, blank=True, null=True) + # idk probably other keys would be useful here? + born = models.DateTimeField(blank=True, null=True) + died = models.DateTimeField(blank=True, null=True) + name = models.CharField(max_length=255) + last_name = models.CharField(max_length=255, blank=True, null=True) + first_name = models.CharField(max_length=255, blank=True, null=True) + aliases = ArrayField( + models.CharField(max_length=255), blank=True, default=list + ) + bio = models.TextField(null=True, blank=True) + + @property + def display_name(self): + ''' Helper to return a displayable name''' + if self.name: + return self.name + # don't want to return a spurious space if all of these are None + if self.first_name and self.last_name: + return self.first_name + ' ' + self.last_name + return self.last_name or self.first_name + + activity_mappings = [ + ActivityMapping('id', 'remote_id'), + ActivityMapping('name', 'name'), + ActivityMapping('born', 'born'), + ActivityMapping('died', 'died'), + ActivityMapping('aliases', 'aliases'), + ActivityMapping('bio', 'bio'), + ActivityMapping('openlibraryKey', 'openlibrary_key'), + ActivityMapping('wikipediaLink', 'wikipedia_link'), + ] + activity_serializer = activitypub.Author diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 8150d650c..8c28c8abb 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -59,20 +59,34 @@ class ActivitypubMixin: def to_activity(self, pure=False): ''' convert from a model to an activity ''' if pure: + # works around bookwyrm-specific fields for vanilla AP services mappings = self.pure_activity_mappings else: + # may include custom fields that bookwyrm instances will understand mappings = self.activity_mappings fields = {} for mapping in mappings: if not hasattr(self, mapping.model_key) or not mapping.activity_key: + # this field on the model isn't serialized continue value = getattr(self, mapping.model_key) if hasattr(value, 'remote_id'): + # this is probably a foreign key field, which we want to + # serialize as just the remote_id url reference value = value.remote_id - if isinstance(value, datetime): + elif isinstance(value, datetime): value = value.isoformat() - fields[mapping.activity_key] = mapping.activity_formatter(value) + + # run the custom formatter function set in the model + result = mapping.activity_formatter(value) + if mapping.activity_key in fields and \ + isinstance(fields[mapping.activity_key], list): + # there can be two database fields that map to the same AP list + # this happens in status tags, which combines user and book tags + fields[mapping.activity_key] += result + else: + fields[mapping.activity_key] = result if pure: return self.pure_activity_serializer( @@ -242,3 +256,32 @@ class ActivityMapping: model_key: str activity_formatter: Callable = lambda x: x model_formatter: Callable = lambda x: x + + +def tag_formatter(items, name_field, activity_type): + ''' helper function to format lists of foreign keys into Tags ''' + tags = [] + for item in items.all(): + tags.append(activitypub.Link( + href=item.remote_id, + name=getattr(item, name_field), + type=activity_type + )) + return tags + + +def image_formatter(image, default_path=None): + ''' convert images into activitypub json ''' + if image and hasattr(image, 'url'): + url = image.url + elif default_path: + url = default_path + else: + return None + url = 'https://%s%s' % (DOMAIN, url) + return activitypub.Image(url=url) + + +def image_attachments_formatter(images): + ''' create a list of image attachments ''' + return [image_formatter(i) for i in images] diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 0c81f0596..aaf2a259e 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -12,10 +12,11 @@ from bookwyrm.utils.fields import ArrayField from .base_model import ActivityMapping, BookWyrmModel from .base_model import ActivitypubMixin, OrderedCollectionPageMixin +from .base_model import image_attachments_formatter class Book(ActivitypubMixin, BookWyrmModel): ''' a generic book, which can mean either an edition or a work ''' - origin_id = models.CharField(max_length=255, null=True) + origin_id = models.CharField(max_length=255, null=True, blank=True) # these identifiers apply to both works and editions openlibrary_key = models.CharField(max_length=255, blank=True, null=True) librarything_key = models.CharField(max_length=255, blank=True, null=True) @@ -60,15 +61,6 @@ class Book(ActivitypubMixin, BookWyrmModel): ''' the activitypub serialization should be a list of author ids ''' return [a.remote_id for a in self.authors.all()] - @property - def ap_cover(self): - ''' an image attachment ''' - if not self.cover or not hasattr(self.cover, 'url'): - return [] - return [activitypub.Image( - url='https://%s%s' % (DOMAIN, self.cover.url), - )] - @property def ap_parent_work(self): ''' reference the work via local id not remote ''' @@ -110,7 +102,12 @@ class Book(ActivitypubMixin, BookWyrmModel): ActivityMapping('lccn', 'lccn'), ActivityMapping('editions', 'editions_path'), - ActivityMapping('attachment', 'ap_cover'), + ActivityMapping( + 'attachment', 'cover', + # this expects an iterable and the field is just an image + lambda x: image_attachments_formatter([x]), + activitypub.image_formatter + ), ] def save(self, *args, **kwargs): @@ -197,7 +194,7 @@ class Edition(Book): if self.isbn_10 and not self.isbn_13: self.isbn_13 = isbn_10_to_13(self.isbn_10) - super().save(*args, **kwargs) + return super().save(*args, **kwargs) def isbn_10_to_13(isbn_10): @@ -241,44 +238,3 @@ def isbn_13_to_10(isbn_13): if checkdigit == 10: checkdigit = 'X' return converted + str(checkdigit) - - -class Author(ActivitypubMixin, BookWyrmModel): - origin_id = models.CharField(max_length=255, null=True) - ''' copy of an author from OL ''' - openlibrary_key = models.CharField(max_length=255, blank=True, null=True) - sync = models.BooleanField(default=True) - last_sync_date = models.DateTimeField(default=timezone.now) - wikipedia_link = models.CharField(max_length=255, blank=True, null=True) - # idk probably other keys would be useful here? - born = models.DateTimeField(blank=True, null=True) - died = models.DateTimeField(blank=True, null=True) - name = models.CharField(max_length=255) - last_name = models.CharField(max_length=255, blank=True, null=True) - first_name = models.CharField(max_length=255, blank=True, null=True) - aliases = ArrayField( - models.CharField(max_length=255), blank=True, default=list - ) - bio = models.TextField(null=True, blank=True) - - @property - def display_name(self): - ''' Helper to return a displayable name''' - if self.name: - return self.name - # don't want to return a spurious space if all of these are None - if self.first_name and self.last_name: - return self.first_name + ' ' + self.last_name - return self.last_name or self.first_name - - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('name', 'display_name'), - ActivityMapping('born', 'born'), - ActivityMapping('died', 'died'), - ActivityMapping('aliases', 'aliases'), - ActivityMapping('bio', 'bio'), - ActivityMapping('openlibrary_key', 'openlibrary_key'), - ActivityMapping('wikipedia_link', 'wikipedia_link'), - ] - activity_serializer = activitypub.Author diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 406de2408..732cdd62b 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -7,6 +7,7 @@ from model_utils.managers import InheritanceManager from bookwyrm import activitypub from .base_model import ActivitypubMixin, OrderedCollectionPageMixin from .base_model import ActivityMapping, BookWyrmModel, PrivacyLevels +from .base_model import tag_formatter, image_attachments_formatter class Status(OrderedCollectionPageMixin, BookWyrmModel): @@ -57,24 +58,6 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): ''' structured replies block ''' return self.to_replies() - @property - def ap_tag(self): - ''' references to books and/or users ''' - - tags = [] - for book in self.mention_books.all(): - tags.append(activitypub.Link( - href=book.remote_id, - name=book.title, - type='Book' - )) - for user in self.mention_users.all(): - tags.append(activitypub.Mention( - href=user.remote_id, - name=user.username, - )) - return tags - @property def ap_status_image(self): ''' attach a book cover, if relevent ''' @@ -94,7 +77,21 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): ActivityMapping('to', 'ap_to'), ActivityMapping('cc', 'ap_cc'), ActivityMapping('replies', 'ap_replies'), - ActivityMapping('tag', 'ap_tag'), + ActivityMapping( + 'tag', 'mention_books', + lambda x: tag_formatter(x, 'title', 'Book'), + lambda x: activitypub.tag_formatter(x, 'Book') + ), + ActivityMapping( + 'tag', 'mention_users', + lambda x: tag_formatter(x, 'username', 'Mention'), + lambda x: activitypub.tag_formatter(x, 'Mention') + ), + ActivityMapping( + 'attachment', 'attachments', + lambda x: image_attachments_formatter(x.all()), + activitypub.image_attachments_formatter + ) ] # serializing to bookwyrm expanded activitypub @@ -148,9 +145,21 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): def save(self, *args, **kwargs): ''' update user active time ''' - self.user.last_active_date = timezone.now() - self.user.save() - super().save(*args, **kwargs) + if self.user.local: + self.user.last_active_date = timezone.now() + self.user.save() + return super().save(*args, **kwargs) + + +class Attachment(BookWyrmModel): + ''' an image (or, in the future, video etc) associated with a status ''' + status = models.ForeignKey( + 'Status', + on_delete=models.CASCADE, + related_name='attachments' + ) + image = models.ImageField(upload_to='status/', null=True, blank=True) + caption = models.TextField(null=True, blank=True) class GeneratedNote(Status): diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 0c7d1a182..b38a4b192 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -10,8 +10,8 @@ from bookwyrm.models.shelf import Shelf from bookwyrm.models.status import Status from bookwyrm.settings import DOMAIN from bookwyrm.signatures import create_key_pair -from .base_model import OrderedCollectionPageMixin -from .base_model import ActivityMapping +from .base_model import ActivityMapping, OrderedCollectionPageMixin +from .base_model import image_formatter class User(OrderedCollectionPageMixin, AbstractUser): @@ -78,16 +78,6 @@ class User(OrderedCollectionPageMixin, AbstractUser): ''' generates url for activitypub followers page ''' return '%s/followers' % self.remote_id - @property - def ap_icon(self): - ''' send default icon if one isn't set ''' - if self.avatar: - url = self.avatar.url - else: - url = '/static/images/default_avi.jpg' - url = 'https://%s%s' % (DOMAIN, url) - return activitypub.Image(url=url) - @property def ap_public_key(self): ''' format the public key block for activitypub ''' @@ -122,7 +112,11 @@ class User(OrderedCollectionPageMixin, AbstractUser): activity_formatter=lambda x: {'sharedInbox': x}, model_formatter=lambda x: x.get('sharedInbox') ), - ActivityMapping('icon', 'ap_icon'), + ActivityMapping( + 'icon', 'avatar', + lambda x: image_formatter(x, '/static/images/default_avi.jpg'), + activitypub.image_formatter + ), ActivityMapping( 'manuallyApprovesFollowers', 'manually_approves_followers' diff --git a/bookwyrm/remote_user.py b/bookwyrm/remote_user.py index 9178451b4..23a805b38 100644 --- a/bookwyrm/remote_user.py +++ b/bookwyrm/remote_user.py @@ -1,9 +1,7 @@ ''' manage remote users ''' from urllib.parse import urlparse -from uuid import uuid4 import requests -from django.core.files.base import ContentFile from django.db import transaction from bookwyrm import activitypub, models @@ -22,14 +20,9 @@ def get_or_create_remote_user(actor): actor_parts = urlparse(actor) with transaction.atomic(): - user = create_remote_user(data) + user = activitypub.Person(**data).to_model(models.User) user.federated_server = get_or_create_remote_server(actor_parts.netloc) user.save() - - avatar = get_avatar(data) - if avatar: - user.avatar.save(*avatar) - if user.bookwyrm_user: get_remote_reviews.delay(user.id) return user @@ -55,12 +48,6 @@ def fetch_user_data(actor): return data -def create_remote_user(data): - ''' parse the activitypub actor data into a user ''' - actor = activitypub.Person(**data) - return actor.to_model(models.User) - - def refresh_remote_user(user): ''' get updated user data from its home instance ''' data = fetch_user_data(user.remote_id) @@ -69,21 +56,6 @@ def refresh_remote_user(user): activity.to_model(models.User, instance=user) -def get_avatar(data): - ''' find the icon attachment and load the image from the remote sever ''' - icon_blob = data.get('icon') - if not icon_blob or not icon_blob.get('url'): - return None - - response = requests.get(icon_blob['url']) - if not response.ok: - return None - - image_name = str(uuid4()) + '.' + icon_blob['url'].split('.')[-1] - image_content = ContentFile(response.content) - return [image_name, image_content] - - @app.task def get_remote_reviews(user_id): ''' ingest reviews by a new remote bookwyrm user ''' diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 46bc25142..5adf960f6 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -40,6 +40,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.humanize', + 'django_rename_app', 'bookwyrm', 'celery', ] diff --git a/bookwyrm/static/css/format.css b/bookwyrm/static/css/format.css index db3c20ef3..9e8a24ba9 100644 --- a/bookwyrm/static/css/format.css +++ b/bookwyrm/static/css/format.css @@ -1,4 +1,7 @@ /* --- --- */ +.image { + overflow: hidden; +} .navbar .logo { max-height: 50px; } diff --git a/bookwyrm/status.py b/bookwyrm/status.py index 4baaed6d3..c950f4ab3 100644 --- a/bookwyrm/status.py +++ b/bookwyrm/status.py @@ -13,7 +13,7 @@ def delete_status(status): def create_status(activity): - ''' unfortunately, it's not QUITE as simple as deserialiing it ''' + ''' unfortunately, it's not QUITE as simple as deserializing it ''' # render the json into an activity object serializer = activitypub.activity_objects[activity['type']] activity = serializer(**activity) diff --git a/bookwyrm/templates/edit_book.html b/bookwyrm/templates/edit_book.html index bd89ac0a9..54cefb0a7 100644 --- a/bookwyrm/templates/edit_book.html +++ b/bookwyrm/templates/edit_book.html @@ -20,6 +20,12 @@ +{% if login_form.non_field_errors %} +
+

{{ login_form.non_field_errors }}

+
+{% endif %} +
{% csrf_token %}
@@ -37,13 +43,40 @@
-
-

Book Identifiers

-

{{ form.isbn_13 }}

-

{{ form.isbn_10 }}

-

{{ form.openlibrary_key }}

-

{{ form.librarything_key }}

-

{{ form.goodreads_key }}

+
+

Metadata

+

{{ form.title }}

+ {% for error in form.title.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.sort_title }}

+ {% for error in form.sort_title.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.subtitle }}

+ {% for error in form.subtitle.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.description }}

+ {% for error in form.description.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.series }}

+ {% for error in form.series.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.series_number }}

+ {% for error in form.series_number.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.first_published_date }}

+ {% for error in form.first_published_date.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.published_date }}

+ {% for error in form.published_date.errors %} +

{{ error | escape }}

+ {% endfor %}
@@ -55,6 +88,9 @@

Cover

{{ form.cover }}

+ {% for error in form.cover.errors %} +

{{ error | escape }}

+ {% endfor %}
@@ -62,22 +98,45 @@

Physical Properties

{{ form.physical_format }}

+ {% for error in form.physical_format.errors %} +

{{ error | escape }}

+ {% endfor %} + {% for error in form.physical_format.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.pages }}

+ {% for error in form.pages.errors %} +

{{ error | escape }}

+ {% endfor %} +
+ +
+

Book Identifiers

+

{{ form.isbn_13 }}

+ {% for error in form.isbn_13.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.isbn_10 }}

+ {% for error in form.isbn_10.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.openlibrary_key }}

+ {% for error in form.openlibrary_key.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.librarything_key }}

+ {% for error in form.librarything_key.errors %} +

{{ error | escape }}

+ {% endfor %} +

{{ form.goodreads_key }}

+ {% for error in form.goodreads_key.errors %} +

{{ error | escape }}

+ {% endfor %}
-
-

Metadata

-

{{ form.title }}

-

{{ form.sort_title }}

-

{{ form.subtitle }}

-

{{ form.description }}

-

{{ form.series }}

-

{{ form.series_number }}

-

{{ form.first_published_date }}

-

{{ form.published_date }}

-
Cancel @@ -85,4 +144,3 @@ {% endblock %} - diff --git a/bookwyrm/templates/feed.html b/bookwyrm/templates/feed.html index 474f4fbc9..b4a810635 100644 --- a/bookwyrm/templates/feed.html +++ b/bookwyrm/templates/feed.html @@ -68,6 +68,7 @@
+

{{ tab | title }} Timeline