From 94fc365a520d6a518aae0c8d573b94453e014eca Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 5 Oct 2020 16:07:37 -0700 Subject: [PATCH 001/356] Allow logo upload --- .../migrations/0053_auto_20201005_2233.py | 28 +++++++++++++++++++ bookwyrm/models/site.py | 12 ++++++++ 2 files changed, 40 insertions(+) create mode 100644 bookwyrm/migrations/0053_auto_20201005_2233.py diff --git a/bookwyrm/migrations/0053_auto_20201005_2233.py b/bookwyrm/migrations/0053_auto_20201005_2233.py new file mode 100644 index 00000000..ce944af2 --- /dev/null +++ b/bookwyrm/migrations/0053_auto_20201005_2233.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.7 on 2020-10-05 22:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0052_auto_20201005_2145'), + ] + + operations = [ + migrations.AddField( + model_name='sitesettings', + name='favicon', + field=models.ImageField(default='/static/images/favicon.ico', upload_to='static/images/'), + ), + migrations.AddField( + model_name='sitesettings', + name='logo', + field=models.ImageField(default='/static/images/logo.png', upload_to='static/images/'), + ), + migrations.AddField( + model_name='sitesettings', + name='logo_small', + field=models.ImageField(default='/static/images/logo-small.png', upload_to='static/images/'), + ), + ] diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index 24eb673c..e4e893da 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -17,6 +17,18 @@ class SiteSettings(models.Model): code_of_conduct = models.TextField( default="Add a code of conduct here.") allow_registration = models.BooleanField(default=True) + logo = models.ImageField( + upload_to='static/images/', + default='/static/images/logo.png' + ) + logo_small = models.ImageField( + upload_to='static/images/', + default='/static/images/logo-small.png' + ) + favicon = models.ImageField( + upload_to='static/images/', + default='/static/images/favicon.ico' + ) @classmethod def get(cls): From ebb82287c2cc2c8e5ebfc8437515b862b9f0b467 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 27 Nov 2020 22:10:38 -0800 Subject: [PATCH 002/356] First pass at recursively resolving foreign keys --- bookwyrm/activitypub/__init__.py | 1 - bookwyrm/activitypub/base_activity.py | 59 +++++++++++++++------------ bookwyrm/models/status.py | 4 +- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py index 85245929..c97e8c49 100644 --- a/bookwyrm/activitypub/__init__.py +++ b/bookwyrm/activitypub/__init__.py @@ -5,7 +5,6 @@ import sys from .base_activity import ActivityEncoder, PublicKey, Signature from .base_activity import Link, Mention from .base_activity import ActivitySerializerError -from .base_activity import tag_formatter from .image import Image from .note import Note, GeneratedNote, Article, Comment, Review, Quotation from .note import Tombstone diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index caa4aeb8..e4d95b0e 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -25,12 +25,13 @@ class ActivityEncoder(JSONEncoder): @dataclass -class Link(): +class Link: ''' for tagging a book in a status ''' href: str name: str type: str = 'Link' + @dataclass class Mention(Link): ''' a subtype of Link for mentioning an actor ''' @@ -125,7 +126,7 @@ class ActivityObject: with transaction.atomic(): if instance: - # updating an existing model isntance + # updating an existing model instance for k, v in mapped_fields.items(): setattr(instance, k, v) instance.save() @@ -133,6 +134,9 @@ class ActivityObject: # creating a new model instance instance = model.objects.create(**mapped_fields) + # --- these are all fields that can't be saved until after the + # instance has an id (after it's been saved). ---------------# + # add images for (model_key, value) in image_fields.items(): formatted_value = image_formatter(value) @@ -140,9 +144,20 @@ class ActivityObject: continue getattr(instance, model_key).save(*formatted_value, save=True) + # add many to many fields for (model_key, values) in many_to_many_fields.items(): # mention books, mention users - getattr(instance, model_key).set(values) + if values == MISSING: + continue + model_field = getattr(instance, model_key) + model = model_field.model + items = [] + for link in values: + items.append( + resolve_foreign_key(model, link.get('href')) + ) + getattr(instance, model_key).set(items) + # add one to many fields for (model_key, values) in one_to_many_fields.items(): @@ -177,36 +192,30 @@ def resolve_foreign_key(model, remote_id): if hasattr(model.objects, 'select_subclasses'): result = result.select_subclasses() + # first, check for an existing copy in the database result = result.filter( remote_id=remote_id ).first() + if result: + return result - if not result: + # failing that, load the data and create the object + try: + response = requests.get( + remote_id, + headers={'Accept': 'application/json; charset=utf-8'}, + ) + except ConnectionError: + raise ActivitySerializerError( + 'Could not connect to host for remote_id in %s model: %s' % \ + (model.__name__, remote_id)) + if not response.ok: raise ActivitySerializerError( 'Could not resolve remote_id in %s model: %s' % \ (model.__name__, remote_id)) - return result - -def tag_formatter(tags, tag_type): - ''' helper function to extract foreign keys from tag activity json ''' - if not isinstance(tags, list): - return [] - items = [] - types = { - 'Book': models.Book, - 'Mention': models.User, - } - for tag in [t for t in tags if t.get('type') == tag_type]: - if not tag_type in types: - continue - remote_id = tag.get('href') - try: - item = resolve_foreign_key(types[tag_type], remote_id) - except ActivitySerializerError: - continue - items.append(item) - return items + item = model.activity_serializer(**response.json()) + return item.to_model(model) def image_formatter(image_slug): diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 9d45379c..b55d2da6 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -80,12 +80,12 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): ActivityMapping( 'tag', 'mention_books', lambda x: tag_formatter(x, 'title', 'Book'), - lambda x: activitypub.tag_formatter(x, 'Book') + lambda x: [i for i in x if x.get('type') == 'Book'] ), ActivityMapping( 'tag', 'mention_users', lambda x: tag_formatter(x, 'username', 'Mention'), - lambda x: activitypub.tag_formatter(x, 'Mention') + lambda x: [i for i in x if x.get('type') == 'Mention'] ), ActivityMapping( 'attachment', 'attachments', From a93b5cf5bc2505f382cd02d8f9821417e395e6b5 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 10:18:24 -0800 Subject: [PATCH 003/356] Use remote_id resolver to load books, user --- bookwyrm/activitypub/__init__.py | 4 +- bookwyrm/activitypub/base_activity.py | 50 ++++++++++++++----- bookwyrm/activitypub/book.py | 26 +++++----- bookwyrm/activitypub/ordered_collection.py | 1 + bookwyrm/activitypub/verbs.py | 8 +++ bookwyrm/books_manager.py | 17 ------- bookwyrm/incoming.py | 14 +----- .../migrations/0016_auto_20201128_1804.py | 24 +++++++++ bookwyrm/models/base_model.py | 4 ++ bookwyrm/models/book.py | 7 +-- bookwyrm/models/shelf.py | 18 ++++++- bookwyrm/models/status.py | 2 - bookwyrm/models/user.py | 1 - bookwyrm/remote_user.py | 29 +---------- bookwyrm/views.py | 3 +- 15 files changed, 115 insertions(+), 93 deletions(-) create mode 100644 bookwyrm/migrations/0016_auto_20201128_1804.py diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py index c97e8c49..eee9345d 100644 --- a/bookwyrm/activitypub/__init__.py +++ b/bookwyrm/activitypub/__init__.py @@ -4,7 +4,7 @@ import sys from .base_activity import ActivityEncoder, PublicKey, Signature from .base_activity import Link, Mention -from .base_activity import ActivitySerializerError +from .base_activity import ActivitySerializerError, resolve_remote_id from .image import Image from .note import Note, GeneratedNote, Article, Comment, Review, Quotation from .note import Tombstone @@ -14,7 +14,7 @@ from .person import Person from .book import Edition, Work, Author from .verbs import Create, Delete, Undo, Update from .verbs import Follow, Accept, Reject -from .verbs import Add, Remove +from .verbs import Add, AddBook, Remove # this creates a list of all the Activity types that we can serialize, # so when an Activity comes in from outside, we can check if it's known diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index e4d95b0e..dd46753f 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -3,15 +3,20 @@ from dataclasses import dataclass, fields, MISSING from json import JSONEncoder from uuid import uuid4 +import dateutil.parser +from dateutil.parser import ParserError from django.core.files.base import ContentFile from django.db import transaction from django.db.models.fields.related_descriptors \ import ForwardManyToOneDescriptor, ManyToManyDescriptor, \ ReverseManyToOneDescriptor +from django.db.models.fields import DateTimeField from django.db.models.fields.files import ImageFileDescriptor +from django.db.models.query_utils import DeferredAttribute +from django.utils import timezone import requests -from bookwyrm import books_manager, models +from bookwyrm import models class ActivitySerializerError(ValueError): @@ -106,11 +111,27 @@ class ActivityObject: model_field = getattr(model, mapping.model_key) formatted_value = mapping.model_formatter(value) - if isinstance(model_field, ForwardManyToOneDescriptor) and \ + if isinstance(model_field, DeferredAttribute) and \ + isinstance(model_field.field, DateTimeField): + print("DATE") + try: + formatted_value = timezone.make_aware( + dateutil.parser.parse(formatted_value) + ) + except ParserError: + formatted_value = None + elif isinstance(model_field, ForwardManyToOneDescriptor) and \ formatted_value: # foreign key remote id reolver (work on Edition, for example) fk_model = model_field.field.related_model - reference = resolve_foreign_key(fk_model, formatted_value) + if isinstance(formatted_value, dict) and \ + formatted_value.get('id'): + # if the AP field is a serialized object (as in Add) + remote_id = formatted_value['id'] + else: + # if the AP field is just a remote_id (as in every other case) + remote_id = formatted_value + reference = resolve_remote_id(fk_model, remote_id) mapped_fields[mapping.model_key] = reference elif isinstance(model_field, ManyToManyDescriptor): # status mentions book/users @@ -122,6 +143,8 @@ class ActivityObject: # image fields need custom handling image_fields[mapping.model_key] = formatted_value else: + if formatted_value == MISSING: + formatted_value = None mapped_fields[mapping.model_key] = formatted_value with transaction.atomic(): @@ -153,12 +176,15 @@ class ActivityObject: model = model_field.model items = [] for link in values: + # check that the Type matches the model (because Status + # tags contain both user mentions and book tags) + if not model.activity_serializer.type == link.get('type'): + continue items.append( - resolve_foreign_key(model, link.get('href')) + resolve_remote_id(model, link.get('href')) ) getattr(instance, model_key).set(items) - # add one to many fields for (model_key, values) in one_to_many_fields.items(): if values == MISSING: @@ -183,11 +209,8 @@ class ActivityObject: return data -def resolve_foreign_key(model, remote_id): - ''' look up the remote_id on an activity json field ''' - if model in [models.Edition, models.Work, models.Book]: - return books_manager.get_or_create_book(remote_id) - +def resolve_remote_id(model, remote_id, refresh=False): + ''' look up the remote_id in the database or load it remotely ''' result = model.objects if hasattr(model.objects, 'select_subclasses'): result = result.select_subclasses() @@ -196,10 +219,10 @@ def resolve_foreign_key(model, remote_id): result = result.filter( remote_id=remote_id ).first() - if result: + if result and not refresh: return result - # failing that, load the data and create the object + # load the data and create the object try: response = requests.get( remote_id, @@ -215,7 +238,8 @@ def resolve_foreign_key(model, remote_id): (model.__name__, remote_id)) item = model.activity_serializer(**response.json()) - return item.to_model(model) + # if we're refreshing, "result" will be set and we'll update it + return item.to_model(model, instance=result) def image_formatter(image_slug): diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py index 02cab281..8a6b88d9 100644 --- a/bookwyrm/activitypub/book.py +++ b/bookwyrm/activitypub/book.py @@ -12,13 +12,13 @@ class Book(ActivityObject): sortTitle: str = '' subtitle: str = '' description: str = '' - languages: List[str] + languages: List[str] = field(default_factory=lambda: []) series: str = '' seriesNumber: str = '' - subjects: List[str] - subjectPlaces: List[str] + subjects: List[str] = field(default_factory=lambda: []) + subjectPlaces: List[str] = field(default_factory=lambda: []) - authors: List[str] + authors: List[str] = field(default_factory=lambda: []) firstPublishedDate: str = '' publishedDate: str = '' @@ -33,22 +33,22 @@ class Book(ActivityObject): @dataclass(init=False) class Edition(Book): ''' Edition instance of a book object ''' - isbn10: str - isbn13: str - oclcNumber: str - asin: str - pages: str - physicalFormat: str - publishers: List[str] - work: str + isbn10: str = '' + isbn13: str = '' + oclcNumber: str = '' + asin: str = '' + pages: str = '' + physicalFormat: str = '' + publishers: List[str] = field(default_factory=lambda: []) + type: str = 'Edition' @dataclass(init=False) class Work(Book): ''' work instance of a book object ''' - lccn: str + lccn: str = '' editions: List[str] type: str = 'Work' diff --git a/bookwyrm/activitypub/ordered_collection.py b/bookwyrm/activitypub/ordered_collection.py index efd23d5a..9aeaf664 100644 --- a/bookwyrm/activitypub/ordered_collection.py +++ b/bookwyrm/activitypub/ordered_collection.py @@ -12,6 +12,7 @@ class OrderedCollection(ActivityObject): first: str last: str = '' name: str = '' + owner: str = '' type: str = 'OrderedCollection' diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py index eb166260..e890d81f 100644 --- a/bookwyrm/activitypub/verbs.py +++ b/bookwyrm/activitypub/verbs.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from typing import List from .base_activity import ActivityObject, Signature +from .book import Book @dataclass(init=False) class Verb(ActivityObject): @@ -69,6 +70,13 @@ class Add(Verb): type: str = 'Add' +@dataclass(init=False) +class AddBook(Verb): + '''Add activity that's aware of the book obj ''' + target: Book + type: str = 'Add' + + @dataclass(init=False) class Remove(Verb): '''Remove activity ''' diff --git a/bookwyrm/books_manager.py b/bookwyrm/books_manager.py index 461017a0..fb0a747e 100644 --- a/bookwyrm/books_manager.py +++ b/bookwyrm/books_manager.py @@ -16,23 +16,6 @@ def get_edition(book_id): return book -def get_or_create_book(remote_id): - ''' pull up a book record by whatever means possible ''' - book = models.Book.objects.select_subclasses().filter( - remote_id=remote_id - ).first() - if book: - return book - - connector = get_or_create_connector(remote_id) - - # raises ConnectorException - book = connector.get_or_create_book(remote_id) - if book: - load_more_data.delay(book.id) - return book - - def get_or_create_connector(remote_id): ''' get the connector related to the author's server ''' url = urlparse(remote_id) diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 0e7c1856..9a348483 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -317,7 +317,7 @@ def handle_tag(activity): user = get_or_create_remote_user(activity['actor']) if not user.local: # ordered collection weirndess so we can't just to_model - book = books_manager.get_or_create_book(activity['object']['id']) + book = (activity['object']['id']) name = activity['object']['target'].split('/')[-1] name = unquote_plus(name) models.Tag.objects.get_or_create( @@ -330,17 +330,7 @@ def handle_tag(activity): @app.task def handle_shelve(activity): ''' putting a book on a shelf ''' - user = get_or_create_remote_user(activity['actor']) - book = books_manager.get_or_create_book(activity['object']) - try: - shelf = models.Shelf.objects.get(remote_id=activity['target']) - except models.Shelf.DoesNotExist: - return - if shelf.user != user: - # this doesn't add up. - return - shelf.books.add(book) - shelf.save() + activitypub.AddBook(**activity).to_model(models.ShelfBook) @app.task diff --git a/bookwyrm/migrations/0016_auto_20201128_1804.py b/bookwyrm/migrations/0016_auto_20201128_1804.py new file mode 100644 index 00000000..9becde82 --- /dev/null +++ b/bookwyrm/migrations/0016_auto_20201128_1804.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.7 on 2020-11-28 18:04 + +import bookwyrm.utils.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0015_auto_20201128_0349'), + ] + + operations = [ + migrations.AlterField( + model_name='book', + name='subject_places', + field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='book', + name='subjects', + field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + ] diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 4109a49b..29301fb5 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -227,12 +227,16 @@ class OrderedCollectionPageMixin(ActivitypubMixin): name = '' if hasattr(self, 'name'): name = self.name + owner = '' + if hasattr(self, 'user'): + owner = self.user.remote_id size = queryset.count() return activitypub.OrderedCollection( id=remote_id, totalItems=size, name=name, + owner=owner, first='%s%s' % (remote_id, self.page()), last='%s%s' % (remote_id, self.page(min_id=0)) ).serialize() diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 132b4c07..8892119a 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -41,10 +41,10 @@ class Book(ActivitypubMixin, BookWyrmModel): series = models.CharField(max_length=255, blank=True, null=True) series_number = models.CharField(max_length=255, blank=True, null=True) subjects = ArrayField( - models.CharField(max_length=255), blank=True, default=list + models.CharField(max_length=255), blank=True, null=True, default=list ) subject_places = ArrayField( - models.CharField(max_length=255), blank=True, default=list + models.CharField(max_length=255), blank=True, null=True, default=list ) # TODO: include an annotation about the type of authorship (ie, translator) authors = models.ManyToManyField('Author') @@ -132,7 +132,8 @@ class Work(OrderedCollectionPageMixin, Book): ''' it'd be nice to serialize the edition instead but, recursion ''' default = self.default_edition ed_list = [ - e.remote_id for e in self.edition_set.filter(~Q(id=default.id)).all() + e.remote_id for e in \ + self.edition_set.filter(~Q(id=default.id)).all() ] return [default.remote_id] + ed_list diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py index e85294ba..3f443f6d 100644 --- a/bookwyrm/models/shelf.py +++ b/bookwyrm/models/shelf.py @@ -3,7 +3,8 @@ import re from django.db import models from bookwyrm import activitypub -from .base_model import BookWyrmModel, OrderedCollectionMixin, PrivacyLevels +from .base_model import ActivityMapping, BookWyrmModel +from .base_model import OrderedCollectionMixin, PrivacyLevels class Shelf(OrderedCollectionMixin, BookWyrmModel): @@ -47,6 +48,12 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): ''' user/shelf unqiueness ''' unique_together = ('user', 'identifier') + activity_mappings = [ + ActivityMapping('id', 'remote_id'), + ActivityMapping('owner', 'user'), + ActivityMapping('name', 'name'), + ] + class ShelfBook(BookWyrmModel): ''' many to many join table for books and shelves ''' @@ -59,6 +66,15 @@ class ShelfBook(BookWyrmModel): on_delete=models.PROTECT ) + activity_mappings = [ + ActivityMapping('id', 'remote_id'), + ActivityMapping('actor', 'added_by'), + ActivityMapping('object', 'book'), + ActivityMapping('target', 'shelf') + ] + + activity_serializer = activitypub.AddBook + def to_add_activity(self, user): ''' AP for shelving a book''' return activitypub.Add( diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index b55d2da6..a520164a 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -80,12 +80,10 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): ActivityMapping( 'tag', 'mention_books', lambda x: tag_formatter(x, 'title', 'Book'), - lambda x: [i for i in x if x.get('type') == 'Book'] ), ActivityMapping( 'tag', 'mention_users', lambda x: tag_formatter(x, 'username', 'Mention'), - lambda x: [i for i in x if x.get('type') == 'Mention'] ), ActivityMapping( 'attachment', 'attachments', diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 4d511d56..a9812037 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -11,7 +11,6 @@ from bookwyrm.models.status import Status from bookwyrm.settings import DOMAIN from bookwyrm.signatures import create_key_pair from .base_model import ActivityMapping, OrderedCollectionPageMixin -from .base_model import image_formatter class User(OrderedCollectionPageMixin, AbstractUser): diff --git a/bookwyrm/remote_user.py b/bookwyrm/remote_user.py index 23a805b3..205b0893 100644 --- a/bookwyrm/remote_user.py +++ b/bookwyrm/remote_user.py @@ -16,11 +16,9 @@ def get_or_create_remote_user(actor): except models.User.DoesNotExist: pass - data = fetch_user_data(actor) - actor_parts = urlparse(actor) with transaction.atomic(): - user = activitypub.Person(**data).to_model(models.User) + user = activitypub.resolve_remote_id(models.User, actor) user.federated_server = get_or_create_remote_server(actor_parts.netloc) user.save() if user.bookwyrm_user: @@ -28,32 +26,9 @@ def get_or_create_remote_user(actor): return user -def fetch_user_data(actor): - ''' load the user's info from the actor url ''' - try: - response = requests.get( - actor, - headers={'Accept': 'application/activity+json'} - ) - except ConnectionError: - return None - - if not response.ok: - response.raise_for_status() - data = response.json() - - # make sure our actor is who they say they are - if actor != data['id']: - raise ValueError("Remote actor id must match url.") - return data - - def refresh_remote_user(user): ''' get updated user data from its home instance ''' - data = fetch_user_data(user.remote_id) - - activity = activitypub.Person(**data) - activity.to_model(models.User, instance=user) + activitypub.resolve_remote_id(user.remote_id, refresh=True) @app.task diff --git a/bookwyrm/views.py b/bookwyrm/views.py index e0feaee7..e2887255 100644 --- a/bookwyrm/views.py +++ b/bookwyrm/views.py @@ -5,8 +5,7 @@ from django.contrib.auth.decorators import login_required, permission_required from django.contrib.postgres.search import TrigramSimilarity from django.core.paginator import Paginator from django.db.models import Avg, Q -from django.http import HttpResponseBadRequest, HttpResponseNotFound,\ - JsonResponse +from django.http import HttpResponseNotFound, JsonResponse from django.core.exceptions import PermissionDenied from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse From fd7e476c9bec84c213d2e8f24a1911ecc126d7db Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 11:00:40 -0800 Subject: [PATCH 004/356] refactors tag model to fit ordered collection structure --- bookwyrm/activitypub/base_activity.py | 7 ++- bookwyrm/incoming.py | 31 ++++--------- .../migrations/0017_auto_20201128_1849.py | 42 ++++++++++++++++++ bookwyrm/models/__init__.py | 2 +- bookwyrm/models/shelf.py | 2 +- bookwyrm/models/tag.py | 43 ++++++++++++++----- bookwyrm/signatures.py | 2 +- bookwyrm/templates/snippets/tag.html | 10 ++--- bookwyrm/view_actions.py | 7 ++- bookwyrm/views.py | 16 +++---- 10 files changed, 110 insertions(+), 52 deletions(-) create mode 100644 bookwyrm/migrations/0017_auto_20201128_1849.py diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index dd46753f..a6204536 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -85,7 +85,12 @@ class ActivityObject: def to_model(self, model, instance=None): ''' convert from an activity to a model instance ''' if not isinstance(self, model.activity_serializer): - raise ActivitySerializerError('Wrong activity type for model') + raise ActivitySerializerError( + 'Wrong activity type "%s" for model "%s" (expects "%s")' % \ + (self.__class__, + model.__name__, + model.activity_serializer) + ) # check for an existing instance, if we're not updating a known obj if not instance: diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 9a348483..ef05cc4e 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -1,6 +1,6 @@ ''' handles all of the activity coming in to the server ''' import json -from urllib.parse import urldefrag, unquote_plus +from urllib.parse import urldefrag import django.db.utils from django.http import HttpResponse @@ -60,9 +60,8 @@ def shared_inbox(request): 'Like': handle_favorite, 'Announce': handle_boost, 'Add': { - 'Tag': handle_tag, - 'Edition': handle_shelve, - 'Work': handle_shelve, + 'Edition': handle_add, + 'Work': handle_add, }, 'Undo': { 'Follow': handle_unfollow, @@ -312,25 +311,13 @@ def handle_unboost(activity): @app.task -def handle_tag(activity): - ''' someone is tagging a book ''' - user = get_or_create_remote_user(activity['actor']) - if not user.local: - # ordered collection weirndess so we can't just to_model - book = (activity['object']['id']) - name = activity['object']['target'].split('/')[-1] - name = unquote_plus(name) - models.Tag.objects.get_or_create( - user=user, - book=book, - name=name - ) - - -@app.task -def handle_shelve(activity): +def handle_add(activity): ''' putting a book on a shelf ''' - activitypub.AddBook(**activity).to_model(models.ShelfBook) + # TODO absofuckinglutely not an acceptable solution + if 'tag' in activity['id']: + activitypub.AddBook(**activity).to_model(models.Tag) + else: + activitypub.AddBook(**activity).to_model(models.ShelfBook) @app.task diff --git a/bookwyrm/migrations/0017_auto_20201128_1849.py b/bookwyrm/migrations/0017_auto_20201128_1849.py new file mode 100644 index 00000000..722458b2 --- /dev/null +++ b/bookwyrm/migrations/0017_auto_20201128_1849.py @@ -0,0 +1,42 @@ +# Generated by Django 3.0.7 on 2020-11-28 18:49 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0016_auto_20201128_1804'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='tag', + unique_together=set(), + ), + migrations.RemoveField( + model_name='tag', + name='book', + ), + migrations.RemoveField( + model_name='tag', + name='user', + ), + migrations.CreateModel( + name='UserTag', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_date', models.DateTimeField(auto_now_add=True)), + ('updated_date', models.DateTimeField(auto_now=True)), + ('remote_id', models.CharField(max_length=255, null=True)), + ('book', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition')), + ('tag', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'book', 'tag')}, + }, + ), + ] diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py index 3d854478..9a4d6014 100644 --- a/bookwyrm/models/__init__.py +++ b/bookwyrm/models/__init__.py @@ -12,7 +12,7 @@ from .status import Status, GeneratedNote, Review, Comment, Quotation from .status import Favorite, Boost, Notification, ReadThrough from .attachment import Image -from .tag import Tag +from .tag import Tag, UserTag from .user import User from .relationship import UserFollows, UserFollowRequest, UserBlocks diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py index 3f443f6d..b8d5ea17 100644 --- a/bookwyrm/models/shelf.py +++ b/bookwyrm/models/shelf.py @@ -70,7 +70,7 @@ class ShelfBook(BookWyrmModel): ActivityMapping('id', 'remote_id'), ActivityMapping('actor', 'added_by'), ActivityMapping('object', 'book'), - ActivityMapping('target', 'shelf') + ActivityMapping('target', 'shelf'), ] activity_serializer = activitypub.AddBook diff --git a/bookwyrm/models/tag.py b/bookwyrm/models/tag.py index cd98e2b1..8b7efceb 100644 --- a/bookwyrm/models/tag.py +++ b/bookwyrm/models/tag.py @@ -5,16 +5,19 @@ from django.db import models from bookwyrm import activitypub from bookwyrm.settings import DOMAIN -from .base_model import OrderedCollectionMixin, BookWyrmModel +from .base_model import OrderedCollectionMixin, BookWyrmModel, ActivityMapping class Tag(OrderedCollectionMixin, BookWyrmModel): ''' freeform tags for books ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - book = models.ForeignKey('Edition', on_delete=models.PROTECT) - name = models.CharField(max_length=100) + name = models.CharField(max_length=100, unique=True) identifier = models.CharField(max_length=100) + activity_mappings = [ + ActivityMapping('id', 'remote_id'), + ActivityMapping('name', 'name'), + ] + @classmethod def book_queryset(cls, identifier): ''' county of books associated with this tag ''' @@ -30,6 +33,30 @@ class Tag(OrderedCollectionMixin, BookWyrmModel): base_path = 'https://%s' % DOMAIN return '%s/tag/%s' % (base_path, self.identifier) + + def save(self, *args, **kwargs): + ''' create a url-safe lookup key for the tag ''' + if not self.id: + # add identifiers to new tags + self.identifier = urllib.parse.quote_plus(self.name) + super().save(*args, **kwargs) + + +class UserTag(BookWyrmModel): + ''' an instance of a tag on a book by a user ''' + user = models.ForeignKey('User', on_delete=models.PROTECT) + book = models.ForeignKey('Edition', on_delete=models.PROTECT) + tag = models.ForeignKey('Tag', on_delete=models.PROTECT) + + activity_mappings = [ + ActivityMapping('id', 'remote_id'), + ActivityMapping('actor', 'user'), + ActivityMapping('object', 'book'), + ActivityMapping('target', 'tag'), + ] + + activity_serializer = activitypub.AddBook + def to_add_activity(self, user): ''' AP for shelving a book''' return activitypub.Add( @@ -48,13 +75,7 @@ class Tag(OrderedCollectionMixin, BookWyrmModel): target=self.to_activity(), ).serialize() - def save(self, *args, **kwargs): - ''' create a url-safe lookup key for the tag ''' - if not self.id: - # add identifiers to new tags - self.identifier = urllib.parse.quote_plus(self.name) - super().save(*args, **kwargs) class Meta: ''' unqiueness constraint ''' - unique_together = ('user', 'book', 'name') + unique_together = ('user', 'book', 'tag') diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py index 57c181df..dbb88d8a 100644 --- a/bookwyrm/signatures.py +++ b/bookwyrm/signatures.py @@ -89,7 +89,7 @@ class Signature: def verify(self, public_key, request): ''' verify rsa signature ''' - if http_date_age(request.headers['date']) > MAX_SIGNATURE_AGE: + if False:#http_date_age(request.headers['date']) > MAX_SIGNATURE_AGE: raise ValueError( "Request too old: %s" % (request.headers['date'],)) public_key = RSA.import_key(public_key) diff --git a/bookwyrm/templates/snippets/tag.html b/bookwyrm/templates/snippets/tag.html index e62167f9..482cffc3 100644 --- a/bookwyrm/templates/snippets/tag.html +++ b/bookwyrm/templates/snippets/tag.html @@ -1,14 +1,14 @@
-
+ {% csrf_token %} - +
- - {{ tag.name }} + + {{ tag.tag.name }} - {% if tag.identifier in user_tags %} + {% if tag.tag.identifier in user_tags %} diff --git a/bookwyrm/view_actions.py b/bookwyrm/view_actions.py index ca306bcb..0f51c66f 100644 --- a/bookwyrm/view_actions.py +++ b/bookwyrm/view_actions.py @@ -531,12 +531,15 @@ def tag(request): book = get_object_or_404(models.Edition, id=book_id) tag_obj, created = models.Tag.objects.get_or_create( name=name, + ) + user_tag = models.UserTag.objects.get_or_create( + user=request.user, book=book, - user=request.user + tag=tag_obj, ) if created: - outgoing.handle_tag(request.user, tag_obj) + outgoing.handle_tag(request.user, user_tag) return redirect('/book/%s' % book_id) diff --git a/bookwyrm/views.py b/bookwyrm/views.py index e2887255..c6992e6b 100644 --- a/bookwyrm/views.py +++ b/bookwyrm/views.py @@ -560,9 +560,9 @@ def book_page(request, book_id): user_tags = [] readthroughs = [] if request.user.is_authenticated: - user_tags = models.Tag.objects.filter( + user_tags = models.UserTag.objects.filter( book=book, user=request.user - ).values_list('identifier', flat=True) + ).values_list('tag__identifier', flat=True) readthroughs = models.ReadThrough.objects.filter( user=request.user, @@ -570,11 +570,9 @@ def book_page(request, book_id): ).order_by('start_date') rating = reviews.aggregate(Avg('rating')) - tags = models.Tag.objects.filter( - book=book - ).values( - 'book', 'name', 'identifier' - ).distinct().all() + tags = models.UserTag.objects.filter( + book=book, + ) data = { 'title': book.title, @@ -664,7 +662,9 @@ def tag_page(request, tag_id): return JsonResponse( tag_obj.to_activity(**request.GET), encoder=ActivityEncoder) - books = models.Edition.objects.filter(tag__identifier=tag_id).distinct() + books = models.Edition.objects.filter( + usertag__tag__identifier=tag_id + ).distinct() data = { 'title': tag_obj.name, 'books': books, From b0202eb8e8248907c650355a4b7b4b0f282d161c Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 11:48:17 -0800 Subject: [PATCH 005/356] Remove special remote user handling code also fixes date parsing --- bookwyrm/activitypub/base_activity.py | 9 +-- bookwyrm/incoming.py | 49 +++++++-------- bookwyrm/models/user.py | 69 ++++++++++++++++++++- bookwyrm/outgoing.py | 7 ++- bookwyrm/remote_user.py | 86 --------------------------- bookwyrm/status.py | 31 ---------- celerywyrm/celery.py | 2 +- 7 files changed, 102 insertions(+), 151 deletions(-) delete mode 100644 bookwyrm/remote_user.py diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index a6204536..0672fbb8 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -118,11 +118,12 @@ class ActivityObject: formatted_value = mapping.model_formatter(value) if isinstance(model_field, DeferredAttribute) and \ isinstance(model_field.field, DateTimeField): - print("DATE") try: - formatted_value = timezone.make_aware( - dateutil.parser.parse(formatted_value) - ) + date_value = dateutil.parser.parse(formatted_value) + try: + formatted_value = timezone.make_aware(date_value) + except ValueError: + formatted_value = date_value except ParserError: formatted_value = None elif isinstance(model_field, ForwardManyToOneDescriptor) and \ diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index ef05cc4e..6ce2ea0b 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -10,7 +10,6 @@ import requests from bookwyrm import activitypub, books_manager, models, outgoing from bookwyrm import status as status_builder -from bookwyrm.remote_user import get_or_create_remote_user, refresh_remote_user from bookwyrm.tasks import app from bookwyrm.signatures import Signature @@ -96,13 +95,15 @@ def has_valid_signature(request, activity): if key_actor != activity.get('actor'): raise ValueError("Wrong actor created signature.") - remote_user = get_or_create_remote_user(key_actor) + remote_user = activitypub.resolve_remote_id(models.User, key_actor) try: signature.verify(remote_user.public_key, request) except ValueError: old_key = remote_user.public_key - refresh_remote_user(remote_user) + activitypub.resolve_remote_id( + models.User, remote_user, refresh=True + ) if remote_user.public_key == old_key: raise # Key unchanged. signature.verify(remote_user.public_key, request) @@ -127,7 +128,7 @@ def handle_follow(activity): return # figure out who the actor is - actor = get_or_create_remote_user(activity['actor']) + actor = activitypub.resolve_remote_id(models.User, activity['actor']) try: relationship = models.UserFollowRequest.objects.create( user_subject=actor, @@ -162,7 +163,7 @@ def handle_follow(activity): def handle_unfollow(activity): ''' unfollow a local user ''' obj = activity['object'] - requester = get_or_create_remote_user(obj['actor']) + requester = activitypub.resolve_remote_id(models.user, obj['actor']) to_unfollow = models.User.objects.get(remote_id=obj['object']) # raises models.User.DoesNotExist @@ -175,7 +176,7 @@ def handle_follow_accept(activity): # figure out who they want to follow requester = models.User.objects.get(remote_id=activity['object']['actor']) # figure out who they are - accepter = get_or_create_remote_user(activity['actor']) + accepter = activitypub.resolve_remote_id(models.User, activity['actor']) try: request = models.UserFollowRequest.objects.get( @@ -192,7 +193,7 @@ def handle_follow_accept(activity): def handle_follow_reject(activity): ''' someone is rejecting a follow request ''' requester = models.User.objects.get(remote_id=activity['object']['actor']) - rejecter = get_or_create_remote_user(activity['actor']) + rejecter = activitypub.resolve_remote_id(models.User, activity['actor']) request = models.UserFollowRequest.objects.get( user_subject=requester, @@ -205,25 +206,27 @@ def handle_follow_reject(activity): @app.task def handle_create(activity): ''' someone did something, good on them ''' - if activity['object'].get('type') not in \ - ['Note', 'Comment', 'Quotation', 'Review', 'GeneratedNote']: - # if it's an article or unknown type, ignore it - return - - user = get_or_create_remote_user(activity['actor']) - if user.local: - # we really oughtn't even be sending in this case - return - # deduplicate incoming activities status_id = activity['object']['id'] if models.Status.objects.filter(remote_id=status_id).count(): return - status = status_builder.create_status(activity['object']) - if not status: + serializer = activitypub.activity_objects[activity['type']] + status = serializer(**activity) + try: + model = models.activity_models[activity.type] + except KeyError: + # not a type of status we are prepared to deserialize return + if activity.type == 'Note': + reply = models.Status.objects.filter( + remote_id=activity.inReplyTo + ).first() + if not reply: + return + + activity.to_model(model) # create a notification if this is a reply if status.reply_parent and status.reply_parent.user.local: status_builder.create_notification( @@ -257,16 +260,14 @@ def handle_favorite(activity): ''' approval of your good good post ''' fav = activitypub.Like(**activity) - liker = get_or_create_remote_user(activity['actor']) - if liker.local: - return - fav = fav.to_model(models.Favorite) + if fav.user.local: + return status_builder.create_notification( fav.status.user, 'FAVORITE', - related_user=liker, + related_user=fav.user, related_status=fav.status, ) diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index a9812037..a32a35b7 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -1,5 +1,6 @@ ''' database schema for user data ''' from urllib.parse import urlparse +import requests from django.contrib.auth.models import AbstractUser from django.db import models @@ -7,10 +8,12 @@ from django.dispatch import receiver from bookwyrm import activitypub from bookwyrm.models.shelf import Shelf -from bookwyrm.models.status import Status +from bookwyrm.models.status import Status, Review from bookwyrm.settings import DOMAIN from bookwyrm.signatures import create_key_pair +from bookwyrm.tasks import app from .base_model import ActivityMapping, OrderedCollectionPageMixin +from .federated_server import FederatedServer class User(OrderedCollectionPageMixin, AbstractUser): @@ -188,7 +191,16 @@ class User(OrderedCollectionPageMixin, AbstractUser): @receiver(models.signals.post_save, sender=User) def execute_after_save(sender, instance, created, *args, **kwargs): ''' create shelves for new users ''' - if not instance.local or not created: + if not created: + return + + if not instance.local: + actor_parts = urlparse(instance.remote_id) + instance.federated_server = \ + get_or_create_remote_server(actor_parts.netloc) + instance.save() + if instance.bookwyrm_user: + get_remote_reviews.delay(instance.outbox) return shelves = [{ @@ -209,3 +221,56 @@ def execute_after_save(sender, instance, created, *args, **kwargs): user=instance, editable=False ).save() + + +def get_or_create_remote_server(domain): + ''' get info on a remote server ''' + try: + return FederatedServer.objects.get( + server_name=domain + ) + except FederatedServer.DoesNotExist: + pass + + response = requests.get( + 'https://%s/.well-known/nodeinfo' % domain, + headers={'Accept': 'application/activity+json'} + ) + + if response.status_code != 200: + return None + + data = response.json() + try: + nodeinfo_url = data.get('links')[0].get('href') + except (TypeError, KeyError): + return None + + response = requests.get( + nodeinfo_url, + headers={'Accept': 'application/activity+json'} + ) + data = response.json() + + server = FederatedServer.objects.create( + server_name=domain, + application_type=data['software']['name'], + application_version=data['software']['version'], + ) + return server + + +@app.task +def get_remote_reviews(outbox): + ''' ingest reviews by a new remote bookwyrm user ''' + outbox_page = outbox + '?page=true' + response = requests.get( + outbox_page, + headers={'Accept': 'application/activity+json'} + ) + data = response.json() + # TODO: pagination? + for activity in data['orderedItems']: + if not activity['type'] == 'Review': + continue + activitypub.Review(**activity).to_model(Review) diff --git a/bookwyrm/outgoing.py b/bookwyrm/outgoing.py index a196fcec..545ac491 100644 --- a/bookwyrm/outgoing.py +++ b/bookwyrm/outgoing.py @@ -12,7 +12,6 @@ from bookwyrm.broadcast import broadcast from bookwyrm.status import create_notification from bookwyrm.status import create_generated_note from bookwyrm.status import delete_status -from bookwyrm.remote_user import get_or_create_remote_user from bookwyrm.settings import DOMAIN from bookwyrm.utils import regex @@ -61,9 +60,11 @@ def handle_remote_webfinger(query): return None data = response.json() for link in data['links']: - if link['rel'] == 'self': + if link.get('rel') == 'self': try: - user = get_or_create_remote_user(link['href']) + user = activitypub.resolve_remote_id( + models.User, link['href'] + ) except KeyError: return None return user diff --git a/bookwyrm/remote_user.py b/bookwyrm/remote_user.py deleted file mode 100644 index 205b0893..00000000 --- a/bookwyrm/remote_user.py +++ /dev/null @@ -1,86 +0,0 @@ -''' manage remote users ''' -from urllib.parse import urlparse -import requests - -from django.db import transaction - -from bookwyrm import activitypub, models -from bookwyrm import status as status_builder -from bookwyrm.tasks import app - - -def get_or_create_remote_user(actor): - ''' look up a remote user or add them ''' - try: - return models.User.objects.get(remote_id=actor) - except models.User.DoesNotExist: - pass - - actor_parts = urlparse(actor) - with transaction.atomic(): - user = activitypub.resolve_remote_id(models.User, actor) - user.federated_server = get_or_create_remote_server(actor_parts.netloc) - user.save() - if user.bookwyrm_user: - get_remote_reviews.delay(user.id) - return user - - -def refresh_remote_user(user): - ''' get updated user data from its home instance ''' - activitypub.resolve_remote_id(user.remote_id, refresh=True) - - -@app.task -def get_remote_reviews(user_id): - ''' ingest reviews by a new remote bookwyrm user ''' - try: - user = models.User.objects.get(id=user_id) - except models.User.DoesNotExist: - return - outbox_page = user.outbox + '?page=true' - response = requests.get( - outbox_page, - headers={'Accept': 'application/activity+json'} - ) - data = response.json() - # TODO: pagination? - for activity in data['orderedItems']: - status_builder.create_status(activity) - - -def get_or_create_remote_server(domain): - ''' get info on a remote server ''' - try: - return models.FederatedServer.objects.get( - server_name=domain - ) - except models.FederatedServer.DoesNotExist: - pass - - response = requests.get( - 'https://%s/.well-known/nodeinfo' % domain, - headers={'Accept': 'application/activity+json'} - ) - - if response.status_code != 200: - return None - - data = response.json() - try: - nodeinfo_url = data.get('links')[0].get('href') - except (TypeError, KeyError): - return None - - response = requests.get( - nodeinfo_url, - headers={'Accept': 'application/activity+json'} - ) - data = response.json() - - server = models.FederatedServer.objects.create( - server_name=domain, - application_type=data['software']['name'], - application_version=data['software']['version'], - ) - return server diff --git a/bookwyrm/status.py b/bookwyrm/status.py index 6a86209f..83a106e5 100644 --- a/bookwyrm/status.py +++ b/bookwyrm/status.py @@ -12,37 +12,6 @@ def delete_status(status): status.save() -def create_status(activity): - ''' unfortunately, it's not QUITE as simple as deserializing it ''' - # render the json into an activity object - serializer = activitypub.activity_objects[activity['type']] - activity = serializer(**activity) - try: - model = models.activity_models[activity.type] - except KeyError: - # not a type of status we are prepared to deserialize - return None - - # ignore notes that aren't replies to known statuses - if activity.type == 'Note': - reply = models.Status.objects.filter( - remote_id=activity.inReplyTo - ).first() - if not reply: - return None - - # look up books - book_urls = [] - if hasattr(activity, 'inReplyToBook'): - book_urls.append(activity.inReplyToBook) - if hasattr(activity, 'tag'): - book_urls += [t['href'] for t in activity.tag if t['type'] == 'Book'] - for remote_id in book_urls: - books_manager.get_or_create_book(remote_id) - - return activity.to_model(model) - - def create_generated_note(user, content, mention_books=None, privacy='public'): ''' a note created by the app about user activity ''' # sanitize input html diff --git a/celerywyrm/celery.py b/celerywyrm/celery.py index 361b76ec..a47aad32 100644 --- a/celerywyrm/celery.py +++ b/celerywyrm/celery.py @@ -24,4 +24,4 @@ app.autodiscover_tasks(['bookwyrm'], related_name='books_manager') app.autodiscover_tasks(['bookwyrm'], related_name='emailing') app.autodiscover_tasks(['bookwyrm'], related_name='goodreads_import') app.autodiscover_tasks(['bookwyrm'], related_name='incoming') -app.autodiscover_tasks(['bookwyrm'], related_name='remote_user') +app.autodiscover_tasks(['bookwyrm'], related_name='models.user') From 76ce20a5e0a1b3b69fb367a31b9b6c258e4c9081 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 11:52:00 -0800 Subject: [PATCH 006/356] Fixes tests --- bookwyrm/tests/test_remote_user.py | 27 --------------------------- bookwyrm/tests/test_signing.py | 10 +++++----- 2 files changed, 5 insertions(+), 32 deletions(-) delete mode 100644 bookwyrm/tests/test_remote_user.py diff --git a/bookwyrm/tests/test_remote_user.py b/bookwyrm/tests/test_remote_user.py deleted file mode 100644 index febf9f67..00000000 --- a/bookwyrm/tests/test_remote_user.py +++ /dev/null @@ -1,27 +0,0 @@ -import json -import pathlib -from django.test import TestCase - -from bookwyrm import models, remote_user - - -class RemoteUser(TestCase): - ''' not too much going on in the books model but here we are ''' - def setUp(self): - self.remote_user = models.User.objects.create_user( - 'rat', 'rat@rat.com', 'ratword', - local=False, - remote_id='https://example.com/users/rat', - inbox='https://example.com/users/rat/inbox', - outbox='https://example.com/users/rat/outbox', - ) - datafile = pathlib.Path(__file__).parent.joinpath( - 'data/ap_user.json' - ) - self.user_data = json.loads(datafile.read_bytes()) - - - def test_get_remote_user(self): - actor = 'https://example.com/users/rat' - user = remote_user.get_or_create_remote_user(actor) - self.assertEqual(user, self.remote_user) diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py index a5039306..129a4333 100644 --- a/bookwyrm/tests/test_signing.py +++ b/bookwyrm/tests/test_signing.py @@ -107,7 +107,7 @@ class Signature(TestCase): status=200 ) - with patch('bookwyrm.remote_user.get_remote_reviews.delay') as _: + with patch('bookwyrm.models.user.get_remote_reviews.delay') as _: response = self.send_test_request(sender=self.fake_remote) self.assertEqual(response.status_code, 200) @@ -145,7 +145,7 @@ class Signature(TestCase): json=data, status=200) - with patch('bookwyrm.remote_user.get_remote_reviews.delay') as _: + with patch('bookwyrm.models.user.get_remote_reviews.delay') as _: # Key correct: response = self.send_test_request(sender=self.fake_remote) self.assertEqual(response.status_code, 200) @@ -177,7 +177,7 @@ class Signature(TestCase): @pytest.mark.integration def test_changed_data(self): '''Message data must match the digest header.''' - with patch('bookwyrm.remote_user.fetch_user_data') as _: + with patch('bookwyrm.activitypub.resolve_remote_id') as _: response = self.send_test_request( self.mouse, send_data=get_follow_data(self.mouse, self.cat)) @@ -185,7 +185,7 @@ class Signature(TestCase): @pytest.mark.integration def test_invalid_digest(self): - with patch('bookwyrm.remote_user.fetch_user_data') as _: + with patch('bookwyrm.activitypub.resolve_remote_id') as _: response = self.send_test_request( self.mouse, digest='SHA-256=AAAAAAAAAAAAAAAAAA') @@ -194,7 +194,7 @@ class Signature(TestCase): @pytest.mark.integration def test_old_message(self): '''Old messages should be rejected to prevent replay attacks.''' - with patch('bookwyrm.remote_user.fetch_user_data') as _: + with patch('bookwyrm.activitypub.resolve_remote_id') as _: response = self.send_test_request( self.mouse, date=http_date(time.time() - 301) From e99394e6f7c2cd732938f059c59ad1c030c0a5dc Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 12:25:53 -0800 Subject: [PATCH 007/356] User serializer to create follow request --- bookwyrm/incoming.py | 43 +++++++++++-------------------------------- 1 file changed, 11 insertions(+), 32 deletions(-) diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 6ce2ea0b..e9b0d019 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -115,26 +115,10 @@ def has_valid_signature(request, activity): @app.task def handle_follow(activity): ''' someone wants to follow a local user ''' - # figure out who they want to follow -- not using get_or_create because - # we only care if you want to follow local users try: - to_follow = models.User.objects.get(remote_id=activity['object']) - except models.User.DoesNotExist: - # some rando, who cares - return - if not to_follow.local: - # just ignore follow alerts about other servers. maybe they should be - # handled. maybe they shouldn't be sent at all. - return - - # figure out who the actor is - actor = activitypub.resolve_remote_id(models.User, activity['actor']) - try: - relationship = models.UserFollowRequest.objects.create( - user_subject=actor, - user_object=to_follow, - remote_id=activity['id'] - ) + relationship = activitypub.Follow( + **activity + ).to_model(models.UserFollowRequest) except django.db.utils.IntegrityError as err: if err.__cause__.diag.constraint_name != 'userfollowrequest_unique': raise @@ -143,20 +127,15 @@ def handle_follow(activity): ) # send the accept normally for a duplicate request - if not to_follow.manually_approves_followers: - status_builder.create_notification( - to_follow, - 'FOLLOW', - related_user=actor - ) + manually_approves = relationship.user_object.manually_approves_followers + + status_builder.create_notification( + relationship.user_object, + 'FOLLOW_REQUEST' if manually_approves else 'FOLLOW', + related_user=relationship.user_subject + ) + if not manually_approves: outgoing.handle_accept(relationship) - else: - # Accept will be triggered manually - status_builder.create_notification( - to_follow, - 'FOLLOW_REQUEST', - related_user=actor - ) @app.task From 421a13fda06de6dd4466130ecae52823f930eccd Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 13:00:04 -0800 Subject: [PATCH 008/356] automatically load authors and editions --- bookwyrm/activitypub/base_activity.py | 17 +++++++++++------ bookwyrm/models/base_model.py | 9 +++++++-- bookwyrm/models/book.py | 19 ++----------------- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 0672fbb8..620579a0 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -124,7 +124,7 @@ class ActivityObject: formatted_value = timezone.make_aware(date_value) except ValueError: formatted_value = date_value - except ParserError: + except (ParserError, TypeError): formatted_value = None elif isinstance(model_field, ForwardManyToOneDescriptor) and \ formatted_value: @@ -182,12 +182,17 @@ class ActivityObject: model = model_field.model items = [] for link in values: - # check that the Type matches the model (because Status - # tags contain both user mentions and book tags) - if not model.activity_serializer.type == link.get('type'): - continue + if isinstance(link, dict): + # check that the Type matches the model (Status + # tags contain both user mentions and book tags) + if not model.activity_serializer.type == \ + link.get('type'): + continue + remote_id = link.get('href') + else: + remote_id = link items.append( - resolve_remote_id(model, link.get('href')) + resolve_remote_id(model, remote_id) ) getattr(instance, model_key).set(items) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 29301fb5..877f094e 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -10,7 +10,9 @@ from Crypto.PublicKey import RSA from Crypto.Signature import pkcs1_15 from Crypto.Hash import SHA256 from django.db import models -from django.db.models.fields.files import ImageFieldFile +from django.db.models.fields.files import ImageFileDescriptor +from django.db.models.fields.related_descriptors \ + import ManyToManyDescriptor from django.dispatch import receiver from bookwyrm import activitypub @@ -72,13 +74,16 @@ class ActivitypubMixin: # this field on the model isn't serialized continue value = getattr(self, mapping.model_key) + model_field_type = getattr(self.__class__, mapping.model_key) if hasattr(value, 'remote_id'): # this is probably a foreign key field, which we want to # serialize as just the remote_id url reference value = value.remote_id + elif isinstance(model_field_type, ManyToManyDescriptor): + value = [i.remote_id for i in value.all()] elif isinstance(value, datetime): value = value.isoformat() - elif isinstance(value, ImageFieldFile): + elif isinstance(model_field_type, ImageFileDescriptor): value = image_formatter(value) # run the custom formatter function set in the model diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 8892119a..70eb8652 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -55,15 +55,10 @@ class Book(ActivitypubMixin, BookWyrmModel): published_date = models.DateTimeField(blank=True, null=True) objects = InheritanceManager() - @property - def ap_authors(self): - ''' the activitypub serialization should be a list of author ids ''' - return [a.remote_id for a in self.authors.all()] - activity_mappings = [ ActivityMapping('id', 'remote_id'), - ActivityMapping('authors', 'ap_authors'), + ActivityMapping('authors', 'authors'), ActivityMapping('firstPublishedDate', 'firstpublished_date'), ActivityMapping('publishedDate', 'published_date'), @@ -91,7 +86,7 @@ class Book(ActivitypubMixin, BookWyrmModel): ActivityMapping('publishers', 'publishers'), ActivityMapping('lccn', 'lccn'), - ActivityMapping('editions', 'editions_path'), + ActivityMapping('editions', 'editions'), ActivityMapping('cover', 'cover'), ] @@ -127,16 +122,6 @@ class Work(OrderedCollectionPageMixin, Book): null=True ) - @property - def editions_path(self): - ''' it'd be nice to serialize the edition instead but, recursion ''' - default = self.default_edition - ed_list = [ - e.remote_id for e in \ - self.edition_set.filter(~Q(id=default.id)).all() - ] - return [default.remote_id] + ed_list - def to_edition_list(self, **kwargs): ''' activitypub serialization for this work's editions ''' From 0a8ef98854bed6df5bf5e7e3ef7c5b67e5cbb9b0 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 13:14:18 -0800 Subject: [PATCH 009/356] use localized remote_ids for books --- bookwyrm/models/book.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 70eb8652..d5e811bb 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -2,7 +2,6 @@ import re from django.db import models -from django.db.models import Q from django.utils import timezone from model_utils.managers import InheritanceManager @@ -94,10 +93,17 @@ class Book(ActivitypubMixin, BookWyrmModel): ''' can't be abstract for query reasons, but you shouldn't USE it ''' if not isinstance(self, Edition) and not isinstance(self, Work): raise ValueError('Books should be added as Editions or Works') + if self.id and not self.remote_id: self.remote_id = self.get_remote_id() - super().save(*args, **kwargs) + if not self.id: + # force set the remote id to a local version + self.origin_id = self.remote_id + saved = super().save(*args, **kwargs) + saved.remote_id = self.get_remote_id() + return saved.save() + return super().save(*args, **kwargs) def get_remote_id(self): ''' editions and works both use "book" instead of model_name ''' From 7ed2e310c02d3b175d2a3566374f09c313cba350 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 13:40:09 -0800 Subject: [PATCH 010/356] User origin ids for books and authors --- bookwyrm/activitypub/base_activity.py | 10 +++++++--- bookwyrm/models/author.py | 23 +++++++++++++++++++++++ bookwyrm/models/book.py | 9 +++++---- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 620579a0..b9b6b9b4 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -222,14 +222,18 @@ class ActivityObject: def resolve_remote_id(model, remote_id, refresh=False): ''' look up the remote_id in the database or load it remotely ''' - result = model.objects + objects = model.objects if hasattr(model.objects, 'select_subclasses'): - result = result.select_subclasses() + objects = objects.select_subclasses() # first, check for an existing copy in the database - result = result.filter( + result = objects.filter( remote_id=remote_id ).first() + if not result and hasattr(model, 'origin_id'): + result = objects.filter( + origin_id=remote_id + ).first() if result and not refresh: return result diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 1d701797..0a934e45 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -1,8 +1,12 @@ ''' database schema for info about authors ''' +from uuid import uuid4 +import re + from django.db import models from django.utils import timezone from bookwyrm import activitypub +from bookwyrm.settings import DOMAIN from bookwyrm.utils.fields import ArrayField from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel @@ -27,6 +31,25 @@ class Author(ActivitypubMixin, BookWyrmModel): ) bio = models.TextField(null=True, blank=True) + def save(self, *args, **kwargs): + ''' can't be abstract for query reasons, but you shouldn't USE it ''' + if self.id and not self.remote_id: + self.remote_id = self.get_remote_id() + + if not self.id: + # force set the remote id to a local version + self.origin_id = self.remote_id + self.remote_id = self.get_remote_id() + return super().save(*args, **kwargs) + + def get_remote_id(self): + ''' editions and works both use "book" instead of model_name ''' + uuid = str(uuid4())[:8] + # in Book, the title is used to make the url more readable, but + # since an author's name can change, I didn't want to lock in a + # potential deadname (or maiden name) in the urk. + return 'https://%s/author/%s' % (DOMAIN, uuid) + @property def display_name(self): ''' Helper to return a displayable name''' diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index d5e811bb..d3460aac 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -1,4 +1,5 @@ ''' database schema for books and shelves ''' +from uuid import uuid4 import re from django.db import models @@ -100,14 +101,14 @@ class Book(ActivitypubMixin, BookWyrmModel): if not self.id: # force set the remote id to a local version self.origin_id = self.remote_id - saved = super().save(*args, **kwargs) - saved.remote_id = self.get_remote_id() - return saved.save() + self.remote_id = self.get_remote_id() return super().save(*args, **kwargs) def get_remote_id(self): ''' editions and works both use "book" instead of model_name ''' - return 'https://%s/book/%d' % (DOMAIN, self.id) + uuid = str(uuid4())[:8] + clean_title = re.sub(r'[\W-]', '', self.title.replace(' ', '-')).lower() + return 'https://%s/author/%s-%s' % (DOMAIN, clean_title, uuid) def __repr__(self): return "<{} key={!r} title={!r}>".format( From 72c7829bab4a997fec24be2b9d1925a21bd6e3bb Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 17:29:03 -0800 Subject: [PATCH 011/356] Preserve remote_id syntax for authors and books --- bookwyrm/activitypub/base_activity.py | 114 +++++++++++++------------- bookwyrm/models/author.py | 12 +-- bookwyrm/models/base_model.py | 10 ++- bookwyrm/models/book.py | 23 ++---- bookwyrm/views.py | 2 +- 5 files changed, 72 insertions(+), 89 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index b9b6b9b4..4c73ab97 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -6,9 +6,8 @@ from uuid import uuid4 import dateutil.parser from dateutil.parser import ParserError from django.core.files.base import ContentFile -from django.db import transaction from django.db.models.fields.related_descriptors \ - import ForwardManyToOneDescriptor, ManyToManyDescriptor, \ + import ForwardManyToOneDescriptor, ManyToManyDescriptor, \ ReverseManyToOneDescriptor from django.db.models.fields import DateTimeField from django.db.models.fields.files import ImageFileDescriptor @@ -16,8 +15,6 @@ from django.db.models.query_utils import DeferredAttribute from django.utils import timezone import requests -from bookwyrm import models - class ActivitySerializerError(ValueError): ''' routine problems serializing activitypub json ''' @@ -135,7 +132,7 @@ class ActivityObject: # if the AP field is a serialized object (as in Add) remote_id = formatted_value['id'] else: - # if the AP field is just a remote_id (as in every other case) + # if the field is just a remote_id (as in every other case) remote_id = formatted_value reference = resolve_remote_id(fk_model, remote_id) mapped_fields[mapping.model_key] = reference @@ -153,62 +150,67 @@ class ActivityObject: formatted_value = None mapped_fields[mapping.model_key] = formatted_value - with transaction.atomic(): - if instance: - # updating an existing model instance - for k, v in mapped_fields.items(): - setattr(instance, k, v) - instance.save() - else: - # creating a new model instance - instance = model.objects.create(**mapped_fields) + if instance: + # updating an existing model instance + for k, v in mapped_fields.items(): + setattr(instance, k, v) + instance.save() + else: + # creating a new model instance + instance = model.objects.create(**mapped_fields) + print('CREATING') + print(instance) + print(instance.id) - # --- these are all fields that can't be saved until after the - # instance has an id (after it's been saved). ---------------# + # --- these are all fields that can't be saved until after the + # instance has an id (after it's been saved). ---------------# - # add images - for (model_key, value) in image_fields.items(): - formatted_value = image_formatter(value) - if not formatted_value: - continue - getattr(instance, model_key).save(*formatted_value, save=True) + # add images + for (model_key, value) in image_fields.items(): + formatted_value = image_formatter(value) + if not formatted_value: + continue + getattr(instance, model_key).save(*formatted_value, save=True) - # add many to many fields - for (model_key, values) in many_to_many_fields.items(): - # mention books, mention users - if values == MISSING: - continue - model_field = getattr(instance, model_key) - model = model_field.model - items = [] - for link in values: - if isinstance(link, dict): - # check that the Type matches the model (Status - # tags contain both user mentions and book tags) - if not model.activity_serializer.type == \ - link.get('type'): - continue - remote_id = link.get('href') - else: - remote_id = link - items.append( - resolve_remote_id(model, remote_id) - ) - getattr(instance, model_key).set(items) + # add many to many fields + for (model_key, values) in many_to_many_fields.items(): + # mention books, mention users + if values == MISSING: + continue + model_field = getattr(instance, model_key) + model = model_field.model + items = [] + for link in values: + if isinstance(link, dict): + # check that the Type matches the model (Status + # tags contain both user mentions and book tags) + if not model.activity_serializer.type == \ + link.get('type'): + continue + remote_id = link.get('href') + else: + remote_id = link + items.append( + resolve_remote_id(model, remote_id) + ) + getattr(instance, model_key).set(items) - # add one to many fields - for (model_key, values) in one_to_many_fields.items(): - if values == MISSING: - continue - model_field = getattr(instance, model_key) - model = model_field.model - for item in values: + # add one to many fields + for (model_key, values) in one_to_many_fields.items(): + if values == MISSING: + continue + model_field = getattr(instance, model_key) + model = model_field.model + for item in values: + if isinstance(item, str): + print(model) + item = resolve_remote_id(model, item) + else: item = model.activity_serializer(**item) - field_name = instance.__class__.__name__.lower() - with transaction.atomic(): - item = item.to_model(model) - setattr(item, field_name, instance) - item.save() + item = item.to_model(model) + field_name = instance.__class__.__name__.lower() + setattr(item, field_name, instance) + item.save() return instance diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 0a934e45..fdaf73d9 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -1,7 +1,4 @@ ''' database schema for info about authors ''' -from uuid import uuid4 -import re - from django.db import models from django.utils import timezone @@ -37,18 +34,13 @@ class Author(ActivitypubMixin, BookWyrmModel): self.remote_id = self.get_remote_id() if not self.id: - # force set the remote id to a local version self.origin_id = self.remote_id - self.remote_id = self.get_remote_id() + self.remote_id = None return super().save(*args, **kwargs) def get_remote_id(self): ''' editions and works both use "book" instead of model_name ''' - uuid = str(uuid4())[:8] - # in Book, the title is used to make the url more readable, but - # since an author's name can change, I didn't want to lock in a - # potential deadname (or maiden name) in the urk. - return 'https://%s/author/%s' % (DOMAIN, uuid) + return 'https://%s/author/%s' % (DOMAIN, self.id) @property def display_name(self): diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 877f094e..71f3c7b7 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -12,7 +12,7 @@ from Crypto.Hash import SHA256 from django.db import models from django.db.models.fields.files import ImageFileDescriptor from django.db.models.fields.related_descriptors \ - import ManyToManyDescriptor + import ManyToManyDescriptor, ReverseManyToOneDescriptor from django.dispatch import receiver from bookwyrm import activitypub @@ -74,16 +74,18 @@ class ActivitypubMixin: # this field on the model isn't serialized continue value = getattr(self, mapping.model_key) - model_field_type = getattr(self.__class__, mapping.model_key) + model_field = getattr(self.__class__, mapping.model_key) + print(mapping.model_key, type(model_field)) if hasattr(value, 'remote_id'): # this is probably a foreign key field, which we want to # serialize as just the remote_id url reference value = value.remote_id - elif isinstance(model_field_type, ManyToManyDescriptor): + elif isinstance(model_field, ManyToManyDescriptor) or \ + isinstance(model_field, ReverseManyToOneDescriptor): value = [i.remote_id for i in value.all()] elif isinstance(value, datetime): value = value.isoformat() - elif isinstance(model_field_type, ImageFileDescriptor): + elif isinstance(model_field, ImageFileDescriptor): value = image_formatter(value) # run the custom formatter function set in the model diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index d3460aac..cc377d21 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -1,5 +1,4 @@ ''' database schema for books and shelves ''' -from uuid import uuid4 import re from django.db import models @@ -99,16 +98,13 @@ class Book(ActivitypubMixin, BookWyrmModel): self.remote_id = self.get_remote_id() if not self.id: - # force set the remote id to a local version self.origin_id = self.remote_id - self.remote_id = self.get_remote_id() + self.remote_id = None return super().save(*args, **kwargs) def get_remote_id(self): ''' editions and works both use "book" instead of model_name ''' - uuid = str(uuid4())[:8] - clean_title = re.sub(r'[\W-]', '', self.title.replace(' ', '-')).lower() - return 'https://%s/author/%s-%s' % (DOMAIN, clean_title, uuid) + return 'https://%s/book/%d' % (DOMAIN, self.id) def __repr__(self): return "<{} key={!r} title={!r}>".format( @@ -129,17 +125,6 @@ class Work(OrderedCollectionPageMixin, Book): null=True ) - - def to_edition_list(self, **kwargs): - ''' activitypub serialization for this work's editions ''' - remote_id = self.remote_id + '/editions' - return self.to_ordered_collection( - self.edition_set, - remote_id=remote_id, - **kwargs - ) - - activity_serializer = activitypub.Work @@ -161,7 +146,8 @@ class Edition(Book): through='ShelfBook', through_fields=('book', 'shelf') ) - parent_work = models.ForeignKey('Work', on_delete=models.PROTECT, null=True) + parent_work = models.ForeignKey( + 'Work', on_delete=models.PROTECT, null=True, related_name='editions') activity_serializer = activitypub.Edition @@ -175,6 +161,7 @@ class Edition(Book): return super().save(*args, **kwargs) + def isbn_10_to_13(isbn_10): ''' convert an isbn 10 into an isbn 13 ''' isbn_10 = re.sub(r'[^0-9X]', '', isbn_10) diff --git a/bookwyrm/views.py b/bookwyrm/views.py index c6992e6b..bd62902c 100644 --- a/bookwyrm/views.py +++ b/bookwyrm/views.py @@ -540,7 +540,7 @@ def book_page(request, book_id): return HttpResponseNotFound() reviews = models.Review.objects.filter( - book__in=work.edition_set.all(), + book__in=work.editions.all(), ) # all reviews for the book reviews = get_activity_feed(request.user, 'federated', model=reviews) From dfd730757d01091500f1d819807945de4aac927c Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 17:41:57 -0800 Subject: [PATCH 012/356] handle unset default editions --- bookwyrm/activitypub/book.py | 1 + .../migrations/0018_auto_20201128_2142.py | 24 +++++++++++++++++++ bookwyrm/models/book.py | 5 ++++ bookwyrm/views.py | 4 ++-- 4 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 bookwyrm/migrations/0018_auto_20201128_2142.py diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py index 8a6b88d9..ae9c334d 100644 --- a/bookwyrm/activitypub/book.py +++ b/bookwyrm/activitypub/book.py @@ -49,6 +49,7 @@ class Edition(Book): class Work(Book): ''' work instance of a book object ''' lccn: str = '' + defaultEdition: str = '' editions: List[str] type: str = 'Work' diff --git a/bookwyrm/migrations/0018_auto_20201128_2142.py b/bookwyrm/migrations/0018_auto_20201128_2142.py new file mode 100644 index 00000000..86d02870 --- /dev/null +++ b/bookwyrm/migrations/0018_auto_20201128_2142.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.7 on 2020-11-28 21:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0017_auto_20201128_1849'), + ] + + operations = [ + migrations.AlterField( + model_name='edition', + name='parent_work', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'), + ), + migrations.AlterField( + model_name='tag', + name='name', + field=models.CharField(max_length=100, unique=True), + ), + ] diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index cc377d21..ffc9dd8a 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -86,6 +86,7 @@ class Book(ActivitypubMixin, BookWyrmModel): ActivityMapping('lccn', 'lccn'), ActivityMapping('editions', 'editions'), + ActivityMapping('defaultEdition', 'default_edition'), ActivityMapping('cover', 'cover'), ] @@ -125,6 +126,10 @@ class Work(OrderedCollectionPageMixin, Book): null=True ) + def get_default_edition(self): + ''' in case the default edition is not set ''' + return self.default_edition or self.editions.first() + activity_serializer = activitypub.Work diff --git a/bookwyrm/views.py b/bookwyrm/views.py index bd62902c..4f3e3f1c 100644 --- a/bookwyrm/views.py +++ b/bookwyrm/views.py @@ -531,7 +531,7 @@ def book_page(request, book_id): return JsonResponse(book.to_activity(), encoder=ActivityEncoder) if isinstance(book, models.Work): - book = book.default_edition + book = book.get_default_edition() if not book: return HttpResponseNotFound() @@ -646,7 +646,7 @@ def author_page(request, author_id): data = { 'title': author.name, 'author': author, - 'books': [b.default_edition for b in books], + 'books': [b.get_default_edition() for b in books], } return TemplateResponse(request, 'author.html', data) From b4fe9f160f47d1936a87fde0715187076ad12f4c Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 18:11:52 -0800 Subject: [PATCH 013/356] Correctly look up books by remote/origin id --- bookwyrm/activitypub/base_activity.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 4c73ab97..5b4d6335 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -91,10 +91,7 @@ class ActivityObject: # check for an existing instance, if we're not updating a known obj if not instance: - try: - return model.objects.get(remote_id=self.id) - except model.DoesNotExist: - pass + instance = find_existing_by_remote_id(model, self.id) model_fields = [m.name for m in model._meta.get_fields()] mapped_fields = {} @@ -123,8 +120,9 @@ class ActivityObject: formatted_value = date_value except (ParserError, TypeError): formatted_value = None - elif isinstance(model_field, ForwardManyToOneDescriptor) and \ - formatted_value: + elif isinstance(model_field, ForwardManyToOneDescriptor): + if not formatted_value: + continue # foreign key remote id reolver (work on Edition, for example) fk_model = model_field.field.related_model if isinstance(formatted_value, dict) and \ @@ -158,9 +156,6 @@ class ActivityObject: else: # creating a new model instance instance = model.objects.create(**mapped_fields) - print('CREATING') - print(instance) - print(instance.id) # --- these are all fields that can't be saved until after the # instance has an id (after it's been saved). ---------------# @@ -203,7 +198,6 @@ class ActivityObject: model = model_field.model for item in values: if isinstance(item, str): - print(model) item = resolve_remote_id(model, item) else: item = model.activity_serializer(**item) @@ -222,8 +216,8 @@ class ActivityObject: return data -def resolve_remote_id(model, remote_id, refresh=False): - ''' look up the remote_id in the database or load it remotely ''' +def find_existing_by_remote_id(model, remote_id): + ''' check for an existing instance of this id in the db ''' objects = model.objects if hasattr(model.objects, 'select_subclasses'): objects = objects.select_subclasses() @@ -232,10 +226,17 @@ def resolve_remote_id(model, remote_id, refresh=False): result = objects.filter( remote_id=remote_id ).first() + if not result and hasattr(model, 'origin_id'): result = objects.filter( origin_id=remote_id ).first() + return result + + +def resolve_remote_id(model, remote_id, refresh=False): + ''' look up the remote_id in the database or load it remotely ''' + result = find_existing_by_remote_id(model, remote_id) if result and not refresh: return result From 1789b091d6d08e5c9c56e04238d2a94ed5f1af72 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 18:23:34 -0800 Subject: [PATCH 014/356] not all that better way to distinguish add book to shelf vs tag --- bookwyrm/incoming.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index e9b0d019..1978e0fe 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -293,11 +293,11 @@ def handle_unboost(activity): @app.task def handle_add(activity): ''' putting a book on a shelf ''' - # TODO absofuckinglutely not an acceptable solution - if 'tag' in activity['id']: - activitypub.AddBook(**activity).to_model(models.Tag) - else: + #this is janky as heck but I haven't thought of a better solution + try: activitypub.AddBook(**activity).to_model(models.ShelfBook) + except activitypub.ActivitySerializerError: + activitypub.AddBook(**activity).to_model(models.Tag) @app.task From 34e8fb3e5caf0bc4545d1d4bb72da38ee6a6d7aa Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 18:28:09 -0800 Subject: [PATCH 015/356] style fix and removing stray print statement --- bookwyrm/models/base_model.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 71f3c7b7..f5f244c5 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -75,13 +75,12 @@ class ActivitypubMixin: continue value = getattr(self, mapping.model_key) model_field = getattr(self.__class__, mapping.model_key) - print(mapping.model_key, type(model_field)) if hasattr(value, 'remote_id'): # this is probably a foreign key field, which we want to # serialize as just the remote_id url reference value = value.remote_id - elif isinstance(model_field, ManyToManyDescriptor) or \ - isinstance(model_field, ReverseManyToOneDescriptor): + elif isinstance(model_field, \ + (ManyToManyDescriptor, ReverseManyToOneDescriptor)): value = [i.remote_id for i in value.all()] elif isinstance(value, datetime): value = value.isoformat() From 9d84346d3c1f825a9c9b74ecc6aef7cb8be1827e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 18:46:50 -0800 Subject: [PATCH 016/356] remove need for get_or_create_book --- bookwyrm/connectors/bookwyrm_connector.py | 86 ++++++----------------- bookwyrm/view_actions.py | 4 +- 2 files changed, 23 insertions(+), 67 deletions(-) diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index 1bc81450..74fce0ac 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -1,83 +1,37 @@ ''' using another bookwyrm instance as a source of book data ''' -from django.db import transaction - from bookwyrm import activitypub, models from .abstract_connector import AbstractConnector, SearchResult -from .abstract_connector import get_data class Connector(AbstractConnector): - ''' interact with other instances ''' - - def update_from_mappings(self, obj, data, mappings): - ''' serialize book data into a model ''' - if self.is_work_data(data): - work_data = activitypub.Work(**data) - return work_data.to_model(models.Work, instance=obj) - edition_data = activitypub.Edition(**data) - return edition_data.to_model(models.Edition, instance=obj) - - - def get_remote_id_from_data(self, data): - return data.get('id') - - - def is_work_data(self, data): - return data.get('type') == 'Work' - - - def get_edition_from_work_data(self, data): - ''' we're served a list of edition urls ''' - path = data['editions'][0] - return get_data(path) - - - def get_work_from_edition_date(self, data): - return get_data(data['work']) - - - def get_authors_from_data(self, data): - ''' load author data ''' - for author_id in data.get('authors', []): - try: - yield models.Author.objects.get(origin_id=author_id) - except models.Author.DoesNotExist: - pass - data = get_data(author_id) - author_data = activitypub.Author(**data) - author = author_data.to_model(models.Author) - yield author - - - def get_cover_from_data(self, data): - pass + ''' this is basically just for search ''' + def get_or_create_book(self, remote_id): + return activitypub.resolve_remote_id(models.Edition, remote_id) def parse_search_data(self, data): return data - def format_search_result(self, search_result): return SearchResult(**search_result) + def get_remote_id_from_data(self, data): + pass + + def is_work_data(self, data): + pass + + def get_edition_from_work_data(self, data): + pass + + def get_work_from_edition_date(self, data): + pass + + def get_cover_from_data(self, data): + pass def expand_book_data(self, book): - work = book - # go from the edition to the work, if necessary - if isinstance(book, models.Edition): - work = book.parent_work + pass - # it may be that we actually want to request this url - editions_url = '%s/editions?page=true' % work.remote_id - edition_options = get_data(editions_url) - for edition_data in edition_options['orderedItems']: - with transaction.atomic(): - edition = self.create_book( - edition_data['id'], - edition_data, - models.Edition - ) - edition.parent_work = work - edition.save() - if not edition.authors.exists() and work.authors.exists(): - edition.authors.set(work.authors.all()) + def get_authors_from_data(self, data): + pass diff --git a/bookwyrm/view_actions.py b/bookwyrm/view_actions.py index 28dd8b87..29dc6406 100644 --- a/bookwyrm/view_actions.py +++ b/bookwyrm/view_actions.py @@ -219,7 +219,9 @@ def edit_profile(request): def resolve_book(request): ''' figure out the local path to a book from a remote_id ''' remote_id = request.POST.get('remote_id') - book = books_manager.get_or_create_book(remote_id) + connector = books_manager.get_or_create_connector(remote_id) + book = connector.get_or_create_book(remote_id) + return redirect('/book/%d' % book.id) From c9433a3c7e937f159113b900202544ef3ec1f6ea Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 18:56:28 -0800 Subject: [PATCH 017/356] Simplify bookwyrm connector abstract --- bookwyrm/connectors/abstract_connector.py | 96 ++++++++++++----------- bookwyrm/connectors/bookwyrm_connector.py | 25 +----- bookwyrm/connectors/openlibrary.py | 4 +- 3 files changed, 54 insertions(+), 71 deletions(-) diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index d709b075..0cf4a011 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -16,20 +16,13 @@ class ConnectorException(HTTPError): ''' when the connector can't do what was asked ''' -class AbstractConnector(ABC): - ''' generic book data connector ''' - +class AbstractMinimalConnector(ABC): + ''' just the bare bones, for other bookwyrm instances ''' def __init__(self, identifier): # load connector settings info = models.Connector.objects.get(identifier=identifier) self.connector = info - self.key_mappings = [] - - # fields we want to look for in book data to copy over - # title we handle separately. - self.book_mappings = [] - # the things in the connector model to copy over self_fields = [ 'base_url', @@ -44,15 +37,6 @@ class AbstractConnector(ABC): for field in self_fields: setattr(self, field, getattr(info, field)) - - def is_available(self): - ''' check if you're allowed to use this connector ''' - if self.max_query_count is not None: - if self.connector.query_count >= self.max_query_count: - return False - return True - - def search(self, query, min_confidence=None): ''' free text search ''' resp = requests.get( @@ -70,9 +54,40 @@ class AbstractConnector(ABC): results.append(self.format_search_result(doc)) return results - + @abstractmethod def get_or_create_book(self, remote_id): ''' pull up a book record by whatever means possible ''' + + @abstractmethod + def parse_search_data(self, data): + ''' turn the result json from a search into a list ''' + + @abstractmethod + def format_search_result(self, search_result): + ''' create a SearchResult obj from json ''' + + +class AbstractConnector(AbstractMinimalConnector): + ''' generic book data connector ''' + def __init__(self, identifier): + super().__init__(identifier) + + self.key_mappings = [] + + # fields we want to look for in book data to copy over + # title we handle separately. + self.book_mappings = [] + + + def is_available(self): + ''' check if you're allowed to use this connector ''' + if self.max_query_count is not None: + if self.connector.query_count >= self.max_query_count: + return False + return True + + + def get_or_create_book(self, remote_id): # try to load the book book = models.Book.objects.select_subclasses().filter( origin_id=remote_id @@ -157,7 +172,7 @@ class AbstractConnector(ABC): def update_book_from_data(self, book, data, update_cover=True): ''' for creating a new book or syncing with data ''' - book = self.update_from_mappings(book, data, self.book_mappings) + book = update_from_mappings(book, data, self.book_mappings) author_text = [] for author in self.get_authors_from_data(data): @@ -246,39 +261,28 @@ class AbstractConnector(ABC): def get_cover_from_data(self, data): ''' load cover ''' - - @abstractmethod - def parse_search_data(self, data): - ''' turn the result json from a search into a list ''' - - - @abstractmethod - def format_search_result(self, search_result): - ''' create a SearchResult obj from json ''' - - @abstractmethod def expand_book_data(self, book): ''' get more info on a book ''' - def update_from_mappings(self, obj, data, mappings): - ''' assign data to model with mappings ''' - for mapping in mappings: - # check if this field is present in the data - value = data.get(mapping.remote_field) - if not value: - continue +def update_from_mappings(obj, data, mappings): + ''' assign data to model with mappings ''' + for mapping in mappings: + # check if this field is present in the data + value = data.get(mapping.remote_field) + if not value: + continue - # extract the value in the right format - try: - value = mapping.formatter(value) - except: - continue + # extract the value in the right format + try: + value = mapping.formatter(value) + except: + continue - # assign the formatted value to the model - obj.__setattr__(mapping.local_field, value) - return obj + # assign the formatted value to the model + obj.__setattr__(mapping.local_field, value) + return obj def get_date(date_string): diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index 74fce0ac..e4d32fd3 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -1,9 +1,9 @@ ''' using another bookwyrm instance as a source of book data ''' from bookwyrm import activitypub, models -from .abstract_connector import AbstractConnector, SearchResult +from .abstract_connector import AbstractMinimalConnector, SearchResult -class Connector(AbstractConnector): +class Connector(AbstractMinimalConnector): ''' this is basically just for search ''' def get_or_create_book(self, remote_id): @@ -14,24 +14,3 @@ class Connector(AbstractConnector): def format_search_result(self, search_result): return SearchResult(**search_result) - - def get_remote_id_from_data(self, data): - pass - - def is_work_data(self, data): - pass - - def get_edition_from_work_data(self, data): - pass - - def get_work_from_edition_date(self, data): - pass - - def get_cover_from_data(self, data): - pass - - def expand_book_data(self, book): - pass - - def get_authors_from_data(self, data): - pass diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 5e18616d..90bd7f28 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -7,7 +7,7 @@ from django.core.files.base import ContentFile from bookwyrm import models from .abstract_connector import AbstractConnector, SearchResult, Mapping from .abstract_connector import ConnectorException -from .abstract_connector import get_date, get_data +from .abstract_connector import get_date, get_data, update_from_mappings from .openlibrary_languages import languages @@ -184,7 +184,7 @@ class Connector(AbstractConnector): data = get_data(url) author = models.Author(openlibrary_key=olkey) - author = self.update_from_mappings(author, data, self.author_mappings) + author = update_from_mappings(author, data, self.author_mappings) name = data.get('name') # TODO this is making some BOLD assumption if name: From d8fdc664507b585f002330f9719a72656f909a81 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 19:03:37 -0800 Subject: [PATCH 018/356] removes outdated update book code --- bookwyrm/books_manager.py | 6 ------ bookwyrm/incoming.py | 13 ++----------- bookwyrm/routine_book_tasks.py | 16 ---------------- 3 files changed, 2 insertions(+), 33 deletions(-) delete mode 100644 bookwyrm/routine_book_tasks.py diff --git a/bookwyrm/books_manager.py b/bookwyrm/books_manager.py index fb0a747e..3b865768 100644 --- a/bookwyrm/books_manager.py +++ b/bookwyrm/books_manager.py @@ -85,12 +85,6 @@ def first_search_result(query, min_confidence=0.1): return None -def update_book(book, data=None): - ''' re-sync with the original data source ''' - connector = load_connector(book.connector) - connector.update_book(book, data=data) - - def get_connectors(): ''' load all connectors ''' for info in models.Connector.objects.order_by('priority').all(): diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 1978e0fe..017be19d 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -8,7 +8,7 @@ from django.http import HttpResponseBadRequest, HttpResponseNotFound from django.views.decorators.csrf import csrf_exempt import requests -from bookwyrm import activitypub, books_manager, models, outgoing +from bookwyrm import activitypub, models, outgoing from bookwyrm import status as status_builder from bookwyrm.tasks import app from bookwyrm.signatures import Signature @@ -317,13 +317,4 @@ def handle_update_user(activity): @app.task def handle_update_book(activity): ''' a remote instance changed a book (Document) ''' - document = activity['object'] - # check if we have their copy and care about their updates - book = models.Book.objects.select_subclasses().filter( - remote_id=document['id'], - sync=True, - ).first() - if not book: - return - - books_manager.update_book(book, data=document) + activitypub.Edition(**activity['object']).to_model(models.Edition) diff --git a/bookwyrm/routine_book_tasks.py b/bookwyrm/routine_book_tasks.py deleted file mode 100644 index eaa28d90..00000000 --- a/bookwyrm/routine_book_tasks.py +++ /dev/null @@ -1,16 +0,0 @@ -''' Routine tasks for keeping your library tidy ''' -from datetime import timedelta -from django.utils import timezone -from bookwyrm import books_manager -from bookwyrm import models - -def sync_book_data(): - ''' update books with any changes to their canonical source ''' - expiry = timezone.now() - timedelta(days=1) - books = models.Edition.objects.filter( - sync=True, - last_sync_date__lte=expiry - ).all() - for book in books: - # TODO: create background tasks - books_manager.update_book(book) From e9be31e9c159e6805860b035663d051e0c35f326 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Nov 2020 19:05:12 -0800 Subject: [PATCH 019/356] merge migrations --- .../migrations/0016_auto_20201128_1804.py | 24 ------------------ ...128_1849.py => 0016_auto_20201129_0304.py} | 25 +++++++++++++++++-- .../migrations/0018_auto_20201128_2142.py | 24 ------------------ 3 files changed, 23 insertions(+), 50 deletions(-) delete mode 100644 bookwyrm/migrations/0016_auto_20201128_1804.py rename bookwyrm/migrations/{0017_auto_20201128_1849.py => 0016_auto_20201129_0304.py} (57%) delete mode 100644 bookwyrm/migrations/0018_auto_20201128_2142.py diff --git a/bookwyrm/migrations/0016_auto_20201128_1804.py b/bookwyrm/migrations/0016_auto_20201128_1804.py deleted file mode 100644 index 9becde82..00000000 --- a/bookwyrm/migrations/0016_auto_20201128_1804.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.0.7 on 2020-11-28 18:04 - -import bookwyrm.utils.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('bookwyrm', '0015_auto_20201128_0349'), - ] - - operations = [ - migrations.AlterField( - model_name='book', - name='subject_places', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), - ), - migrations.AlterField( - model_name='book', - name='subjects', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), - ), - ] diff --git a/bookwyrm/migrations/0017_auto_20201128_1849.py b/bookwyrm/migrations/0016_auto_20201129_0304.py similarity index 57% rename from bookwyrm/migrations/0017_auto_20201128_1849.py rename to bookwyrm/migrations/0016_auto_20201129_0304.py index 722458b2..2bf820e1 100644 --- a/bookwyrm/migrations/0017_auto_20201128_1849.py +++ b/bookwyrm/migrations/0016_auto_20201129_0304.py @@ -1,5 +1,6 @@ -# Generated by Django 3.0.7 on 2020-11-28 18:49 +# Generated by Django 3.0.7 on 2020-11-29 03:04 +import bookwyrm.utils.fields from django.conf import settings from django.db import migrations, models import django.db.models.deletion @@ -8,10 +9,30 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('bookwyrm', '0016_auto_20201128_1804'), + ('bookwyrm', '0015_auto_20201128_0349'), ] operations = [ + migrations.AlterField( + model_name='book', + name='subject_places', + field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='book', + name='subjects', + field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='edition', + name='parent_work', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'), + ), + migrations.AlterField( + model_name='tag', + name='name', + field=models.CharField(max_length=100, unique=True), + ), migrations.AlterUniqueTogether( name='tag', unique_together=set(), diff --git a/bookwyrm/migrations/0018_auto_20201128_2142.py b/bookwyrm/migrations/0018_auto_20201128_2142.py deleted file mode 100644 index 86d02870..00000000 --- a/bookwyrm/migrations/0018_auto_20201128_2142.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.0.7 on 2020-11-28 21:42 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('bookwyrm', '0017_auto_20201128_1849'), - ] - - operations = [ - migrations.AlterField( - model_name='edition', - name='parent_work', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'), - ), - migrations.AlterField( - model_name='tag', - name='name', - field=models.CharField(max_length=100, unique=True), - ), - ] From 9ddd60ce16d2eac044127af5c1587a162c0f96bb Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 29 Nov 2020 09:40:15 -0800 Subject: [PATCH 020/356] Fixes broadcast tests --- bookwyrm/activitypub/base_activity.py | 22 +++++++--------------- bookwyrm/connectors/__init__.py | 1 + bookwyrm/connectors/abstract_connector.py | 11 +++++++++++ bookwyrm/tests/test_broadcast.py | 16 +++++++++------- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 5b4d6335..9944e368 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -15,6 +15,7 @@ from django.db.models.query_utils import DeferredAttribute from django.utils import timezone import requests +from bookwyrm.connectors import ConnectorException, get_data, get_image class ActivitySerializerError(ValueError): ''' routine problems serializing activitypub json ''' @@ -242,20 +243,13 @@ def resolve_remote_id(model, remote_id, refresh=False): # load the data and create the object try: - response = requests.get( - remote_id, - headers={'Accept': 'application/json; charset=utf-8'}, - ) - except ConnectionError: + data = get_data(remote_id) + except (ConnectorException, ConnectionError): raise ActivitySerializerError( 'Could not connect to host for remote_id in %s model: %s' % \ (model.__name__, remote_id)) - if not response.ok: - raise ActivitySerializerError( - 'Could not resolve remote_id in %s model: %s' % \ - (model.__name__, remote_id)) - item = model.activity_serializer(**response.json()) + item = model.activity_serializer(**data) # if we're refreshing, "result" will be set and we'll update it return item.to_model(model, instance=result) @@ -272,11 +266,9 @@ def image_formatter(image_slug): return None if not url: return None - try: - response = requests.get(url) - except ConnectionError: - return None - if not response.ok: + + response = get_image(url) + if not response: return None image_name = str(uuid4()) + '.' + url.split('.')[-1] diff --git a/bookwyrm/connectors/__init__.py b/bookwyrm/connectors/__init__.py index b5d93b47..4eb91de4 100644 --- a/bookwyrm/connectors/__init__.py +++ b/bookwyrm/connectors/__init__.py @@ -1,3 +1,4 @@ ''' bring connectors into the namespace ''' from .settings import CONNECTORS from .abstract_connector import ConnectorException +from .abstract_connector import get_data, get_image diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 0cf4a011..4e756d8c 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -318,6 +318,17 @@ def get_data(url): return data +def get_image(url): + ''' wrapper for requesting an image ''' + try: + resp = requests.get(url) + except RequestError: + return None + if not resp.ok: + return None + return resp + + @dataclass class SearchResult: ''' standardized search result object ''' diff --git a/bookwyrm/tests/test_broadcast.py b/bookwyrm/tests/test_broadcast.py index 1112b3fa..96faf892 100644 --- a/bookwyrm/tests/test_broadcast.py +++ b/bookwyrm/tests/test_broadcast.py @@ -1,3 +1,4 @@ +from unittest.mock import patch from django.test import TestCase from bookwyrm import models, broadcast @@ -37,13 +38,14 @@ class Book(TestCase): 'joe', 'joe@mouse.mouse', 'jeoword') self.user.followers.add(local_follower) - models.User.objects.create_user( - 'nutria', 'nutria@mouse.mouse', 'nuword', - remote_id='http://example.com/u/4', - outbox='http://example.com/u/4/o', - shared_inbox='http://example.com/inbox', - inbox='http://example.com/u/4/inbox', - local=False) + with patch('bookwyrm.models.user.get_remote_reviews.delay'): + models.User.objects.create_user( + 'nutria', 'nutria@mouse.mouse', 'nuword', + remote_id='http://example.com/u/4', + outbox='http://example.com/u/4/o', + shared_inbox='http://example.com/inbox', + inbox='http://example.com/u/4/inbox', + local=False) def test_get_public_recipients(self): From 205fa0d465ba8a1db31ad34b6cb1ba66ee98ae05 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 29 Nov 2020 10:08:19 -0800 Subject: [PATCH 021/356] set user's remote server in a celery task --- bookwyrm/models/user.py | 43 ++++++++------------ bookwyrm/tests/activitypub/test_quotation.py | 16 ++++---- bookwyrm/tests/test_broadcast.py | 2 +- 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index a32a35b7..5b5af77e 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -7,6 +7,7 @@ from django.db import models from django.dispatch import receiver from bookwyrm import activitypub +from bookwyrm.connectors import get_data from bookwyrm.models.shelf import Shelf from bookwyrm.models.status import Status, Review from bookwyrm.settings import DOMAIN @@ -195,13 +196,7 @@ def execute_after_save(sender, instance, created, *args, **kwargs): return if not instance.local: - actor_parts = urlparse(instance.remote_id) - instance.federated_server = \ - get_or_create_remote_server(actor_parts.netloc) - instance.save() - if instance.bookwyrm_user: - get_remote_reviews.delay(instance.outbox) - return + set_remote_server.delay(instance.id) shelves = [{ 'name': 'To Read', @@ -222,6 +217,18 @@ def execute_after_save(sender, instance, created, *args, **kwargs): editable=False ).save() +@app.task +def set_remote_server(user_id): + ''' figure out the user's remote server in the background ''' + user = User.objects.get(id=user_id) + actor_parts = urlparse(user.remote_id) + user.federated_server = \ + get_or_create_remote_server(actor_parts.netloc) + user.save() + if user.bookwyrm_user: + get_remote_reviews.delay(user.outbox) + return + def get_or_create_remote_server(domain): ''' get info on a remote server ''' @@ -232,25 +239,14 @@ def get_or_create_remote_server(domain): except FederatedServer.DoesNotExist: pass - response = requests.get( - 'https://%s/.well-known/nodeinfo' % domain, - headers={'Accept': 'application/activity+json'} - ) + data = get_data('https://%s/.well-known/nodeinfo' % domain) - if response.status_code != 200: - return None - - data = response.json() try: nodeinfo_url = data.get('links')[0].get('href') except (TypeError, KeyError): return None - response = requests.get( - nodeinfo_url, - headers={'Accept': 'application/activity+json'} - ) - data = response.json() + data = get_data(nodeinfo_url) server = FederatedServer.objects.create( server_name=domain, @@ -264,11 +260,8 @@ def get_or_create_remote_server(domain): def get_remote_reviews(outbox): ''' ingest reviews by a new remote bookwyrm user ''' outbox_page = outbox + '?page=true' - response = requests.get( - outbox_page, - headers={'Accept': 'application/activity+json'} - ) - data = response.json() + data = get_data(outbox_page) + # TODO: pagination? for activity in data['orderedItems']: if not activity['type'] == 'Review': diff --git a/bookwyrm/tests/activitypub/test_quotation.py b/bookwyrm/tests/activitypub/test_quotation.py index 50d0ac86..b0699571 100644 --- a/bookwyrm/tests/activitypub/test_quotation.py +++ b/bookwyrm/tests/activitypub/test_quotation.py @@ -1,5 +1,6 @@ import json import pathlib +from unittest.mock import patch from django.test import TestCase from bookwyrm import activitypub, models @@ -8,13 +9,14 @@ from bookwyrm import activitypub, models class Quotation(TestCase): ''' we have hecka ways to create statuses ''' def setUp(self): - self.user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.mouse', 'mouseword', - local=False, - inbox='https://example.com/user/mouse/inbox', - outbox='https://example.com/user/mouse/outbox', - remote_id='https://example.com/user/mouse', - ) + with patch('bookwyrm.models.user.set_remote_server.delay'): + self.user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword', + local=False, + inbox='https://example.com/user/mouse/inbox', + outbox='https://example.com/user/mouse/outbox', + remote_id='https://example.com/user/mouse', + ) self.book = models.Edition.objects.create( title='Example Edition', remote_id='https://example.com/book/1', diff --git a/bookwyrm/tests/test_broadcast.py b/bookwyrm/tests/test_broadcast.py index 96faf892..3ee4eeba 100644 --- a/bookwyrm/tests/test_broadcast.py +++ b/bookwyrm/tests/test_broadcast.py @@ -38,7 +38,7 @@ class Book(TestCase): 'joe', 'joe@mouse.mouse', 'jeoword') self.user.followers.add(local_follower) - with patch('bookwyrm.models.user.get_remote_reviews.delay'): + with patch('bookwyrm.models.user.set_remote_server.delay'): models.User.objects.create_user( 'nutria', 'nutria@mouse.mouse', 'nuword', remote_id='http://example.com/u/4', From 96563598bf5fb01952188daca6c6542935147666 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 29 Nov 2020 10:13:30 -0800 Subject: [PATCH 022/356] mock celery tasks for broadcast tests --- bookwyrm/tests/test_broadcast.py | 61 ++++++++++++++++---------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/bookwyrm/tests/test_broadcast.py b/bookwyrm/tests/test_broadcast.py index 3ee4eeba..ae4e6145 100644 --- a/bookwyrm/tests/test_broadcast.py +++ b/bookwyrm/tests/test_broadcast.py @@ -6,39 +6,40 @@ from bookwyrm import models, broadcast class Book(TestCase): def setUp(self): - self.user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.mouse', 'mouseword') + with patch('bookwyrm.models.user.get_remote_reviews.delay'): + self.user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword') - follower = models.User.objects.create_user( - 'rat', 'rat@mouse.mouse', 'ratword', local=False, - remote_id='http://example.com/u/1', - outbox='http://example.com/u/1/o', - shared_inbox='http://example.com/inbox', - inbox='http://example.com/u/1/inbox') - self.user.followers.add(follower) - - no_inbox_follower = models.User.objects.create_user( - 'hamster', 'hamster@mouse.mouse', 'hamword', - shared_inbox=None, local=False, - remote_id='http://example.com/u/2', - outbox='http://example.com/u/2/o', - inbox='http://example.com/u/2/inbox') - self.user.followers.add(no_inbox_follower) - - non_fr_follower = models.User.objects.create_user( - 'gerbil', 'gerb@mouse.mouse', 'gerbword', - remote_id='http://example.com/u/3', - outbox='http://example2.com/u/3/o', - inbox='http://example2.com/u/3/inbox', - shared_inbox='http://example2.com/inbox', - bookwyrm_user=False, local=False) - self.user.followers.add(non_fr_follower) - - local_follower = models.User.objects.create_user( - 'joe', 'joe@mouse.mouse', 'jeoword') - self.user.followers.add(local_follower) + local_follower = models.User.objects.create_user( + 'joe', 'joe@mouse.mouse', 'jeoword') + self.user.followers.add(local_follower) with patch('bookwyrm.models.user.set_remote_server.delay'): + follower = models.User.objects.create_user( + 'rat', 'rat@mouse.mouse', 'ratword', local=False, + remote_id='http://example.com/u/1', + outbox='http://example.com/u/1/o', + shared_inbox='http://example.com/inbox', + inbox='http://example.com/u/1/inbox') + self.user.followers.add(follower) + + no_inbox_follower = models.User.objects.create_user( + 'hamster', 'hamster@mouse.mouse', 'hamword', + shared_inbox=None, local=False, + remote_id='http://example.com/u/2', + outbox='http://example.com/u/2/o', + inbox='http://example.com/u/2/inbox') + self.user.followers.add(no_inbox_follower) + + non_fr_follower = models.User.objects.create_user( + 'gerbil', 'gerb@mouse.mouse', 'gerbword', + remote_id='http://example.com/u/3', + outbox='http://example2.com/u/3/o', + inbox='http://example2.com/u/3/inbox', + shared_inbox='http://example2.com/inbox', + bookwyrm_user=False, local=False) + self.user.followers.add(non_fr_follower) + models.User.objects.create_user( 'nutria', 'nutria@mouse.mouse', 'nuword', remote_id='http://example.com/u/4', From 74a58e52672f2380ce7434c265fa05d02cc15e24 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 30 Nov 2020 10:32:13 -0800 Subject: [PATCH 023/356] Use custom model fields in user model --- bookwyrm/activitypub/__init__.py | 4 +- bookwyrm/activitypub/base_activity.py | 9 - bookwyrm/activitypub/person.py | 11 +- .../migrations/0017_auto_20201130_1819.py | 188 ++++++++++++++++++ bookwyrm/models/base_model.py | 56 ++---- bookwyrm/models/fields.py | 165 +++++++++++++++ bookwyrm/models/user.py | 117 +++++------ bookwyrm/view_actions.py | 3 +- 8 files changed, 429 insertions(+), 124 deletions(-) create mode 100644 bookwyrm/migrations/0017_auto_20201130_1819.py create mode 100644 bookwyrm/models/fields.py diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py index eee9345d..b5b124ec 100644 --- a/bookwyrm/activitypub/__init__.py +++ b/bookwyrm/activitypub/__init__.py @@ -2,7 +2,7 @@ import inspect import sys -from .base_activity import ActivityEncoder, PublicKey, Signature +from .base_activity import ActivityEncoder, Signature from .base_activity import Link, Mention from .base_activity import ActivitySerializerError, resolve_remote_id from .image import Image @@ -10,7 +10,7 @@ from .note import Note, GeneratedNote, Article, Comment, Review, Quotation from .note import Tombstone from .interaction import Boost, Like from .ordered_collection import OrderedCollection, OrderedCollectionPage -from .person import Person +from .person import Person, PublicKey from .book import Edition, Work, Author from .verbs import Create, Delete, Undo, Update from .verbs import Follow, Accept, Reject diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 9944e368..9e1b5b82 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -13,7 +13,6 @@ from django.db.models.fields import DateTimeField from django.db.models.fields.files import ImageFileDescriptor from django.db.models.query_utils import DeferredAttribute from django.utils import timezone -import requests from bookwyrm.connectors import ConnectorException, get_data, get_image @@ -41,14 +40,6 @@ class Mention(Link): type: str = 'Mention' -@dataclass -class PublicKey: - ''' public key block ''' - id: str - owner: str - publicKeyPem: str - - @dataclass class Signature: ''' public key block ''' diff --git a/bookwyrm/activitypub/person.py b/bookwyrm/activitypub/person.py index e7d720ec..88349c02 100644 --- a/bookwyrm/activitypub/person.py +++ b/bookwyrm/activitypub/person.py @@ -2,9 +2,18 @@ from dataclasses import dataclass, field from typing import Dict -from .base_activity import ActivityObject, PublicKey +from .base_activity import ActivityObject from .image import Image + +@dataclass(init=False) +class PublicKey(ActivityObject): + ''' public key block ''' + owner: str + publicKeyPem: str + type: str = 'PublicKey' + + @dataclass(init=False) class Person(ActivityObject): ''' actor activitypub json ''' diff --git a/bookwyrm/migrations/0017_auto_20201130_1819.py b/bookwyrm/migrations/0017_auto_20201130_1819.py new file mode 100644 index 00000000..bab454cb --- /dev/null +++ b/bookwyrm/migrations/0017_auto_20201130_1819.py @@ -0,0 +1,188 @@ +# Generated by Django 3.0.7 on 2020-11-30 18:19 + +import bookwyrm.models.base_model +import bookwyrm.models.fields +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + +def copy_rsa_keys(app_registry, schema_editor): + db_alias = schema_editor.connection.alias + users = app_registry.get_model('bookwyrm', 'User') + keypair = app_registry.get_model('bookwyrm', 'KeyPair') + for user in users.objects.using(db_alias): + if user.public_key or user.private_key: + user.key_pair = keypair.objects.create( + private_key=user.private_key, + public_key=user.public_key + ) + user.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0016_auto_20201129_0304'), + ] + operations = [ + migrations.CreateModel( + name='KeyPair', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_date', models.DateTimeField(auto_now_add=True)), + ('updated_date', models.DateTimeField(auto_now=True)), + ('remote_id', bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id])), + ('private_key', models.TextField(blank=True, null=True)), + ('public_key', bookwyrm.models.fields.TextField(blank=True, null=True)), + ], + options={ + 'abstract': False, + }, + bases=(bookwyrm.models.base_model.ActivitypubMixin, models.Model), + ), + migrations.AddField( + model_name='user', + name='followers', + field=bookwyrm.models.fields.ManyToManyField(related_name='following', through='bookwyrm.UserFollows', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='author', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='book', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='connector', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='favorite', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='federatedserver', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='image', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='notification', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='readthrough', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='shelf', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='shelfbook', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='status', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='tag', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='avatar', + field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='avatars/'), + ), + migrations.AlterField( + model_name='user', + name='bookwyrm_user', + field=bookwyrm.models.fields.BooleanField(default=True), + ), + migrations.AlterField( + model_name='user', + name='inbox', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='local', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='user', + name='manually_approves_followers', + field=bookwyrm.models.fields.BooleanField(default=False), + ), + migrations.AlterField( + model_name='user', + name='name', + field=bookwyrm.models.fields.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name='user', + name='outbox', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='shared_inbox', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='summary', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='user', + name='username', + field=bookwyrm.models.fields.UsernameField(), + ), + migrations.AlterField( + model_name='userblocks', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='userfollowrequest', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='userfollows', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='usertag', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AddField( + model_name='user', + name='key_pair', + field=bookwyrm.models.fields.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='owner', to='bookwyrm.KeyPair'), + ), + migrations.RunPython(copy_rsa_keys), + ] diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index f5f244c5..90b889e0 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -17,6 +17,7 @@ from django.dispatch import receiver from bookwyrm import activitypub from bookwyrm.settings import DOMAIN +from .fields import RemoteIdField PrivacyLevels = models.TextChoices('Privacy', [ @@ -30,7 +31,7 @@ class BookWyrmModel(models.Model): ''' shared fields ''' created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now=True) - remote_id = models.CharField(max_length=255, null=True) + remote_id = RemoteIdField(null=True, activitypub_field='id') def get_remote_id(self): ''' generate a url that resolves to the local object ''' @@ -61,49 +62,18 @@ class ActivitypubMixin: def to_activity(self, pure=False): ''' convert from a model to an activity ''' - if pure: - # works around bookwyrm-specific fields for vanilla AP services - mappings = self.pure_activity_mappings - else: - # may include custom fields that bookwyrm instances will understand - mappings = self.activity_mappings + activity = {} + for field in self.__class__._meta.fields: + key, value = field.to_activity(getattr(self, field.name)) + activity[key] = value + for related_object in self.__class__.meta.related_objects: + # TODO: check if it's serializable + related_model = related_object.related_model + key = related_object.name + related_values = getattr(self, key) + activity[key] = [i.remote_id for i in related_values] - fields = {} - for mapping in mappings: - if not hasattr(self, mapping.model_key) or not mapping.activity_key: - # this field on the model isn't serialized - continue - value = getattr(self, mapping.model_key) - model_field = getattr(self.__class__, mapping.model_key) - if hasattr(value, 'remote_id'): - # this is probably a foreign key field, which we want to - # serialize as just the remote_id url reference - value = value.remote_id - elif isinstance(model_field, \ - (ManyToManyDescriptor, ReverseManyToOneDescriptor)): - value = [i.remote_id for i in value.all()] - elif isinstance(value, datetime): - value = value.isoformat() - elif isinstance(model_field, ImageFileDescriptor): - value = image_formatter(value) - - # run the custom formatter function set in the model - formatted_value = mapping.activity_formatter(value) - if mapping.activity_key in fields and \ - isinstance(fields[mapping.activity_key], list): - # there can be two database fields that map to the same AP list - # this happens in status tags, which combines user and book tags - fields[mapping.activity_key] += formatted_value - else: - fields[mapping.activity_key] = formatted_value - - if pure: - return self.pure_activity_serializer( - **fields - ).serialize() - return self.activity_serializer( - **fields - ).serialize() + return self.activity_serializer(**activity_json).serialize() def to_create_activity(self, user, pure=False): diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py new file mode 100644 index 00000000..c231039c --- /dev/null +++ b/bookwyrm/models/fields.py @@ -0,0 +1,165 @@ +''' activitypub-aware django model fields ''' +import re +from uuid import uuid4 + +from django.contrib.auth.models import AbstractUser +from django.core.exceptions import ValidationError +from django.core.files.base import ContentFile +from django.db import models +from django.utils.translation import gettext_lazy as _ +from bookwyrm import activitypub +from bookwyrm.settings import DOMAIN +from bookwyrm.connectors import get_image + + +def validate_remote_id(value): + ''' make sure the remote_id looks like a url ''' + if not re.match(r'^http.?:\/\/[^\s]+$', value): + raise ValidationError( + _('%(value)s is not a valid remote_id'), + params={'value': value}, + ) + + +def to_camel_case(snake_string): + ''' model_field_name to activitypubFieldName ''' + components = snake_string.split('_') + return components[0] + ''.join(x.title() for x in components[1:]) + + +class ActivitypubFieldMixin: + ''' make a database field serializable ''' + def __init__(self, *args, \ + activitypub_field=None, activitypub_wrapper=None, **kwargs): + self.activitypub_wrapper = activitypub_wrapper + self.activitypub_field = activitypub_field + super().__init__(*args, **kwargs) + + def to_activity(self, value): + ''' formatter to convert a model value into activitypub ''' + if self.activitypub_wrapper: + value = {self.activitypub_wrapper: value} + return (self.activitypub_field, value) + + def from_activity(self, activity_data): + ''' formatter to convert activitypub into a model value ''' + value = activity_data.get(self.activitypub_field) + if self.activitypub_wrapper: + value = value.get(self.activitypub_wrapper) + return value + + +class RemoteIdField(ActivitypubFieldMixin, models.CharField): + ''' a url that serves as a unique identifier ''' + def __init__(self, *args, max_length=255, validators=None, **kwargs): + validators = validators or [validate_remote_id] + super().__init__( + *args, max_length=max_length, validators=validators, + **kwargs + ) + + +class UsernameField(ActivitypubFieldMixin, models.CharField): + ''' activitypub-aware username field ''' + def __init__(self, activitypub_field='preferredUsername'): + self.activitypub_field = activitypub_field + super(ActivitypubFieldMixin, self).__init__( + _('username'), + max_length=150, + unique=True, + validators=[AbstractUser.username_validator], + error_messages={ + 'unique': _('A user with that username already exists.'), + }, + ) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + del kwargs['verbose_name'] + del kwargs['max_length'] + del kwargs['unique'] + del kwargs['validators'] + del kwargs['error_messages'] + return name, path, args, kwargs + + def to_activity(self, value): + return value.split('@')[0] + + +class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): + ''' activitypub-aware foreign key field ''' + def to_activity(self, value): + return value.remote_id + def from_activity(self, activity_data): + pass# TODO + + +class OneToOneField(ActivitypubFieldMixin, models.OneToOneField): + ''' activitypub-aware foreign key field ''' + def __init__(self, *args, **kwargs): + super(ActivitypubFieldMixin, self).__init__(*args, **kwargs) + + def to_activity(self, value): + return value.remote_id + def from_activity(self, activity_data): + pass# TODO + + +class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): + ''' activitypub-aware many to many field ''' + def __init__(self, *args, link_only=False, **kwargs): + self.link_only = link_only + super().__init__(*args, **kwargs) + + def to_activity(self, value): + if self.link_only: + return '%s/followers' % self.instance.remote_id + return [i.remote_id for i in value] + + def from_activity(self, activity_data): + if self.link_only: + return + values = super().from_activity(self, activity_data) + return values# TODO + + +class ImageField(ActivitypubFieldMixin, models.ImageField): + ''' activitypub-aware image field ''' + def to_activity(self, value): + if value and hasattr(value, 'url'): + url = value.url + else: + return None + url = 'https://%s%s' % (DOMAIN, url) + return activitypub.Image(url=url) + + def from_activity(self, activity_data): + image_slug = super().from_activity(activity_data) + # when it's an inline image (User avatar/icon, Book cover), it's a json + # blob, but when it's an attached image, it's just a url + if isinstance(image_slug, dict): + url = image_slug.get('url') + elif isinstance(image_slug, str): + url = image_slug + else: + return None + if not url: + return None + + response = get_image(url) + if not response: + return None + + image_name = str(uuid4()) + '.' + url.split('.')[-1] + image_content = ContentFile(response.content) + return [image_name, image_content] + + +class CharField(ActivitypubFieldMixin, models.CharField): + ''' activitypub-aware char field ''' + +class TextField(ActivitypubFieldMixin, models.TextField): + ''' activitypub-aware text field ''' + +class BooleanField(ActivitypubFieldMixin, models.BooleanField): + ''' activitypub-aware boolean field ''' diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 5b5af77e..19746eb1 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -1,6 +1,5 @@ ''' database schema for user data ''' from urllib.parse import urlparse -import requests from django.contrib.auth.models import AbstractUser from django.db import models @@ -13,40 +12,51 @@ from bookwyrm.models.status import Status, Review from bookwyrm.settings import DOMAIN from bookwyrm.signatures import create_key_pair from bookwyrm.tasks import app -from .base_model import ActivityMapping, OrderedCollectionPageMixin +from .base_model import OrderedCollectionPageMixin +from .base_model import ActivitypubMixin, BookWyrmModel from .federated_server import FederatedServer +from . import fields class User(OrderedCollectionPageMixin, AbstractUser): ''' a user who wants to read books ''' - private_key = models.TextField(blank=True, null=True) - public_key = models.TextField(blank=True, null=True) - inbox = models.CharField(max_length=255, unique=True) - shared_inbox = models.CharField(max_length=255, blank=True, null=True) + username = fields.UsernameField() + + key_pair = fields.OneToOneField( + 'KeyPair', + on_delete=models.CASCADE, + blank=True, null=True, + related_name='owner' + ) + inbox = fields.RemoteIdField(unique=True) + shared_inbox = fields.RemoteIdField( + activitypub_wrapper='endpoints', null=True) federated_server = models.ForeignKey( 'FederatedServer', on_delete=models.PROTECT, null=True, blank=True, ) - outbox = models.CharField(max_length=255, unique=True) - summary = models.TextField(blank=True, null=True) - local = models.BooleanField(default=True) - bookwyrm_user = models.BooleanField(default=True) + outbox = fields.RemoteIdField(unique=True) + summary = fields.TextField(blank=True, null=True) + local = models.BooleanField(default=False) + bookwyrm_user = fields.BooleanField(default=True) localname = models.CharField( max_length=255, null=True, unique=True ) # name is your display name, which you can change at will - name = models.CharField(max_length=100, blank=True, null=True) - avatar = models.ImageField(upload_to='avatars/', blank=True, null=True) - following = models.ManyToManyField( + name = fields.CharField(max_length=100, blank=True, null=True) + avatar = fields.ImageField( + upload_to='avatars/', blank=True, null=True, activitypub_field='icon') + followers = fields.ManyToManyField( 'self', + link_only=True, symmetrical=False, through='UserFollows', - through_fields=('user_subject', 'user_object'), - related_name='followers' + through_fields=('user_object', 'user_subject'), + related_name='following' ) follow_requests = models.ManyToManyField( 'self', @@ -69,60 +79,13 @@ class User(OrderedCollectionPageMixin, AbstractUser): through_fields=('user', 'status'), related_name='favorite_statuses' ) - remote_id = models.CharField(max_length=255, null=True, unique=True) + remote_id = fields.RemoteIdField( + null=True, unique=True, activitypub_field='id') created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now=True) last_active_date = models.DateTimeField(auto_now=True) - manually_approves_followers = models.BooleanField(default=False) + manually_approves_followers = fields.BooleanField(default=False) - # ---- activitypub serialization settings for this model ----- # - @property - def ap_followers(self): - ''' generates url for activitypub followers page ''' - return '%s/followers' % self.remote_id - - @property - def ap_public_key(self): - ''' format the public key block for activitypub ''' - return activitypub.PublicKey(**{ - 'id': '%s/#main-key' % self.remote_id, - 'owner': self.remote_id, - 'publicKeyPem': self.public_key, - }) - - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping( - 'preferredUsername', - 'username', - activity_formatter=lambda x: x.split('@')[0] - ), - ActivityMapping('name', 'name'), - ActivityMapping('bookwyrmUser', 'bookwyrm_user'), - ActivityMapping('inbox', 'inbox'), - ActivityMapping('outbox', 'outbox'), - ActivityMapping('followers', 'ap_followers'), - ActivityMapping('summary', 'summary'), - ActivityMapping( - 'publicKey', - 'public_key', - model_formatter=lambda x: x.get('publicKeyPem') - ), - ActivityMapping('publicKey', 'ap_public_key'), - ActivityMapping( - 'endpoints', - 'shared_inbox', - activity_formatter=lambda x: {'sharedInbox': x}, - model_formatter=lambda x: x.get('sharedInbox') - ), - ActivityMapping('icon', 'avatar'), - ActivityMapping( - 'manuallyApprovesFollowers', - 'manually_approves_followers' - ), - # this field isn't in the activity but should always be false - ActivityMapping(None, 'local', model_formatter=lambda x: False), - ] activity_serializer = activitypub.Person def to_outbox(self, **kwargs): @@ -183,12 +146,30 @@ class User(OrderedCollectionPageMixin, AbstractUser): self.inbox = '%s/inbox' % self.remote_id self.shared_inbox = 'https://%s/inbox' % DOMAIN self.outbox = '%s/outbox' % self.remote_id - if not self.private_key: - self.private_key, self.public_key = create_key_pair() + if not self.key_pair: + self.key_pair = KeyPair.objects.create() return super().save(*args, **kwargs) +class KeyPair(ActivitypubMixin, BookWyrmModel): + ''' public and private keys for a user ''' + private_key = models.TextField(blank=True, null=True) + public_key = fields.TextField( + blank=True, null=True, activitypub_field='publicKeyPem') + + activity_serializer = activitypub.PublicKey + + def get_remote_id(self): + # self.owner is set by the OneToOneField on User + return '%s/#main-key' % self.owner.remote_id + + def save(self, *args, **kwargs): + ''' create a key pair ''' + self.private_key, self.public_key = create_key_pair() + return super().save(*args, **kwargs) + + @receiver(models.signals.post_save, sender=User) def execute_after_save(sender, instance, created, *args, **kwargs): ''' create shelves for new users ''' @@ -217,6 +198,7 @@ def execute_after_save(sender, instance, created, *args, **kwargs): editable=False ).save() + @app.task def set_remote_server(user_id): ''' figure out the user's remote server in the background ''' @@ -227,7 +209,6 @@ def set_remote_server(user_id): user.save() if user.bookwyrm_user: get_remote_reviews.delay(user.outbox) - return def get_or_create_remote_server(domain): diff --git a/bookwyrm/view_actions.py b/bookwyrm/view_actions.py index 29dc6406..cf0e7644 100644 --- a/bookwyrm/view_actions.py +++ b/bookwyrm/view_actions.py @@ -84,7 +84,8 @@ def register(request): } return TemplateResponse(request, 'login.html', data) - user = models.User.objects.create_user(username, email, password) + user = models.User.objects.create_user( + username, email, password, local=True) if invite: invite.times_used += 1 invite.save() From 8bc0a57bd41ac48bf2ee3cf540e3f1a5d4637b0e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 30 Nov 2020 10:32:54 -0800 Subject: [PATCH 024/356] Remove outdated user fields --- .../migrations/0017_auto_20201130_1819.py | 1 + .../migrations/0018_auto_20201130_1832.py | 25 +++++++++++++ .../migrations/0019_auto_20201130_1939.py | 24 +++++++++++++ bookwyrm/models/__init__.py | 2 +- bookwyrm/models/base_model.py | 36 +++++++++++-------- bookwyrm/models/fields.py | 25 ++++++------- bookwyrm/models/user.py | 16 +++++++-- 7 files changed, 96 insertions(+), 33 deletions(-) create mode 100644 bookwyrm/migrations/0018_auto_20201130_1832.py create mode 100644 bookwyrm/migrations/0019_auto_20201130_1939.py diff --git a/bookwyrm/migrations/0017_auto_20201130_1819.py b/bookwyrm/migrations/0017_auto_20201130_1819.py index bab454cb..ce9f1cc7 100644 --- a/bookwyrm/migrations/0017_auto_20201130_1819.py +++ b/bookwyrm/migrations/0017_auto_20201130_1819.py @@ -13,6 +13,7 @@ def copy_rsa_keys(app_registry, schema_editor): for user in users.objects.using(db_alias): if user.public_key or user.private_key: user.key_pair = keypair.objects.create( + remote_id='%s/#main-key' % user.remote_id, private_key=user.private_key, public_key=user.public_key ) diff --git a/bookwyrm/migrations/0018_auto_20201130_1832.py b/bookwyrm/migrations/0018_auto_20201130_1832.py new file mode 100644 index 00000000..278446cf --- /dev/null +++ b/bookwyrm/migrations/0018_auto_20201130_1832.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.7 on 2020-11-30 18:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0017_auto_20201130_1819'), + ] + + operations = [ + migrations.RemoveField( + model_name='user', + name='following', + ), + migrations.RemoveField( + model_name='user', + name='private_key', + ), + migrations.RemoveField( + model_name='user', + name='public_key', + ), + ] diff --git a/bookwyrm/migrations/0019_auto_20201130_1939.py b/bookwyrm/migrations/0019_auto_20201130_1939.py new file mode 100644 index 00000000..c69bdbce --- /dev/null +++ b/bookwyrm/migrations/0019_auto_20201130_1939.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.7 on 2020-11-30 19:39 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0018_auto_20201130_1832'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='name', + field=bookwyrm.models.fields.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='user', + name='summary', + field=bookwyrm.models.fields.TextField(default=''), + ), + ] diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py index 9a4d6014..86bdf219 100644 --- a/bookwyrm/models/__init__.py +++ b/bookwyrm/models/__init__.py @@ -14,7 +14,7 @@ from .attachment import Image from .tag import Tag, UserTag -from .user import User +from .user import User, KeyPair from .relationship import UserFollows, UserFollowRequest, UserBlocks from .federated_server import FederatedServer diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 90b889e0..91fda5a3 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -1,5 +1,4 @@ ''' base model with default fields ''' -from datetime import datetime from base64 import b64encode from dataclasses import dataclass from typing import Callable @@ -10,9 +9,6 @@ from Crypto.PublicKey import RSA from Crypto.Signature import pkcs1_15 from Crypto.Hash import SHA256 from django.db import models -from django.db.models.fields.files import ImageFileDescriptor -from django.db.models.fields.related_descriptors \ - import ManyToManyDescriptor, ReverseManyToOneDescriptor from django.dispatch import receiver from bookwyrm import activitypub @@ -56,24 +52,34 @@ def execute_after_save(sender, instance, created, *args, **kwargs): instance.save() +def get_field_name(field): + ''' model_field_name to activitypubFieldName ''' + if field.activitypub_field: + return field.activitypub_field + name = field.name.split('.')[-1] + components = name.split('_') + return components[0] + ''.join(x.title() for x in components[1:]) + + class ActivitypubMixin: ''' add this mixin for models that are AP serializable ''' activity_serializer = lambda: {} - def to_activity(self, pure=False): + def to_activity(self): ''' convert from a model to an activity ''' activity = {} - for field in self.__class__._meta.fields: - key, value = field.to_activity(getattr(self, field.name)) - activity[key] = value - for related_object in self.__class__.meta.related_objects: - # TODO: check if it's serializable - related_model = related_object.related_model - key = related_object.name - related_values = getattr(self, key) - activity[key] = [i.remote_id for i in related_values] + for field in self.__class__._meta.get_fields(): + if not hasattr(field, 'to_activity'): + continue + key = get_field_name(field) + value = field.to_activity(getattr(self, field.name)) + if value is not None: + activity[key] = value + if hasattr(self, 'serialize_reverse_fields'): + for field_name in self.serialize_reverse_fields: + activity[field_name] = getattr(self, field_name).remote_id - return self.activity_serializer(**activity_json).serialize() + return self.activity_serializer(**activity).serialize() def to_create_activity(self, user, pure=False): diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index c231039c..10e1b100 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -21,25 +21,22 @@ def validate_remote_id(value): ) -def to_camel_case(snake_string): - ''' model_field_name to activitypubFieldName ''' - components = snake_string.split('_') - return components[0] + ''.join(x.title() for x in components[1:]) - - class ActivitypubFieldMixin: ''' make a database field serializable ''' def __init__(self, *args, \ activitypub_field=None, activitypub_wrapper=None, **kwargs): - self.activitypub_wrapper = activitypub_wrapper - self.activitypub_field = activitypub_field + if activitypub_wrapper: + self.activitypub_wrapper = activitypub_field + self.activitypub_field = activitypub_wrapper + else: + self.activitypub_field = activitypub_field super().__init__(*args, **kwargs) def to_activity(self, value): ''' formatter to convert a model value into activitypub ''' - if self.activitypub_wrapper: + if hasattr(self, 'activitypub_wrapper'): value = {self.activitypub_wrapper: value} - return (self.activitypub_field, value) + return value def from_activity(self, activity_data): ''' formatter to convert activitypub into a model value ''' @@ -96,11 +93,9 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): class OneToOneField(ActivitypubFieldMixin, models.OneToOneField): ''' activitypub-aware foreign key field ''' - def __init__(self, *args, **kwargs): - super(ActivitypubFieldMixin, self).__init__(*args, **kwargs) - def to_activity(self, value): - return value.remote_id + return value.to_activity() + def from_activity(self, activity_data): pass# TODO @@ -113,7 +108,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): def to_activity(self, value): if self.link_only: - return '%s/followers' % self.instance.remote_id + return '%s/followers' % value.instance.remote_id return [i.remote_id for i in value] def from_activity(self, activity_data): diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 19746eb1..f480476f 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -26,10 +26,12 @@ class User(OrderedCollectionPageMixin, AbstractUser): 'KeyPair', on_delete=models.CASCADE, blank=True, null=True, + activitypub_field='publicKey', related_name='owner' ) inbox = fields.RemoteIdField(unique=True) shared_inbox = fields.RemoteIdField( + activitypub_field='sharedInbox', activitypub_wrapper='endpoints', null=True) federated_server = models.ForeignKey( 'FederatedServer', @@ -38,7 +40,7 @@ class User(OrderedCollectionPageMixin, AbstractUser): blank=True, ) outbox = fields.RemoteIdField(unique=True) - summary = fields.TextField(blank=True, null=True) + summary = fields.TextField(default='') local = models.BooleanField(default=False) bookwyrm_user = fields.BooleanField(default=True) localname = models.CharField( @@ -47,7 +49,7 @@ class User(OrderedCollectionPageMixin, AbstractUser): unique=True ) # name is your display name, which you can change at will - name = fields.CharField(max_length=100, blank=True, null=True) + name = fields.CharField(max_length=100, default='') avatar = fields.ImageField( upload_to='avatars/', blank=True, null=True, activitypub_field='icon') followers = fields.ManyToManyField( @@ -87,6 +89,7 @@ class User(OrderedCollectionPageMixin, AbstractUser): manually_approves_followers = fields.BooleanField(default=False) activity_serializer = activitypub.Person + serialize_related = [] def to_outbox(self, **kwargs): ''' an ordered collection of statuses ''' @@ -159,6 +162,7 @@ class KeyPair(ActivitypubMixin, BookWyrmModel): blank=True, null=True, activitypub_field='publicKeyPem') activity_serializer = activitypub.PublicKey + serialize_reverse_fields = ['owner'] def get_remote_id(self): # self.owner is set by the OneToOneField on User @@ -169,6 +173,14 @@ class KeyPair(ActivitypubMixin, BookWyrmModel): self.private_key, self.public_key = create_key_pair() return super().save(*args, **kwargs) + def to_activity(self): + ''' override default AP serializer to add context object + idk if this is the best way to go about this ''' + activity_object = super().to_activity() + del activity_object['@context'] + del activity_object['type'] + return activity_object + @receiver(models.signals.post_save, sender=User) def execute_after_save(sender, instance, created, *args, **kwargs): From 3966c84e08931908e6e3759afb5368a49e53ae37 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 30 Nov 2020 14:24:31 -0800 Subject: [PATCH 025/356] Updates status model and serializer --- bookwyrm/activitypub/note.py | 10 +- bookwyrm/models/attachment.py | 15 ++- bookwyrm/models/base_model.py | 55 ++++------ bookwyrm/models/book.py | 1 + bookwyrm/models/fields.py | 57 +++++++++-- bookwyrm/models/status.py | 185 ++++++++++++++-------------------- bookwyrm/models/user.py | 10 +- 7 files changed, 167 insertions(+), 166 deletions(-) diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py index aeb078dc..df28bf8d 100644 --- a/bookwyrm/activitypub/note.py +++ b/bookwyrm/activitypub/note.py @@ -8,7 +8,6 @@ from .image import Image @dataclass(init=False) class Tombstone(ActivityObject): ''' the placeholder for a deleted status ''' - url: str published: str deleted: str type: str = 'Tombstone' @@ -17,14 +16,13 @@ class Tombstone(ActivityObject): @dataclass(init=False) class Note(ActivityObject): ''' Note activity ''' - url: str - inReplyTo: str published: str attributedTo: str - to: List[str] - cc: List[str] content: str - replies: Dict + to: List[str] = field(default_factory=lambda: []) + cc: List[str] = field(default_factory=lambda: []) + replies: Dict = field(default_factory=lambda: {}) + inReplyTo: str = '' tag: List[Link] = field(default_factory=lambda: []) attachment: List[Image] = field(default_factory=lambda: []) sensitive: bool = False diff --git a/bookwyrm/models/attachment.py b/bookwyrm/models/attachment.py index 7329e65d..6a92240d 100644 --- a/bookwyrm/models/attachment.py +++ b/bookwyrm/models/attachment.py @@ -3,30 +3,27 @@ from django.db import models from bookwyrm import activitypub from .base_model import ActivitypubMixin -from .base_model import ActivityMapping, BookWyrmModel +from .base_model import BookWyrmModel +from . import fields class Attachment(ActivitypubMixin, BookWyrmModel): ''' an image (or, in the future, video etc) associated with a status ''' - status = models.ForeignKey( + status = fields.ForeignKey( 'Status', on_delete=models.CASCADE, related_name='attachments', null=True ) + reverse_unfurl = True class Meta: ''' one day we'll have other types of attachments besides images ''' abstract = True - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('url', 'image'), - ActivityMapping('name', 'caption'), - ] class Image(Attachment): ''' an image attachment ''' - image = models.ImageField(upload_to='status/', null=True, blank=True) - caption = models.TextField(null=True, blank=True) + image = fields.ImageField(upload_to='status/', null=True, blank=True) + caption = fields.TextField(null=True, blank=True) activity_serializer = activitypub.Image diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 91fda5a3..34c4e5bc 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -61,9 +61,19 @@ def get_field_name(field): return components[0] + ''.join(x.title() for x in components[1:]) +def unfurl_related_field(related_field): + ''' load reverse lookups (like public key owner or Status attachment ''' + if hasattr(related_field, 'all'): + return [unfurl_related_field(i) for i in related_field.all()] + if related_field.reverse_unfurl: + return related_field.to_activity() + return related_field.remote_id + + class ActivitypubMixin: ''' add this mixin for models that are AP serializable ''' activity_serializer = lambda: {} + reverse_unfurl = False def to_activity(self): ''' convert from a model to an activity ''' @@ -73,18 +83,24 @@ class ActivitypubMixin: continue key = get_field_name(field) value = field.to_activity(getattr(self, field.name)) - if value is not None: + if value is None: + continue + + if key in activity and isinstance(activity[key], list): + activity[key] += value + else: activity[key] = value if hasattr(self, 'serialize_reverse_fields'): for field_name in self.serialize_reverse_fields: - activity[field_name] = getattr(self, field_name).remote_id + related_field = getattr(self, field_name) + activity[field_name] = unfurl_related_field(related_field) return self.activity_serializer(**activity).serialize() - def to_create_activity(self, user, pure=False): + def to_create_activity(self, user): ''' returns the object wrapped in a Create activity ''' - activity_object = self.to_activity(pure=pure) + activity_object = self.to_activity() signer = pkcs1_15.new(RSA.import_key(user.private_key)) content = activity_object['content'] @@ -100,8 +116,8 @@ class ActivitypubMixin: return activitypub.Create( id=create_id, actor=user.remote_id, - to=['%s/followers' % user.remote_id], - cc=['https://www.w3.org/ns/activitystreams#Public'], + to=activity_object['to'], + cc=activity_object['cc'], object=activity_object, signature=signature, ).serialize() @@ -245,30 +261,3 @@ class ActivityMapping: model_key: str activity_formatter: Callable = lambda x: x model_formatter: Callable = lambda x: x - - -def tag_formatter(items, name_field, activity_type): - ''' helper function to format lists of foreign keys into Tags ''' - tags = [] - for item in items.all(): - tags.append(activitypub.Link( - href=item.remote_id, - name=getattr(item, name_field), - type=activity_type - )) - return tags - - -def image_formatter(image): - ''' convert images into activitypub json ''' - if image and hasattr(image, 'url'): - url = image.url - else: - return None - url = 'https://%s%s' % (DOMAIN, url) - return activitypub.Image(url=url) - - -def image_attachments_formatter(images): - ''' create a list of image attachments ''' - return [image_formatter(i) for i in images] diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index ffc9dd8a..fc7c9e91 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -155,6 +155,7 @@ class Edition(Book): 'Work', on_delete=models.PROTECT, null=True, related_name='editions') activity_serializer = activitypub.Edition + name_field = 'title' def save(self, *args, **kwargs): ''' calculate isbn 10/13 ''' diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 10e1b100..e419c9b8 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -71,6 +71,7 @@ class UsernameField(ActivitypubFieldMixin, models.CharField): ) def deconstruct(self): + ''' implementation of models.Field deconstruct ''' name, path, args, kwargs = super().deconstruct() del kwargs['verbose_name'] del kwargs['max_length'] @@ -86,6 +87,8 @@ class UsernameField(ActivitypubFieldMixin, models.CharField): class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): ''' activitypub-aware foreign key field ''' def to_activity(self, value): + if not value: + return None return value.remote_id def from_activity(self, activity_data): pass# TODO @@ -94,6 +97,10 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): class OneToOneField(ActivitypubFieldMixin, models.OneToOneField): ''' activitypub-aware foreign key field ''' def to_activity(self, value): + print('HIIIII') + print(value) + if not value: + return None return value.to_activity() def from_activity(self, activity_data): @@ -113,20 +120,45 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): def from_activity(self, activity_data): if self.link_only: - return - values = super().from_activity(self, activity_data) + return None + values = super().from_activity(activity_data) return values# TODO +class TagField(ManyToManyField): + ''' special case of many to many that uses Tags ''' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.activitypub_field = 'tag' + + def to_activity(self, value): + tags = [] + for item in value.all(): + activity_type = item.__class__.__name__ + if activity_type == 'User': + activity_type = 'Mention' + tags.append(activitypub.Link( + href=item.remote_id, + name=getattr(item, item.name_field), + type=activity_type + )) + return tags + + +def image_serializer(value): + ''' helper for serializing images ''' + print(value) + if value and hasattr(value, 'url'): + url = value.url + else: + return None + url = 'https://%s%s' % (DOMAIN, url) + return activitypub.Image(url=url) + class ImageField(ActivitypubFieldMixin, models.ImageField): ''' activitypub-aware image field ''' def to_activity(self, value): - if value and hasattr(value, 'url'): - url = value.url - else: - return None - url = 'https://%s%s' % (DOMAIN, url) - return activitypub.Image(url=url) + return image_serializer(value) def from_activity(self, activity_data): image_slug = super().from_activity(activity_data) @@ -150,6 +182,12 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): return [image_name, image_content] +class DateTimeField(ActivitypubFieldMixin, models.DateTimeField): + ''' activitypub-aware datetime field ''' + def to_activity(self, value): + return value.isoformat() + + class CharField(ActivitypubFieldMixin, models.CharField): ''' activitypub-aware char field ''' @@ -158,3 +196,6 @@ class TextField(ActivitypubFieldMixin, models.TextField): class BooleanField(ActivitypubFieldMixin, models.BooleanField): ''' activitypub-aware boolean field ''' + +class IntegerField(ActivitypubFieldMixin, models.IntegerField): + ''' activitypub-aware boolean field ''' diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index a520164a..c47a33bb 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -6,26 +6,27 @@ from model_utils.managers import InheritanceManager from bookwyrm import activitypub from .base_model import ActivitypubMixin, OrderedCollectionPageMixin -from .base_model import ActivityMapping, BookWyrmModel, PrivacyLevels -from .base_model import tag_formatter, image_attachments_formatter - +from .base_model import BookWyrmModel, PrivacyLevels +from . import fields +from .fields import image_serializer class Status(OrderedCollectionPageMixin, BookWyrmModel): ''' any post, like a reply to a review, etc ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - content = models.TextField(blank=True, null=True) - mention_users = models.ManyToManyField('User', related_name='mention_user') - mention_books = models.ManyToManyField( - 'Edition', related_name='mention_book') + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='attributedTo') + content = fields.TextField(blank=True, null=True) + mention_users = fields.TagField('User', related_name='mention_user') + mention_books = fields.TagField('Edition', related_name='mention_book') local = models.BooleanField(default=True) privacy = models.CharField( max_length=255, default='public', choices=PrivacyLevels.choices ) - sensitive = models.BooleanField(default=False) + sensitive = fields.BooleanField(default=False) # the created date can't be this, because of receiving federated posts - published_date = models.DateTimeField(default=timezone.now) + published_date = fields.DateTimeField( + default=timezone.now, activitypub_field='published') deleted = models.BooleanField(default=False) deleted_date = models.DateTimeField(blank=True, null=True) favorites = models.ManyToManyField( @@ -35,79 +36,21 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): through_fields=('status', 'user'), related_name='user_favorites' ) - reply_parent = models.ForeignKey( + reply_parent = fields.ForeignKey( 'self', null=True, - on_delete=models.PROTECT + on_delete=models.PROTECT, + activitypub_field='inReplyTo', ) objects = InheritanceManager() - # ---- activitypub serialization settings for this model ----- # - @property - def ap_to(self): - ''' should be related to post privacy I think ''' - return ['https://www.w3.org/ns/activitystreams#Public'] - - @property - def ap_cc(self): - ''' should be related to post privacy I think ''' - return [self.user.ap_followers] - @property def ap_replies(self): ''' structured replies block ''' return self.to_replies() - @property - def ap_status_image(self): - ''' attach a book cover, if relevent ''' - if hasattr(self, 'book'): - return self.book.ap_cover - if self.mention_books.first(): - return self.mention_books.first().ap_cover - return None - - - shared_mappings = [ - ActivityMapping('url', 'remote_id', lambda x: None), - ActivityMapping('id', 'remote_id'), - ActivityMapping('inReplyTo', 'reply_parent'), - ActivityMapping('published', 'published_date'), - ActivityMapping('attributedTo', 'user'), - ActivityMapping('to', 'ap_to'), - ActivityMapping('cc', 'ap_cc'), - ActivityMapping('replies', 'ap_replies'), - ActivityMapping( - 'tag', 'mention_books', - lambda x: tag_formatter(x, 'title', 'Book'), - ), - ActivityMapping( - 'tag', 'mention_users', - lambda x: tag_formatter(x, 'username', 'Mention'), - ), - ActivityMapping( - 'attachment', 'attachments', - lambda x: image_attachments_formatter(x.all()), - ) - ] - - # serializing to bookwyrm expanded activitypub - activity_mappings = shared_mappings + [ - ActivityMapping('name', 'name'), - ActivityMapping('inReplyToBook', 'book'), - ActivityMapping('rating', 'rating'), - ActivityMapping('quote', 'quote'), - ActivityMapping('content', 'content'), - ] - - # for serializing to standard activitypub without extended types - pure_activity_mappings = shared_mappings + [ - ActivityMapping('name', 'ap_pure_name'), - ActivityMapping('content', 'ap_pure_content'), - ActivityMapping('attachment', 'ap_status_image'), - ] - activity_serializer = activitypub.Note + serialize_reverse_fields = ['attachments'] #----- replies collection activitypub ----# @classmethod @@ -138,7 +81,43 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): deleted=self.deleted_date.isoformat(), published=self.deleted_date.isoformat() ).serialize() - return ActivitypubMixin.to_activity(self, pure=pure) + activity = ActivitypubMixin.to_activity(self) + activity['replies'] = self.to_replies() + + # privacy controls + public = 'https://www.w3.org/ns/activitystreams#Public' + mentions = [u.remote_id for u in self.mention_users.all()] + # this is a link to the followers list: + followers = self.user.__class__._meta.get_field('followers')\ + .to_activity(self.user.followers) + if self.privacy == 'public': + activity['to'] = [public] + activity['cc'] = [followers] + mentions + elif self.privacy == 'unlisted': + activity['to'] = [followers] + activity['cc'] = [public] + mentions + elif self.privacy == 'followers': + activity['to'] = [followers] + activity['cc'] = mentions + if self.privacy == 'direct': + activity['to'] = mentions + activity['cc'] = [] + + # "pure" serialization for non-bookwyrm instances + if pure: + activity['content'] = self.pure_content + if 'name' in activity: + activity['name'] = self.pure_name + activity['type'] = self.pure_type + activity['attachment'] = [ + image_serializer(b.cover) for b in self.mention_books.all() \ + if b.cover] + if hasattr(self, 'book'): + activity['attachment'].append( + image_serializer(self.book.cover) + ) + return activity + def save(self, *args, **kwargs): ''' update user active time ''' @@ -151,40 +130,40 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): class GeneratedNote(Status): ''' these are app-generated messages about user activity ''' @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' message = self.content books = ', '.join( - '"%s"' % (self.book.remote_id, self.book.title) \ + '"%s"' % (book.remote_id, book.title) \ for book in self.mention_books.all() ) - return '%s %s' % (message, books) + return '%s %s %s' % (self.user.display_name, message, books) activity_serializer = activitypub.GeneratedNote - pure_activity_serializer = activitypub.Note + pure_type = 'Note' class Comment(Status): ''' like a review but without a rating and transient ''' - book = models.ForeignKey('Edition', on_delete=models.PROTECT) + book = fields.ForeignKey('Edition', on_delete=models.PROTECT) @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' return self.content + '

(comment on "%s")' % \ (self.book.remote_id, self.book.title) activity_serializer = activitypub.Comment - pure_activity_serializer = activitypub.Note + pure_type = 'Note' class Quotation(Status): ''' like a review but without a rating and transient ''' - quote = models.TextField() - book = models.ForeignKey('Edition', on_delete=models.PROTECT) + quote = fields.TextField() + book = fields.ForeignKey('Edition', on_delete=models.PROTECT) @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' return '"%s"
-- "%s"

%s' % ( self.quote, @@ -194,14 +173,14 @@ class Quotation(Status): ) activity_serializer = activitypub.Quotation - pure_activity_serializer = activitypub.Note + pure_type = 'Note' class Review(Status): ''' a book review ''' - name = models.CharField(max_length=255, null=True) - book = models.ForeignKey('Edition', on_delete=models.PROTECT) - rating = models.IntegerField( + name = fields.CharField(max_length=255, null=True) + book = fields.ForeignKey('Edition', on_delete=models.PROTECT) + rating = fields.IntegerField( default=None, null=True, blank=True, @@ -209,7 +188,7 @@ class Review(Status): ) @property - def ap_pure_name(self): + def pure_name(self): ''' clarify review names for mastodon serialization ''' if self.rating: return 'Review of "%s" (%d stars): %s' % ( @@ -223,26 +202,21 @@ class Review(Status): ) @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' return self.content + '

("%s")' % \ (self.book.remote_id, self.book.title) activity_serializer = activitypub.Review - pure_activity_serializer = activitypub.Article + pure_type = 'Article' class Favorite(ActivitypubMixin, BookWyrmModel): ''' fav'ing a post ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - status = models.ForeignKey('Status', on_delete=models.PROTECT) - - # ---- activitypub serialization settings for this model ----- # - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'user'), - ActivityMapping('object', 'status'), - ] + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='actor') + status = fields.ForeignKey( + 'Status', on_delete=models.PROTECT, activitypub_field='object') activity_serializer = activitypub.Like @@ -252,7 +226,6 @@ class Favorite(ActivitypubMixin, BookWyrmModel): self.user.save() super().save(*args, **kwargs) - class Meta: ''' can't fav things twice ''' unique_together = ('user', 'status') @@ -260,16 +233,12 @@ class Favorite(ActivitypubMixin, BookWyrmModel): class Boost(Status): ''' boost'ing a post ''' - boosted_status = models.ForeignKey( + boosted_status = fields.ForeignKey( 'Status', on_delete=models.PROTECT, - related_name="boosters") - - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'user'), - ActivityMapping('object', 'boosted_status'), - ] + related_name='boosters', + activitypub_field='object', + ) activity_serializer = activitypub.Boost diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index f480476f..95dd1e79 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -88,8 +88,14 @@ class User(OrderedCollectionPageMixin, AbstractUser): last_active_date = models.DateTimeField(auto_now=True) manually_approves_followers = fields.BooleanField(default=False) + @property + def display_name(self): + ''' show the cleanest version of the user's name possible ''' + if self.name != '': + return self.name + return self.localname or self.username + activity_serializer = activitypub.Person - serialize_related = [] def to_outbox(self, **kwargs): ''' an ordered collection of statuses ''' @@ -112,7 +118,7 @@ class User(OrderedCollectionPageMixin, AbstractUser): return self.to_ordered_collection(self.followers, \ remote_id=remote_id, id_only=True, **kwargs) - def to_activity(self, pure=False): + def to_activity(self): ''' override default AP serializer to add context object idk if this is the best way to go about this ''' activity_object = super().to_activity() From 77aead722d7bb4fd4ffcfadf243f2c496ded20ce Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 30 Nov 2020 14:40:26 -0800 Subject: [PATCH 026/356] serialize book and author models --- bookwyrm/models/author.py | 28 ++++------- bookwyrm/models/book.py | 98 +++++++++++++-------------------------- bookwyrm/models/fields.py | 11 +++-- 3 files changed, 49 insertions(+), 88 deletions(-) diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index fdaf73d9..5b70b57f 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -4,29 +4,29 @@ from django.utils import timezone from bookwyrm import activitypub from bookwyrm.settings import DOMAIN -from bookwyrm.utils.fields import ArrayField -from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel +from .base_model import ActivitypubMixin, BookWyrmModel +from . import fields class Author(ActivitypubMixin, BookWyrmModel): ''' basic biographic info ''' origin_id = models.CharField(max_length=255, null=True) ''' copy of an author from OL ''' - openlibrary_key = models.CharField(max_length=255, blank=True, null=True) + openlibrary_key = fields.CharField(max_length=255, blank=True, null=True) sync = models.BooleanField(default=True) last_sync_date = models.DateTimeField(default=timezone.now) - wikipedia_link = models.CharField(max_length=255, blank=True, null=True) + wikipedia_link = fields.CharField(max_length=255, blank=True, null=True) # idk probably other keys would be useful here? - born = models.DateTimeField(blank=True, null=True) - died = models.DateTimeField(blank=True, null=True) - name = models.CharField(max_length=255) + born = fields.DateTimeField(blank=True, null=True) + died = fields.DateTimeField(blank=True, null=True) + name = fields.CharField(max_length=255) last_name = models.CharField(max_length=255, blank=True, null=True) first_name = models.CharField(max_length=255, blank=True, null=True) - aliases = ArrayField( + aliases = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) - bio = models.TextField(null=True, blank=True) + bio = fields.TextField(null=True, blank=True) def save(self, *args, **kwargs): ''' can't be abstract for query reasons, but you shouldn't USE it ''' @@ -52,14 +52,4 @@ class Author(ActivitypubMixin, BookWyrmModel): return self.first_name + ' ' + self.last_name return self.last_name or self.first_name - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('name', 'name'), - ActivityMapping('born', 'born'), - ActivityMapping('died', 'died'), - ActivityMapping('aliases', 'aliases'), - ActivityMapping('bio', 'bio'), - ActivityMapping('openlibraryKey', 'openlibrary_key'), - ActivityMapping('wikipediaLink', 'wikipedia_link'), - ] activity_serializer = activitypub.Author diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index fc7c9e91..da532561 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -7,18 +7,18 @@ from model_utils.managers import InheritanceManager from bookwyrm import activitypub from bookwyrm.settings import DOMAIN -from bookwyrm.utils.fields import ArrayField -from .base_model import ActivityMapping, BookWyrmModel +from .base_model import BookWyrmModel from .base_model import ActivitypubMixin, OrderedCollectionPageMixin +from . import fields class Book(ActivitypubMixin, BookWyrmModel): ''' a generic book, which can mean either an edition or a work ''' origin_id = models.CharField(max_length=255, null=True, blank=True) # these identifiers apply to both works and editions - openlibrary_key = models.CharField(max_length=255, blank=True, null=True) - librarything_key = models.CharField(max_length=255, blank=True, null=True) - goodreads_key = models.CharField(max_length=255, blank=True, null=True) + openlibrary_key = fields.CharField(max_length=255, blank=True, null=True) + librarything_key = fields.CharField(max_length=255, blank=True, null=True) + goodreads_key = fields.CharField(max_length=255, blank=True, null=True) # info about where the data comes from and where/if to sync sync = models.BooleanField(default=True) @@ -30,66 +30,31 @@ class Book(ActivitypubMixin, BookWyrmModel): # TODO: edit history # book/work metadata - title = models.CharField(max_length=255) - sort_title = models.CharField(max_length=255, blank=True, null=True) - subtitle = models.CharField(max_length=255, blank=True, null=True) - description = models.TextField(blank=True, null=True) - languages = ArrayField( + title = fields.CharField(max_length=255) + sort_title = fields.CharField(max_length=255, blank=True, null=True) + subtitle = fields.CharField(max_length=255, blank=True, null=True) + description = fields.TextField(blank=True, null=True) + languages = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) - series = models.CharField(max_length=255, blank=True, null=True) - series_number = models.CharField(max_length=255, blank=True, null=True) - subjects = ArrayField( + series = fields.CharField(max_length=255, blank=True, null=True) + series_number = fields.CharField(max_length=255, blank=True, null=True) + subjects = fields.ArrayField( models.CharField(max_length=255), blank=True, null=True, default=list ) - subject_places = ArrayField( + subject_places = fields.ArrayField( models.CharField(max_length=255), blank=True, null=True, default=list ) # TODO: include an annotation about the type of authorship (ie, translator) - authors = models.ManyToManyField('Author') + authors = fields.ManyToManyField('Author') # preformatted authorship string for search and easier display author_text = models.CharField(max_length=255, blank=True, null=True) - cover = models.ImageField(upload_to='covers/', blank=True, null=True) - first_published_date = models.DateTimeField(blank=True, null=True) - published_date = models.DateTimeField(blank=True, null=True) + cover = fields.ImageField(upload_to='covers/', blank=True, null=True) + first_published_date = fields.DateTimeField(blank=True, null=True) + published_date = fields.DateTimeField(blank=True, null=True) + objects = InheritanceManager() - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - - ActivityMapping('authors', 'authors'), - ActivityMapping('firstPublishedDate', 'firstpublished_date'), - ActivityMapping('publishedDate', 'published_date'), - - ActivityMapping('title', 'title'), - ActivityMapping('sortTitle', 'sort_title'), - ActivityMapping('subtitle', 'subtitle'), - ActivityMapping('description', 'description'), - ActivityMapping('languages', 'languages'), - ActivityMapping('series', 'series'), - ActivityMapping('seriesNumber', 'series_number'), - ActivityMapping('subjects', 'subjects'), - ActivityMapping('subjectPlaces', 'subject_places'), - - ActivityMapping('openlibraryKey', 'openlibrary_key'), - ActivityMapping('librarythingKey', 'librarything_key'), - ActivityMapping('goodreadsKey', 'goodreads_key'), - - ActivityMapping('work', 'parent_work'), - ActivityMapping('isbn10', 'isbn_10'), - ActivityMapping('isbn13', 'isbn_13'), - ActivityMapping('oclcNumber', 'oclc_number'), - ActivityMapping('asin', 'asin'), - ActivityMapping('pages', 'pages'), - ActivityMapping('physicalFormat', 'physical_format'), - ActivityMapping('publishers', 'publishers'), - - ActivityMapping('lccn', 'lccn'), - ActivityMapping('editions', 'editions'), - ActivityMapping('defaultEdition', 'default_edition'), - ActivityMapping('cover', 'cover'), - ] - def save(self, *args, **kwargs): ''' can't be abstract for query reasons, but you shouldn't USE it ''' if not isinstance(self, Edition) and not isinstance(self, Work): @@ -118,9 +83,9 @@ class Book(ActivitypubMixin, BookWyrmModel): class Work(OrderedCollectionPageMixin, Book): ''' a work (an abstract concept of a book that manifests in an edition) ''' # library of congress catalog control number - lccn = models.CharField(max_length=255, blank=True, null=True) + lccn = fields.CharField(max_length=255, blank=True, null=True) # this has to be nullable but should never be null - default_edition = models.ForeignKey( + default_edition = fields.ForeignKey( 'Edition', on_delete=models.PROTECT, null=True @@ -131,18 +96,19 @@ class Work(OrderedCollectionPageMixin, Book): return self.default_edition or self.editions.first() activity_serializer = activitypub.Work + serialize_reverse_fields = ['editions'] class Edition(Book): ''' an edition of a book ''' # these identifiers only apply to editions, not works - isbn_10 = models.CharField(max_length=255, blank=True, null=True) - isbn_13 = models.CharField(max_length=255, blank=True, null=True) - oclc_number = models.CharField(max_length=255, blank=True, null=True) - asin = models.CharField(max_length=255, blank=True, null=True) - pages = models.IntegerField(blank=True, null=True) - physical_format = models.CharField(max_length=255, blank=True, null=True) - publishers = ArrayField( + isbn_10 = fields.CharField(max_length=255, blank=True, null=True) + isbn_13 = fields.CharField(max_length=255, blank=True, null=True) + oclc_number = fields.CharField(max_length=255, blank=True, null=True) + asin = fields.CharField(max_length=255, blank=True, null=True) + pages = fields.IntegerField(blank=True, null=True) + physical_format = fields.CharField(max_length=255, blank=True, null=True) + publishers = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) shelves = models.ManyToManyField( @@ -151,8 +117,9 @@ class Edition(Book): through='ShelfBook', through_fields=('book', 'shelf') ) - parent_work = models.ForeignKey( - 'Work', on_delete=models.PROTECT, null=True, related_name='editions') + parent_work = fields.ForeignKey( + 'Work', on_delete=models.PROTECT, null=True, + related_name='editions', activitypub_field='work') activity_serializer = activitypub.Edition name_field = 'title' @@ -167,7 +134,6 @@ class Edition(Book): return super().save(*args, **kwargs) - def isbn_10_to_13(isbn_10): ''' convert an isbn 10 into an isbn 13 ''' isbn_10 = re.sub(r'[^0-9X]', '', isbn_10) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index e419c9b8..b03c5492 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -3,6 +3,7 @@ import re from uuid import uuid4 from django.contrib.auth.models import AbstractUser +from django.contrib.postgres.fields import ArrayField as DjangoArrayField from django.core.exceptions import ValidationError from django.core.files.base import ContentFile from django.db import models @@ -97,8 +98,6 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): class OneToOneField(ActivitypubFieldMixin, models.OneToOneField): ''' activitypub-aware foreign key field ''' def to_activity(self, value): - print('HIIIII') - print(value) if not value: return None return value.to_activity() @@ -116,7 +115,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): def to_activity(self, value): if self.link_only: return '%s/followers' % value.instance.remote_id - return [i.remote_id for i in value] + return [i.remote_id for i in value.all()] def from_activity(self, activity_data): if self.link_only: @@ -185,8 +184,14 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): class DateTimeField(ActivitypubFieldMixin, models.DateTimeField): ''' activitypub-aware datetime field ''' def to_activity(self, value): + if not value: + return None return value.isoformat() +class ArrayField(ActivitypubFieldMixin, DjangoArrayField): + ''' activitypub-aware array field ''' + def to_activity(self, value): + return [str(i) for i in value] class CharField(ActivitypubFieldMixin, models.CharField): ''' activitypub-aware char field ''' From 1ec2f204864e5b8da3d0fbeb4f7c4a9f007472d9 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 30 Nov 2020 14:54:45 -0800 Subject: [PATCH 027/356] avoid naming clash is to_activity for field vs model --- bookwyrm/models/base_model.py | 6 +++--- bookwyrm/models/fields.py | 18 ++++++++--------- bookwyrm/models/relationship.py | 18 ++++++++--------- bookwyrm/models/shelf.py | 34 +++++++++++++-------------------- bookwyrm/models/status.py | 2 +- 5 files changed, 34 insertions(+), 44 deletions(-) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 34c4e5bc..af65a36a 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -66,7 +66,7 @@ def unfurl_related_field(related_field): if hasattr(related_field, 'all'): return [unfurl_related_field(i) for i in related_field.all()] if related_field.reverse_unfurl: - return related_field.to_activity() + return related_field.field_to_activity() return related_field.remote_id @@ -79,10 +79,10 @@ class ActivitypubMixin: ''' convert from a model to an activity ''' activity = {} for field in self.__class__._meta.get_fields(): - if not hasattr(field, 'to_activity'): + if not hasattr(field, 'field_to_activity'): continue key = get_field_name(field) - value = field.to_activity(getattr(self, field.name)) + value = field.field_to_activity(getattr(self, field.name)) if value is None: continue diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index b03c5492..197b6e8e 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -33,7 +33,7 @@ class ActivitypubFieldMixin: self.activitypub_field = activitypub_field super().__init__(*args, **kwargs) - def to_activity(self, value): + def field_to_activity(self, value): ''' formatter to convert a model value into activitypub ''' if hasattr(self, 'activitypub_wrapper'): value = {self.activitypub_wrapper: value} @@ -81,13 +81,13 @@ class UsernameField(ActivitypubFieldMixin, models.CharField): del kwargs['error_messages'] return name, path, args, kwargs - def to_activity(self, value): + def field_to_activity(self, value): return value.split('@')[0] class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): ''' activitypub-aware foreign key field ''' - def to_activity(self, value): + def field_to_activity(self, value): if not value: return None return value.remote_id @@ -97,7 +97,7 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): class OneToOneField(ActivitypubFieldMixin, models.OneToOneField): ''' activitypub-aware foreign key field ''' - def to_activity(self, value): + def field_to_activity(self, value): if not value: return None return value.to_activity() @@ -112,7 +112,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): self.link_only = link_only super().__init__(*args, **kwargs) - def to_activity(self, value): + def field_to_activity(self, value): if self.link_only: return '%s/followers' % value.instance.remote_id return [i.remote_id for i in value.all()] @@ -129,7 +129,7 @@ class TagField(ManyToManyField): super().__init__(*args, **kwargs) self.activitypub_field = 'tag' - def to_activity(self, value): + def field_to_activity(self, value): tags = [] for item in value.all(): activity_type = item.__class__.__name__ @@ -156,7 +156,7 @@ def image_serializer(value): class ImageField(ActivitypubFieldMixin, models.ImageField): ''' activitypub-aware image field ''' - def to_activity(self, value): + def field_to_activity(self, value): return image_serializer(value) def from_activity(self, activity_data): @@ -183,14 +183,14 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): class DateTimeField(ActivitypubFieldMixin, models.DateTimeField): ''' activitypub-aware datetime field ''' - def to_activity(self, value): + def field_to_activity(self, value): if not value: return None return value.isoformat() class ArrayField(ActivitypubFieldMixin, DjangoArrayField): ''' activitypub-aware array field ''' - def to_activity(self, value): + def field_to_activity(self, value): return [str(i) for i in value] class CharField(ActivitypubFieldMixin, models.CharField): diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index dbf99778..8913b9ab 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -2,20 +2,23 @@ from django.db import models from bookwyrm import activitypub -from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel +from .base_model import ActivitypubMixin, BookWyrmModel +from . import fields class UserRelationship(ActivitypubMixin, BookWyrmModel): ''' many-to-many through table for followers ''' - user_subject = models.ForeignKey( + user_subject = fields.ForeignKey( 'User', on_delete=models.PROTECT, - related_name='%(class)s_user_subject' + related_name='%(class)s_user_subject', + activitypub_field='actor', ) - user_object = models.ForeignKey( + user_object = fields.ForeignKey( 'User', on_delete=models.PROTECT, - related_name='%(class)s_user_object' + related_name='%(class)s_user_object', + activitypub_field='object', ) class Meta: @@ -32,11 +35,6 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel): ) ] - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'user_subject'), - ActivityMapping('object', 'user_object'), - ] activity_serializer = activitypub.Follow def get_remote_id(self, status=None): diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py index b8d5ea17..fc63d198 100644 --- a/bookwyrm/models/shelf.py +++ b/bookwyrm/models/shelf.py @@ -3,17 +3,19 @@ import re from django.db import models from bookwyrm import activitypub -from .base_model import ActivityMapping, BookWyrmModel +from .base_model import BookWyrmModel from .base_model import OrderedCollectionMixin, PrivacyLevels +from . import fields class Shelf(OrderedCollectionMixin, BookWyrmModel): ''' a list of books owned by a user ''' - name = models.CharField(max_length=100) + name = fields.CharField(max_length=100) identifier = models.CharField(max_length=100) - user = models.ForeignKey('User', on_delete=models.PROTECT) + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='owner') editable = models.BooleanField(default=True) - privacy = models.CharField( + privacy = fields.CharField( max_length=255, default='public', choices=PrivacyLevels.choices @@ -48,31 +50,21 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): ''' user/shelf unqiueness ''' unique_together = ('user', 'identifier') - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('owner', 'user'), - ActivityMapping('name', 'name'), - ] - class ShelfBook(BookWyrmModel): ''' many to many join table for books and shelves ''' - book = models.ForeignKey('Edition', on_delete=models.PROTECT) - shelf = models.ForeignKey('Shelf', on_delete=models.PROTECT) - added_by = models.ForeignKey( + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='object') + shelf = fields.ForeignKey( + 'Shelf', on_delete=models.PROTECT, activitypub_field='target') + added_by = fields.ForeignKey( 'User', blank=True, null=True, - on_delete=models.PROTECT + on_delete=models.PROTECT, + activitypub_field='actor' ) - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'added_by'), - ActivityMapping('object', 'book'), - ActivityMapping('target', 'shelf'), - ] - activity_serializer = activitypub.AddBook def to_add_activity(self, user): diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index c47a33bb..371a6345 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -89,7 +89,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): mentions = [u.remote_id for u in self.mention_users.all()] # this is a link to the followers list: followers = self.user.__class__._meta.get_field('followers')\ - .to_activity(self.user.followers) + .field_to_activity(self.user.followers) if self.privacy == 'public': activity['to'] = [public] activity['cc'] = [followers] + mentions From eb6206252d7509b45f79fe06a769adb3f7c56f88 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 30 Nov 2020 19:01:43 -0800 Subject: [PATCH 028/356] cleans up ordered collection mixin --- bookwyrm/models/base_model.py | 126 ++++++++++------------------------ bookwyrm/models/fields.py | 17 +++-- bookwyrm/models/tag.py | 26 +++---- bookwyrm/models/user.py | 4 +- 4 files changed, 59 insertions(+), 114 deletions(-) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index af65a36a..0f317dbc 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -1,18 +1,16 @@ ''' base model with default fields ''' from base64 import b64encode -from dataclasses import dataclass -from typing import Callable from uuid import uuid4 -from urllib.parse import urlencode from Crypto.PublicKey import RSA from Crypto.Signature import pkcs1_15 from Crypto.Hash import SHA256 +from django.core.paginator import Paginator from django.db import models from django.dispatch import receiver from bookwyrm import activitypub -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import DOMAIN, PAGE_LENGTH from .fields import RemoteIdField @@ -52,15 +50,6 @@ def execute_after_save(sender, instance, created, *args, **kwargs): instance.save() -def get_field_name(field): - ''' model_field_name to activitypubFieldName ''' - if field.activitypub_field: - return field.activitypub_field - name = field.name.split('.')[-1] - components = name.split('_') - return components[0] + ''.join(x.title() for x in components[1:]) - - def unfurl_related_field(related_field): ''' load reverse lookups (like public key owner or Status attachment ''' if hasattr(related_field, 'all'): @@ -78,15 +67,16 @@ class ActivitypubMixin: def to_activity(self): ''' convert from a model to an activity ''' activity = {} - for field in self.__class__._meta.get_fields(): + for field in self._meta.get_fields(): if not hasattr(field, 'field_to_activity'): continue - key = get_field_name(field) value = field.field_to_activity(getattr(self, field.name)) if value is None: continue + key = field.get_activitypub_field() if key in activity and isinstance(activity[key], list): + # handles tags on status, which accumulate across fields activity[key] += value else: activity[key] = value @@ -125,15 +115,12 @@ class ActivitypubMixin: def to_delete_activity(self, user): ''' notice of deletion ''' - # this should be a tombstone - activity_object = self.to_activity() - return activitypub.Delete( id=self.remote_id + '/activity', actor=user.remote_id, to=['%s/followers' % user.remote_id], cc=['https://www.w3.org/ns/activitystreams#Public'], - object=activity_object, + object=self.to_activity(), ).serialize() @@ -165,81 +152,53 @@ class OrderedCollectionPageMixin(ActivitypubMixin): ''' this can be overriden if there's a special remote id, ie outbox ''' return self.remote_id - def page(self, min_id=None, max_id=None): - ''' helper function to create the pagination url ''' - params = {'page': 'true'} - if min_id: - params['min_id'] = min_id - if max_id: - params['max_id'] = max_id - return '?%s' % urlencode(params) - - def next_page(self, items): - ''' use the max id of the last item ''' - if not items.count(): - return '' - return self.page(max_id=items[items.count() - 1].id) - - def prev_page(self, items): - ''' use the min id of the first item ''' - if not items.count(): - return '' - return self.page(min_id=items[0].id) - - def to_ordered_collection_page(self, queryset, remote_id, \ - id_only=False, min_id=None, max_id=None): - ''' serialize and pagiante a queryset ''' - # TODO: weird place to define this - limit = 20 - # filters for use in the django queryset min/max - filters = {} - if min_id is not None: - filters['id__gt'] = min_id - if max_id is not None: - filters['id__lte'] = max_id - page_id = self.page(min_id=min_id, max_id=max_id) - - items = queryset.filter( - **filters - ).all()[:limit] - - if id_only: - page = [s.remote_id for s in items] - else: - page = [s.to_activity() for s in items] - return activitypub.OrderedCollectionPage( - id='%s%s' % (remote_id, page_id), - partOf=remote_id, - orderedItems=page, - next='%s%s' % (remote_id, self.next_page(items)), - prev='%s%s' % (remote_id, self.prev_page(items)) - ).serialize() def to_ordered_collection(self, queryset, \ remote_id=None, page=False, **kwargs): ''' an ordered collection of whatevers ''' remote_id = remote_id or self.remote_id if page: - return self.to_ordered_collection_page( + return to_ordered_collection_page( queryset, remote_id, **kwargs) - name = '' - if hasattr(self, 'name'): - name = self.name - owner = '' - if hasattr(self, 'user'): - owner = self.user.remote_id + name = self.name if hasattr(self, 'name') else None + owner = self.user.remote_id if hasattr(self, 'user') else '' - size = queryset.count() + paginated = Paginator(queryset, PAGE_LENGTH) return activitypub.OrderedCollection( id=remote_id, - totalItems=size, + totalItems=paginated.count, name=name, owner=owner, - first='%s%s' % (remote_id, self.page()), - last='%s%s' % (remote_id, self.page(min_id=0)) + first='%s?page=1' % remote_id, + last='%s?page=%d' % (remote_id, paginated.num_pages) ).serialize() +def to_ordered_collection_page(queryset, remote_id, id_only=False, page=1): + ''' serialize and pagiante a queryset ''' + paginated = Paginator(queryset, PAGE_LENGTH) + + activity_page = paginated.page(page) + if id_only: + items = [s.remote_id for s in activity_page.object_list] + else: + items = [s.to_activity() for s in activity_page.object_list] + + prev_page = next_page = None + if activity_page.has_next(): + next_page = '%s?page=%d' % (remote_id, activity_page.next_page_number()) + if activity_page.has_previous(): + prev_page = '%s?page=%d' % \ + (remote_id, activity_page.previous_page_number()) + return activitypub.OrderedCollectionPage( + id='%s?page=%s' % (remote_id, page), + partOf=remote_id, + orderedItems=items, + next=next_page, + prev=prev_page + ).serialize() + + class OrderedCollectionMixin(OrderedCollectionPageMixin): ''' extends activitypub models to work as ordered collections ''' @property @@ -252,12 +211,3 @@ class OrderedCollectionMixin(OrderedCollectionPageMixin): def to_activity(self, **kwargs): ''' an ordered collection of the specified model queryset ''' return self.to_ordered_collection(self.collection_queryset, **kwargs) - - -@dataclass(frozen=True) -class ActivityMapping: - ''' translate between an activitypub json field and a model field ''' - activity_key: str - model_key: str - activity_formatter: Callable = lambda x: x - model_formatter: Callable = lambda x: x diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 197b6e8e..ab0f5892 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -36,7 +36,7 @@ class ActivitypubFieldMixin: def field_to_activity(self, value): ''' formatter to convert a model value into activitypub ''' if hasattr(self, 'activitypub_wrapper'): - value = {self.activitypub_wrapper: value} + return {self.activitypub_wrapper: value} return value def from_activity(self, activity_data): @@ -46,6 +46,14 @@ class ActivitypubFieldMixin: value = value.get(self.activitypub_wrapper) return value + def get_activitypub_field(self): + ''' model_field_name to activitypubFieldName ''' + if self.activitypub_field: + return self.activitypub_field + name = self.name.split('.')[-1] + components = name.split('_') + return components[0] + ''.join(x.title() for x in components[1:]) + class RemoteIdField(ActivitypubFieldMixin, models.CharField): ''' a url that serves as a unique identifier ''' @@ -91,8 +99,6 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): if not value: return None return value.remote_id - def from_activity(self, activity_data): - pass# TODO class OneToOneField(ActivitypubFieldMixin, models.OneToOneField): @@ -102,9 +108,6 @@ class OneToOneField(ActivitypubFieldMixin, models.OneToOneField): return None return value.to_activity() - def from_activity(self, activity_data): - pass# TODO - class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): ''' activitypub-aware many to many field ''' @@ -123,6 +126,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): values = super().from_activity(activity_data) return values# TODO + class TagField(ManyToManyField): ''' special case of many to many that uses Tags ''' def __init__(self, *args, **kwargs): @@ -145,7 +149,6 @@ class TagField(ManyToManyField): def image_serializer(value): ''' helper for serializing images ''' - print(value) if value and hasattr(value, 'url'): url = value.url else: diff --git a/bookwyrm/models/tag.py b/bookwyrm/models/tag.py index 8b7efceb..940b4192 100644 --- a/bookwyrm/models/tag.py +++ b/bookwyrm/models/tag.py @@ -5,19 +5,15 @@ from django.db import models from bookwyrm import activitypub from bookwyrm.settings import DOMAIN -from .base_model import OrderedCollectionMixin, BookWyrmModel, ActivityMapping +from .base_model import OrderedCollectionMixin, BookWyrmModel +from . import fields class Tag(OrderedCollectionMixin, BookWyrmModel): ''' freeform tags for books ''' - name = models.CharField(max_length=100, unique=True) + name = fields.CharField(max_length=100, unique=True) identifier = models.CharField(max_length=100) - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('name', 'name'), - ] - @classmethod def book_queryset(cls, identifier): ''' county of books associated with this tag ''' @@ -44,16 +40,12 @@ class Tag(OrderedCollectionMixin, BookWyrmModel): class UserTag(BookWyrmModel): ''' an instance of a tag on a book by a user ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - book = models.ForeignKey('Edition', on_delete=models.PROTECT) - tag = models.ForeignKey('Tag', on_delete=models.PROTECT) - - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'user'), - ActivityMapping('object', 'book'), - ActivityMapping('target', 'tag'), - ] + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='actor') + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='object') + tag = fields.ForeignKey( + 'Tag', on_delete=models.PROTECT, activitypub_field='target') activity_serializer = activitypub.AddBook diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 95dd1e79..0097fcde 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -109,13 +109,13 @@ class User(OrderedCollectionPageMixin, AbstractUser): def to_following_activity(self, **kwargs): ''' activitypub following list ''' remote_id = '%s/following' % self.remote_id - return self.to_ordered_collection(self.following, \ + return self.to_ordered_collection(self.following.all(), \ remote_id=remote_id, id_only=True, **kwargs) def to_followers_activity(self, **kwargs): ''' activitypub followers list ''' remote_id = '%s/followers' % self.remote_id - return self.to_ordered_collection(self.followers, \ + return self.to_ordered_collection(self.followers.all(), \ remote_id=remote_id, id_only=True, **kwargs) def to_activity(self): From fee5846aa88293c22778ce6e2529ab4679580f40 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 30 Nov 2020 19:33:50 -0800 Subject: [PATCH 029/356] Fixes generating new key paris for user and the broadcast test --- bookwyrm/broadcast.py | 2 +- bookwyrm/incoming.py | 8 ++++---- bookwyrm/models/user.py | 5 +++-- bookwyrm/tests/models/test_user_model.py | 4 ++-- bookwyrm/tests/test_broadcast.py | 11 +++++------ 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/bookwyrm/broadcast.py b/bookwyrm/broadcast.py index a1eebaee..a98b6774 100644 --- a/bookwyrm/broadcast.py +++ b/bookwyrm/broadcast.py @@ -65,7 +65,7 @@ def sign_and_send(sender, data, destination): ''' crpyto whatever and http junk ''' now = http_date() - if not sender.private_key: + if not sender.key_pair.private_key: # this shouldn't happen. it would be bad if it happened. raise ValueError('No private key found for sender') diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 017be19d..b5b2523d 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -98,15 +98,15 @@ def has_valid_signature(request, activity): remote_user = activitypub.resolve_remote_id(models.User, key_actor) try: - signature.verify(remote_user.public_key, request) + signature.verify(remote_user.key_pair.public_key, request) except ValueError: - old_key = remote_user.public_key + old_key = remote_user.key_pair.public_key activitypub.resolve_remote_id( models.User, remote_user, refresh=True ) - if remote_user.public_key == old_key: + if remote_user.key_pair.public_key == old_key: raise # Key unchanged. - signature.verify(remote_user.public_key, request) + signature.verify(remote_user.key_pair.public_key, request) except (ValueError, requests.exceptions.HTTPError): return False return True diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 0097fcde..a200ec51 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -155,8 +155,6 @@ class User(OrderedCollectionPageMixin, AbstractUser): self.inbox = '%s/inbox' % self.remote_id self.shared_inbox = 'https://%s/inbox' % DOMAIN self.outbox = '%s/outbox' % self.remote_id - if not self.key_pair: - self.key_pair = KeyPair.objects.create() return super().save(*args, **kwargs) @@ -197,6 +195,9 @@ def execute_after_save(sender, instance, created, *args, **kwargs): if not instance.local: set_remote_server.delay(instance.id) + instance.key_pair = KeyPair.objects.create( + remote_id='%s/#main-key' % instance.remote_id) + shelves = [{ 'name': 'To Read', 'identifier': 'to-read', diff --git a/bookwyrm/tests/models/test_user_model.py b/bookwyrm/tests/models/test_user_model.py index e82f91be..c648bf6b 100644 --- a/bookwyrm/tests/models/test_user_model.py +++ b/bookwyrm/tests/models/test_user_model.py @@ -19,8 +19,8 @@ class User(TestCase): self.assertEqual(self.user.shared_inbox, 'https://%s/inbox' % DOMAIN) self.assertEqual(self.user.inbox, '%s/inbox' % expected_id) self.assertEqual(self.user.outbox, '%s/outbox' % expected_id) - self.assertIsNotNone(self.user.private_key) - self.assertIsNotNone(self.user.public_key) + self.assertIsNotNone(self.user.key_pair.private_key) + self.assertIsNotNone(self.user.key_pair.public_key) def test_user_shelves(self): diff --git a/bookwyrm/tests/test_broadcast.py b/bookwyrm/tests/test_broadcast.py index ae4e6145..b8051219 100644 --- a/bookwyrm/tests/test_broadcast.py +++ b/bookwyrm/tests/test_broadcast.py @@ -6,13 +6,12 @@ from bookwyrm import models, broadcast class Book(TestCase): def setUp(self): - with patch('bookwyrm.models.user.get_remote_reviews.delay'): - self.user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.mouse', 'mouseword') + self.user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) - local_follower = models.User.objects.create_user( - 'joe', 'joe@mouse.mouse', 'jeoword') - self.user.followers.add(local_follower) + local_follower = models.User.objects.create_user( + 'joe', 'joe@mouse.mouse', 'jeoword', local=True) + self.user.followers.add(local_follower) with patch('bookwyrm.models.user.set_remote_server.delay'): follower = models.User.objects.create_user( From 1610d81ce673f99d7e2b2690486a58a59c0a01d1 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 30 Nov 2020 19:53:42 -0800 Subject: [PATCH 030/356] fixes some of the signing test issues --- bookwyrm/incoming.py | 2 ++ bookwyrm/signatures.py | 2 +- bookwyrm/tests/test_signing.py | 19 +++++++++++-------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index b5b2523d..9a7c6e63 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -96,6 +96,8 @@ def has_valid_signature(request, activity): raise ValueError("Wrong actor created signature.") remote_user = activitypub.resolve_remote_id(models.User, key_actor) + if not remote_user: + return False try: signature.verify(remote_user.key_pair.public_key, request) diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py index dbb88d8a..a3e1fccc 100644 --- a/bookwyrm/signatures.py +++ b/bookwyrm/signatures.py @@ -31,7 +31,7 @@ def make_signature(sender, destination, date, digest): 'digest: %s' % digest, ] message_to_sign = '\n'.join(signature_headers) - signer = pkcs1_15.new(RSA.import_key(sender.private_key)) + signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key)) signed_message = signer.sign(SHA256.new(message_to_sign.encode('utf8'))) signature = { 'keyId': '%s#main-key' % sender.remote_id, diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py index 129a4333..58e8cb7f 100644 --- a/bookwyrm/tests/test_signing.py +++ b/bookwyrm/tests/test_signing.py @@ -25,20 +25,23 @@ def get_follow_data(follower, followee): ).serialize() return json.dumps(follow_activity) -Sender = namedtuple('Sender', ('remote_id', 'private_key', 'public_key')) +KeyPair = namedtuple('KeyPair', ('private_key', 'public_key')) +Sender = namedtuple('Sender', ('remote_id', 'key_pair')) class Signature(TestCase): def setUp(self): - self.mouse = User.objects.create_user('mouse', 'mouse@example.com', '') - self.rat = User.objects.create_user('rat', 'rat@example.com', '') - self.cat = User.objects.create_user('cat', 'cat@example.com', '') + self.mouse = User.objects.create_user( + 'mouse', 'mouse@example.com', '', local=True) + self.rat = User.objects.create_user( + 'rat', 'rat@example.com', '', local=True) + self.cat = User.objects.create_user( + 'cat', 'cat@example.com', '', local=True) private_key, public_key = create_key_pair() self.fake_remote = Sender( 'http://localhost/user/remote', - private_key, - public_key, + KeyPair(private_key, public_key) ) def send(self, signature, now, data, digest): @@ -89,7 +92,7 @@ class Signature(TestCase): datafile = pathlib.Path(__file__).parent.joinpath('data/ap_user.json') data = json.loads(datafile.read_bytes()) data['id'] = self.fake_remote.remote_id - data['publicKey']['publicKeyPem'] = self.fake_remote.public_key + data['publicKey']['publicKeyPem'] = self.fake_remote.key_pair.public_key del data['icon'] # Avoid having to return an avatar. responses.add( responses.GET, @@ -116,7 +119,7 @@ class Signature(TestCase): datafile = pathlib.Path(__file__).parent.joinpath('data/ap_user.json') data = json.loads(datafile.read_bytes()) data['id'] = self.fake_remote.remote_id - data['publicKey']['publicKeyPem'] = self.fake_remote.public_key + data['publicKey']['publicKeyPem'] = self.fake_remote.key_pair.public_key del data['icon'] # Avoid having to return an avatar. responses.add( responses.GET, From d7e9a00cf10a3044f07fdc38937b1cf682f71e4b Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 2 Dec 2020 15:19:29 -0800 Subject: [PATCH 031/356] Fixes password reset flow --- bookwyrm/settings.py | 7 +++++++ bookwyrm/templates/login.html | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 5adf960f..ecabb8d7 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -15,6 +15,13 @@ CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' +# email +EMAIL_HOST = env('EMAIL_HOST') +EMAIL_PORT = env('EMAIL_PORT', 587) +EMAIL_HOST_USER = env('EMAIL_HOST_USER') +EMAIL_HOST_PASSWORD = env('EMAIL_HOST_PASSWORD') +EMAIL_USE_TLS = env('EMAIL_USE_TLS', True) + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/bookwyrm/templates/login.html b/bookwyrm/templates/login.html index 21ba2884..0142d390 100644 --- a/bookwyrm/templates/login.html +++ b/bookwyrm/templates/login.html @@ -30,7 +30,7 @@
- Forgot your password? + Forgot your password?
From b5e92fe65e559b93d6e942e0bb557d9dae977aad Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 2 Dec 2020 15:21:15 -0800 Subject: [PATCH 032/356] Allow usernames with dots --- bookwyrm/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 3b3c1ab5..092bec3b 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -6,7 +6,7 @@ from django.urls import path, re_path from bookwyrm import incoming, outgoing, views, settings, wellknown from bookwyrm import view_actions as actions -username_regex = r'(?P[\w\-_]+@[\w\-\_\.]+)' +username_regex = r'(?P[\w\-_\.]+@[\w\-\_\.]+)' localname_regex = r'(?P[\w\-_]+)' user_path = r'^user/%s' % username_regex local_user_path = r'^user/%s' % localname_regex From 2cf73188f2d42360a707b0e65853033d533415d8 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 2 Dec 2020 15:24:43 -0800 Subject: [PATCH 033/356] fixes localname regex --- bookwyrm/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 092bec3b..a9792038 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -7,7 +7,7 @@ from bookwyrm import incoming, outgoing, views, settings, wellknown from bookwyrm import view_actions as actions username_regex = r'(?P[\w\-_\.]+@[\w\-\_\.]+)' -localname_regex = r'(?P[\w\-_]+)' +localname_regex = r'(?P[\w\-_\.]+)' user_path = r'^user/%s' % username_regex local_user_path = r'^user/%s' % localname_regex @@ -61,8 +61,8 @@ urlpatterns = [ # should return a ui view or activitypub json blob as requested # users re_path(r'%s/?$' % user_path, views.user_page), - re_path(r'%s/?$' % local_user_path, views.user_page), re_path(r'%s\.json$' % local_user_path, views.user_page), + re_path(r'%s/?$' % local_user_path, views.user_page), re_path(r'%s/shelves/?$' % local_user_path, views.user_shelves_page), re_path(r'%s/followers(.json)?/?$' % local_user_path, views.followers_page), re_path(r'%s/following(.json)?/?$' % local_user_path, views.following_page), From a85043b351d9b3f70c3860c4733def6ba77635f0 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 12:35:57 -0800 Subject: [PATCH 034/356] Updates to_model to use fields --- bookwyrm/activitypub/base_activity.py | 82 ++++++------------- bookwyrm/connectors/abstract_connector.py | 3 +- .../migrations/0019_auto_20201130_1939.py | 12 +++ bookwyrm/models/__init__.py | 5 ++ bookwyrm/models/base_model.py | 2 + bookwyrm/models/fields.py | 36 ++++++-- bookwyrm/models/user.py | 4 +- 7 files changed, 78 insertions(+), 66 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 9e1b5b82..786a2aab 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -1,20 +1,13 @@ ''' basics for an activitypub serializer ''' from dataclasses import dataclass, fields, MISSING from json import JSONEncoder -from uuid import uuid4 -import dateutil.parser -from dateutil.parser import ParserError -from django.core.files.base import ContentFile from django.db.models.fields.related_descriptors \ import ForwardManyToOneDescriptor, ManyToManyDescriptor, \ ReverseManyToOneDescriptor -from django.db.models.fields import DateTimeField from django.db.models.fields.files import ImageFileDescriptor -from django.db.models.query_utils import DeferredAttribute -from django.utils import timezone -from bookwyrm.connectors import ConnectorException, get_data, get_image +from bookwyrm.connectors import ConnectorException, get_data class ActivitySerializerError(ValueError): ''' routine problems serializing activitypub json ''' @@ -84,35 +77,28 @@ class ActivityObject: # check for an existing instance, if we're not updating a known obj if not instance: instance = find_existing_by_remote_id(model, self.id) + # TODO: deduplicate books by identifiers - model_fields = [m.name for m in model._meta.get_fields()] mapped_fields = {} many_to_many_fields = {} one_to_many_fields = {} image_fields = {} - for mapping in model.activity_mappings: - if mapping.model_key not in model_fields: + for field in model._meta.get_fields(): + if not hasattr(field, 'field_to_activity'): continue + activitypub_field = field.get_activitypub_field() + value = field.field_from_activity(getattr(self, activitypub_field)) + if value is None: + continue + # value is None if there's a default that isn't supplied # in the activity but is supplied in the formatter - value = None - if mapping.activity_key: - value = getattr(self, mapping.activity_key) - model_field = getattr(model, mapping.model_key) + value = getattr(self, activitypub_field) + model_field = getattr(model, field.name) - formatted_value = mapping.model_formatter(value) - if isinstance(model_field, DeferredAttribute) and \ - isinstance(model_field.field, DateTimeField): - try: - date_value = dateutil.parser.parse(formatted_value) - try: - formatted_value = timezone.make_aware(date_value) - except ValueError: - formatted_value = date_value - except (ParserError, TypeError): - formatted_value = None - elif isinstance(model_field, ForwardManyToOneDescriptor): + formatted_value = field.field_from_activity(value) + if isinstance(model_field, ForwardManyToOneDescriptor): if not formatted_value: continue # foreign key remote id reolver (work on Edition, for example) @@ -120,25 +106,30 @@ class ActivityObject: if isinstance(formatted_value, dict) and \ formatted_value.get('id'): # if the AP field is a serialized object (as in Add) - remote_id = formatted_value['id'] + # or PublicKey + related_model = field.related_model + related_activity = related_model.activity_serializer + mapped_fields[field.name] = related_activity( + **formatted_value + ).to_model(related_model) else: # if the field is just a remote_id (as in every other case) remote_id = formatted_value - reference = resolve_remote_id(fk_model, remote_id) - mapped_fields[mapping.model_key] = reference + reference = resolve_remote_id(fk_model, remote_id) + mapped_fields[field.name] = reference elif isinstance(model_field, ManyToManyDescriptor): # status mentions book/users - many_to_many_fields[mapping.model_key] = formatted_value + many_to_many_fields[field.name] = formatted_value elif isinstance(model_field, ReverseManyToOneDescriptor): # attachments on Status, for example - one_to_many_fields[mapping.model_key] = formatted_value + one_to_many_fields[field.name] = formatted_value elif isinstance(model_field, ImageFileDescriptor): # image fields need custom handling - image_fields[mapping.model_key] = formatted_value + image_fields[field.name] = formatted_value else: if formatted_value == MISSING: formatted_value = None - mapped_fields[mapping.model_key] = formatted_value + mapped_fields[field.name] = formatted_value if instance: # updating an existing model instance @@ -154,7 +145,6 @@ class ActivityObject: # add images for (model_key, value) in image_fields.items(): - formatted_value = image_formatter(value) if not formatted_value: continue getattr(instance, model_key).save(*formatted_value, save=True) @@ -243,25 +233,3 @@ def resolve_remote_id(model, remote_id, refresh=False): item = model.activity_serializer(**data) # if we're refreshing, "result" will be set and we'll update it return item.to_model(model, instance=result) - - -def image_formatter(image_slug): - ''' helper function to load images and format them for a model ''' - # when it's an inline image (User avatar/icon, Book cover), it's a json - # blob, but when it's an attached image, it's just a url - if isinstance(image_slug, dict): - url = image_slug.get('url') - elif isinstance(image_slug, str): - url = image_slug - else: - return None - if not url: - return None - - response = get_image(url) - if not response: - return None - - image_name = str(uuid4()) + '.' + url.split('.')[-1] - image_content = ContentFile(response.content) - return [image_name, image_content] diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 4e756d8c..8c5d9658 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -8,6 +8,7 @@ from django.db import transaction from dateutil import parser import requests from requests import HTTPError +from requests.exceptions import SSLError from bookwyrm import models @@ -322,7 +323,7 @@ def get_image(url): ''' wrapper for requesting an image ''' try: resp = requests.get(url) - except RequestError: + except (RequestError, SSLError): return None if not resp.ok: return None diff --git a/bookwyrm/migrations/0019_auto_20201130_1939.py b/bookwyrm/migrations/0019_auto_20201130_1939.py index c69bdbce..11cf6a3b 100644 --- a/bookwyrm/migrations/0019_auto_20201130_1939.py +++ b/bookwyrm/migrations/0019_auto_20201130_1939.py @@ -3,6 +3,17 @@ import bookwyrm.models.fields from django.db import migrations +def update_notnull(app_registry, schema_editor): + db_alias = schema_editor.connection.alias + users = app_registry.get_model('bookwyrm', 'User') + for user in users.objects.using(db_alias): + if user.name and user.summary: + continue + if not user.summary: + user.summary = '' + if not user.name: + user.name = '' + user.save() class Migration(migrations.Migration): @@ -11,6 +22,7 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RunPython(update_notnull), migrations.AlterField( model_name='user', name='name', diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py index 86bdf219..b9a2814e 100644 --- a/bookwyrm/models/__init__.py +++ b/bookwyrm/models/__init__.py @@ -25,3 +25,8 @@ from .site import SiteSettings, SiteInvite, PasswordReset cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass) activity_models = {c[1].activity_serializer.__name__: c[1] \ for c in cls_members if hasattr(c[1], 'activity_serializer')} + +def to_activity(activity_json): + ''' link up models and activities ''' + activity_type = activity_json.get('type') + return activity_models[activity_type].to_activity(activity_json) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 0f317dbc..520864c2 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -80,7 +80,9 @@ class ActivitypubMixin: activity[key] += value else: activity[key] = value + if hasattr(self, 'serialize_reverse_fields'): + # for example, editions of a work for field_name in self.serialize_reverse_fields: related_field = getattr(self, field_name) activity[field_name] = unfurl_related_field(related_field) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index ab0f5892..960f3605 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -2,11 +2,14 @@ import re from uuid import uuid4 +import dateutil.parser +from dateutil.parser import ParserError from django.contrib.auth.models import AbstractUser from django.contrib.postgres.fields import ArrayField as DjangoArrayField from django.core.exceptions import ValidationError from django.core.files.base import ContentFile from django.db import models +from django.utils import timezone from django.utils.translation import gettext_lazy as _ from bookwyrm import activitypub from bookwyrm.settings import DOMAIN @@ -39,10 +42,9 @@ class ActivitypubFieldMixin: return {self.activitypub_wrapper: value} return value - def from_activity(self, activity_data): + def field_from_activity(self, value): ''' formatter to convert activitypub into a model value ''' - value = activity_data.get(self.activitypub_field) - if self.activitypub_wrapper: + if hasattr(self, 'activitypub_wrapper'): value = value.get(self.activitypub_wrapper) return value @@ -100,6 +102,16 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): return None return value.remote_id + def field_from_activity(self, value): + if isinstance(value, dict) and value.get('id'): + # if the AP field is a serialized object (as in Add) + remote_id = value['id'] + else: + # if the field is just a remote_id (as in every other case) + remote_id = value + + return resolve_remote_id(remote_id) + class OneToOneField(ActivitypubFieldMixin, models.OneToOneField): ''' activitypub-aware foreign key field ''' @@ -120,10 +132,10 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): return '%s/followers' % value.instance.remote_id return [i.remote_id for i in value.all()] - def from_activity(self, activity_data): + def field_from_activity(self, valueactivity_data): if self.link_only: return None - values = super().from_activity(activity_data) + values = super().field_from_activity(values) return values# TODO @@ -162,8 +174,8 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): def field_to_activity(self, value): return image_serializer(value) - def from_activity(self, activity_data): - image_slug = super().from_activity(activity_data) + def field_from_activity(self, value): + image_slug = value # when it's an inline image (User avatar/icon, Book cover), it's a json # blob, but when it's an attached image, it's just a url if isinstance(image_slug, dict): @@ -191,6 +203,16 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField): return None return value.isoformat() + def field_from_activity(self, value): + try: + date_value = dateutil.parser.parse(value) + try: + return timezone.make_aware(date_value) + except ValueError: + return date_value + except (ParserError, TypeError): + return None + class ArrayField(ActivitypubFieldMixin, DjangoArrayField): ''' activitypub-aware array field ''' def field_to_activity(self, value): diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index a200ec51..ecfc3459 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -174,7 +174,8 @@ class KeyPair(ActivitypubMixin, BookWyrmModel): def save(self, *args, **kwargs): ''' create a key pair ''' - self.private_key, self.public_key = create_key_pair() + if not self.public_key: + self.private_key, self.public_key = create_key_pair() return super().save(*args, **kwargs) def to_activity(self): @@ -194,6 +195,7 @@ def execute_after_save(sender, instance, created, *args, **kwargs): if not instance.local: set_remote_server.delay(instance.id) + return instance.key_pair = KeyPair.objects.create( remote_id='%s/#main-key' % instance.remote_id) From 77a1fc26f13cc4b133784c73d9d3c42ea801c41e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 12:45:01 -0800 Subject: [PATCH 035/356] Save key pair when creating new User --- bookwyrm/models/user.py | 1 + bookwyrm/tests/test_signing.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index ecfc3459..ac729fd3 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -199,6 +199,7 @@ def execute_after_save(sender, instance, created, *args, **kwargs): instance.key_pair = KeyPair.objects.create( remote_id='%s/#main-key' % instance.remote_id) + instance.save() shelves = [{ 'name': 'To Read', diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py index 58e8cb7f..7427dbdf 100644 --- a/bookwyrm/tests/test_signing.py +++ b/bookwyrm/tests/test_signing.py @@ -138,10 +138,10 @@ class Signature(TestCase): ) # Second and subsequent fetches get a different key: - new_private_key, new_public_key = create_key_pair() + key_pair = KeyPair(*create_key_pair()) new_sender = Sender( - self.fake_remote.remote_id, new_private_key, new_public_key) - data['publicKey']['publicKeyPem'] = new_public_key + self.fake_remote.remote_id, key_pair) + data['publicKey']['publicKeyPem'] = key_pair.public_key responses.add( responses.GET, self.fake_remote.remote_id, From 6d137ccadadc3389bb70f592632c2f1853737714 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 12:50:21 -0800 Subject: [PATCH 036/356] mock celery task in test signing --- bookwyrm/tests/test_signing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py index 7427dbdf..f2997ff9 100644 --- a/bookwyrm/tests/test_signing.py +++ b/bookwyrm/tests/test_signing.py @@ -73,8 +73,9 @@ class Signature(TestCase): digest = digest or make_digest(data) signature = make_signature( signer or sender, self.rat.inbox, now, digest) - with patch('bookwyrm.incoming.handle_follow.delay') as _: - return self.send(signature, now, send_data or data, digest) + with patch('bookwyrm.incoming.handle_follow.delay'): + with patch('bookwyrm.models.user.set_remote_server.delay'): + return self.send(signature, now, send_data or data, digest) def test_correct_signature(self): response = self.send_test_request(sender=self.mouse) From e87236d78f0ef8984f85483b19f49e44b78cddd8 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 12:57:44 -0800 Subject: [PATCH 037/356] Remove unnecessary user create action --- bookwyrm/tests/activitypub/test_person.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bookwyrm/tests/activitypub/test_person.py b/bookwyrm/tests/activitypub/test_person.py index bec9e19b..3e0d74e0 100644 --- a/bookwyrm/tests/activitypub/test_person.py +++ b/bookwyrm/tests/activitypub/test_person.py @@ -2,14 +2,11 @@ import json import pathlib from django.test import TestCase -from bookwyrm import activitypub, models +from bookwyrm import activitypub class Person(TestCase): def setUp(self): - self.user = models.User.objects.create_user( - 'rat', 'rat@rat.com', 'ratword', - ) datafile = pathlib.Path(__file__).parent.joinpath( '../data/ap_user.json' ) From 3a751273012c9f77bf0614b7c18340e2f17edd3a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 13:00:04 -0800 Subject: [PATCH 038/356] Removes half-baked field serializers --- bookwyrm/models/fields.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 960f3605..2b0b3b47 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -102,16 +102,6 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): return None return value.remote_id - def field_from_activity(self, value): - if isinstance(value, dict) and value.get('id'): - # if the AP field is a serialized object (as in Add) - remote_id = value['id'] - else: - # if the field is just a remote_id (as in every other case) - remote_id = value - - return resolve_remote_id(remote_id) - class OneToOneField(ActivitypubFieldMixin, models.OneToOneField): ''' activitypub-aware foreign key field ''' @@ -132,12 +122,6 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): return '%s/followers' % value.instance.remote_id return [i.remote_id for i in value.all()] - def field_from_activity(self, valueactivity_data): - if self.link_only: - return None - values = super().field_from_activity(values) - return values# TODO - class TagField(ManyToManyField): ''' special case of many to many that uses Tags ''' From de7e64932a166a7525a15b18222646b541c72b57 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 13:14:04 -0800 Subject: [PATCH 039/356] Fixes name of book field on status --- bookwyrm/models/status.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 371a6345..6fa9f995 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -145,7 +145,8 @@ class GeneratedNote(Status): class Comment(Status): ''' like a review but without a rating and transient ''' - book = fields.ForeignKey('Edition', on_delete=models.PROTECT) + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') @property def pure_content(self): @@ -160,7 +161,8 @@ class Comment(Status): class Quotation(Status): ''' like a review but without a rating and transient ''' quote = fields.TextField() - book = fields.ForeignKey('Edition', on_delete=models.PROTECT) + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') @property def pure_content(self): @@ -179,7 +181,8 @@ class Quotation(Status): class Review(Status): ''' a book review ''' name = fields.CharField(max_length=255, null=True) - book = fields.ForeignKey('Edition', on_delete=models.PROTECT) + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') rating = fields.IntegerField( default=None, null=True, From 9c6db1cc0e27f7206a17584cd86eb28cd43e420d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 13:22:50 -0800 Subject: [PATCH 040/356] Updates connector tests --- .../connectors/test_abstract_connector.py | 4 ++-- .../connectors/test_bookwyrm_connector.py | 21 ++++++------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/bookwyrm/tests/connectors/test_abstract_connector.py b/bookwyrm/tests/connectors/test_abstract_connector.py index c01b00db..f05645ab 100644 --- a/bookwyrm/tests/connectors/test_abstract_connector.py +++ b/bookwyrm/tests/connectors/test_abstract_connector.py @@ -3,7 +3,7 @@ from django.test import TestCase from bookwyrm import models from bookwyrm.connectors.abstract_connector import Mapping -from bookwyrm.connectors.bookwyrm_connector import Connector +from bookwyrm.connectors.openlibrary import Connector class AbstractConnector(TestCase): @@ -12,7 +12,7 @@ class AbstractConnector(TestCase): models.Connector.objects.create( identifier='example.com', - connector_file='bookwyrm_connector', + connector_file='openlibrary', base_url='https://example.com', books_url='https:/example.com', covers_url='https://example.com', diff --git a/bookwyrm/tests/connectors/test_bookwyrm_connector.py b/bookwyrm/tests/connectors/test_bookwyrm_connector.py index c41b454c..8d866ca2 100644 --- a/bookwyrm/tests/connectors/test_bookwyrm_connector.py +++ b/bookwyrm/tests/connectors/test_bookwyrm_connector.py @@ -1,16 +1,17 @@ ''' testing book data connectors ''' -from dateutil import parser -from django.test import TestCase import json import pathlib +from django.test import TestCase from bookwyrm import models from bookwyrm.connectors.bookwyrm_connector import Connector -from bookwyrm.connectors.abstract_connector import SearchResult, get_date +from bookwyrm.connectors.abstract_connector import SearchResult class BookWyrmConnector(TestCase): + ''' this connector doesn't do much, just search ''' def setUp(self): + ''' create the connector ''' models.Connector.objects.create( identifier='example.com', connector_file='bookwyrm_connector', @@ -29,13 +30,9 @@ class BookWyrmConnector(TestCase): self.edition_data = json.loads(edition_file.read_bytes()) - def test_is_work_data(self): - self.assertEqual(self.connector.is_work_data(self.work_data), True) - self.assertEqual(self.connector.is_work_data(self.edition_data), False) - - def test_format_search_result(self): - datafile = pathlib.Path(__file__).parent.joinpath('../data/fr_search.json') + datafile = pathlib.Path(__file__).parent.joinpath( + '../data/fr_search.json') search_data = json.loads(datafile.read_bytes()) results = self.connector.parse_search_data(search_data) self.assertIsInstance(results, list) @@ -46,9 +43,3 @@ class BookWyrmConnector(TestCase): self.assertEqual(result.key, 'https://example.com/book/122') self.assertEqual(result.author, 'Susanna Clarke') self.assertEqual(result.year, 2017) - - - def test_get_date(self): - date = get_date(self.edition_data['published_date']) - expected = parser.parse("2020-09-15T00:00:00+00:00") - self.assertEqual(date, expected) From b1640c5dc9718d5f55c6c772f75d571fd5f7da17 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 13:35:51 -0800 Subject: [PATCH 041/356] Sets mocks up for incoming tests --- bookwyrm/tests/incoming/test_favorite.py | 19 +++++++++++-------- bookwyrm/tests/incoming/test_follow.py | 18 ++++++++++-------- bookwyrm/tests/incoming/test_follow_accept.py | 19 +++++++++++-------- bookwyrm/tests/incoming/test_update_user.py | 15 +++++++++------ 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/bookwyrm/tests/incoming/test_favorite.py b/bookwyrm/tests/incoming/test_favorite.py index c528d38c..912657da 100644 --- a/bookwyrm/tests/incoming/test_favorite.py +++ b/bookwyrm/tests/incoming/test_favorite.py @@ -1,5 +1,6 @@ import json import pathlib +from unittest.mock import patch from django.test import TestCase from bookwyrm import models, incoming @@ -7,15 +8,17 @@ from bookwyrm import models, incoming class Favorite(TestCase): def setUp(self): - self.remote_user = models.User.objects.create_user( - 'rat', 'rat@rat.com', 'ratword', - local=False, - remote_id='https://example.com/users/rat', - inbox='https://example.com/users/rat/inbox', - outbox='https://example.com/users/rat/outbox', - ) + with patch('bookwyrm.models.user.set_remote_server.delay'): + with patch('bookwyrm.models.user.get_remote_reviews.delay'): + self.remote_user = models.User.objects.create_user( + 'rat', 'rat@rat.com', 'ratword', + local=False, + remote_id='https://example.com/users/rat', + inbox='https://example.com/users/rat/inbox', + outbox='https://example.com/users/rat/outbox', + ) self.local_user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.com', 'mouseword', + 'mouse', 'mouse@mouse.com', 'mouseword', local=True, remote_id='http://local.com/user/mouse') self.status = models.Status.objects.create( diff --git a/bookwyrm/tests/incoming/test_follow.py b/bookwyrm/tests/incoming/test_follow.py index 50e47d88..8e0c7598 100644 --- a/bookwyrm/tests/incoming/test_follow.py +++ b/bookwyrm/tests/incoming/test_follow.py @@ -6,15 +6,17 @@ from bookwyrm import models, incoming class IncomingFollow(TestCase): def setUp(self): - self.remote_user = models.User.objects.create_user( - 'rat', 'rat@rat.com', 'ratword', - local=False, - remote_id='https://example.com/users/rat', - inbox='https://example.com/users/rat/inbox', - outbox='https://example.com/users/rat/outbox', - ) + with patch('bookwyrm.models.user.set_remote_server.delay'): + with patch('bookwyrm.models.user.get_remote_reviews.delay'): + self.remote_user = models.User.objects.create_user( + 'rat', 'rat@rat.com', 'ratword', + local=False, + remote_id='https://example.com/users/rat', + inbox='https://example.com/users/rat/inbox', + outbox='https://example.com/users/rat/outbox', + ) self.local_user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.com', 'mouseword') + 'mouse', 'mouse@mouse.com', 'mouseword', local=True) self.local_user.remote_id = 'http://local.com/user/mouse' self.local_user.save() diff --git a/bookwyrm/tests/incoming/test_follow_accept.py b/bookwyrm/tests/incoming/test_follow_accept.py index ba88bb40..d6e048fb 100644 --- a/bookwyrm/tests/incoming/test_follow_accept.py +++ b/bookwyrm/tests/incoming/test_follow_accept.py @@ -1,3 +1,4 @@ +from unittest.mock import patch from django.test import TestCase from bookwyrm import models, incoming @@ -5,15 +6,17 @@ from bookwyrm import models, incoming class IncomingFollowAccept(TestCase): def setUp(self): - self.remote_user = models.User.objects.create_user( - 'rat', 'rat@rat.com', 'ratword', - local=False, - remote_id='https://example.com/users/rat', - inbox='https://example.com/users/rat/inbox', - outbox='https://example.com/users/rat/outbox', - ) + with patch('bookwyrm.models.user.set_remote_server.delay'): + with patch('bookwyrm.models.user.get_remote_reviews.delay'): + self.remote_user = models.User.objects.create_user( + 'rat', 'rat@rat.com', 'ratword', + local=False, + remote_id='https://example.com/users/rat', + inbox='https://example.com/users/rat/inbox', + outbox='https://example.com/users/rat/outbox', + ) self.local_user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.com', 'mouseword') + 'mouse', 'mouse@mouse.com', 'mouseword', local=True) self.local_user.remote_id = 'http://local.com/user/mouse' self.local_user.save() diff --git a/bookwyrm/tests/incoming/test_update_user.py b/bookwyrm/tests/incoming/test_update_user.py index 7ac038eb..5e8b5f2e 100644 --- a/bookwyrm/tests/incoming/test_update_user.py +++ b/bookwyrm/tests/incoming/test_update_user.py @@ -1,6 +1,7 @@ ''' when a remote user changes their profile ''' import json import pathlib +from unittest.mock import patch from django.test import TestCase from bookwyrm import models, incoming @@ -8,12 +9,14 @@ from bookwyrm import models, incoming class UpdateUser(TestCase): def setUp(self): - self.user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.com', 'mouseword', - remote_id='https://example.com/user/mouse', - local=False, - localname='mouse' - ) + with patch('bookwyrm.models.user.set_remote_server.delay'): + with patch('bookwyrm.models.user.get_remote_reviews.delay'): + self.user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.com', 'mouseword', + remote_id='https://example.com/user/mouse', + local=False, + localname='mouse' + ) datafile = pathlib.Path(__file__).parent.joinpath( '../data/ap_user.json' From 27c45c05848755864fea0b17511e051267347b4d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 13:42:02 -0800 Subject: [PATCH 042/356] Catch json decode error in loading data --- bookwyrm/connectors/abstract_connector.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 8c5d9658..a696232d 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -315,7 +315,11 @@ def get_data(url): raise ConnectorException() if not resp.ok: resp.raise_for_status() - data = resp.json() + try: + data = resp.json() + except ValueError: + raise ConnectorException() + return data From bbbfbe721e03ef2bb6d88ffae6a2876fbf9d8599 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 16:23:13 -0800 Subject: [PATCH 043/356] Removes update user test --- bookwyrm/tests/incoming/test_follow.py | 21 ------------- bookwyrm/tests/incoming/test_update_user.py | 34 --------------------- 2 files changed, 55 deletions(-) delete mode 100644 bookwyrm/tests/incoming/test_update_user.py diff --git a/bookwyrm/tests/incoming/test_follow.py b/bookwyrm/tests/incoming/test_follow.py index 8e0c7598..799907da 100644 --- a/bookwyrm/tests/incoming/test_follow.py +++ b/bookwyrm/tests/incoming/test_follow.py @@ -75,24 +75,3 @@ class IncomingFollow(TestCase): # the follow relationship should not exist follow = models.UserFollows.objects.all() self.assertEqual(list(follow), []) - - - def test_nonexistent_user_follow(self): - activity = { - "@context": "https://www.w3.org/ns/activitystreams", - "id": "https://example.com/users/rat/follows/123", - "type": "Follow", - "actor": "https://example.com/users/rat", - "object": "http://local.com/user/nonexistent-user" - } - - with patch('bookwyrm.broadcast.broadcast_task.delay') as _: - incoming.handle_follow(activity) - - # do nothing - notifications = models.Notification.objects.all() - self.assertEqual(list(notifications), []) - requests = models.UserFollowRequest.objects.all() - self.assertEqual(list(requests), []) - follows = models.UserFollows.objects.all() - self.assertEqual(list(follows), []) diff --git a/bookwyrm/tests/incoming/test_update_user.py b/bookwyrm/tests/incoming/test_update_user.py deleted file mode 100644 index 5e8b5f2e..00000000 --- a/bookwyrm/tests/incoming/test_update_user.py +++ /dev/null @@ -1,34 +0,0 @@ -''' when a remote user changes their profile ''' -import json -import pathlib -from unittest.mock import patch -from django.test import TestCase - -from bookwyrm import models, incoming - - -class UpdateUser(TestCase): - def setUp(self): - with patch('bookwyrm.models.user.set_remote_server.delay'): - with patch('bookwyrm.models.user.get_remote_reviews.delay'): - self.user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.com', 'mouseword', - remote_id='https://example.com/user/mouse', - local=False, - localname='mouse' - ) - - datafile = pathlib.Path(__file__).parent.joinpath( - '../data/ap_user.json' - ) - self.user_data = json.loads(datafile.read_bytes()) - - def test_handle_update_user(self): - self.assertIsNone(self.user.name) - self.assertEqual(self.user.localname, 'mouse') - - incoming.handle_update_user({'object': self.user_data}) - self.user = models.User.objects.get(id=self.user.id) - - self.assertEqual(self.user.name, 'MOUSE?? MOUSE!!') - self.assertEqual(self.user.localname, 'mouse') From 646ced80ce7963565677a83b8b32ba9f772a7795 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 17:18:23 -0800 Subject: [PATCH 044/356] Test fixes --- bookwyrm/tests/models/test_base_model.py | 2 +- bookwyrm/tests/models/test_book_model.py | 4 ++-- bookwyrm/tests/models/test_import_model.py | 2 +- .../tests/models/test_relationship_models.py | 18 +++++++------- bookwyrm/tests/models/test_status_model.py | 12 +--------- bookwyrm/tests/models/test_user_model.py | 3 +-- bookwyrm/tests/outgoing/test_follow.py | 24 ++++++++++--------- 7 files changed, 29 insertions(+), 36 deletions(-) diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py index 14f8b7fb..7e383a46 100644 --- a/bookwyrm/tests/models/test_base_model.py +++ b/bookwyrm/tests/models/test_base_model.py @@ -15,7 +15,7 @@ class BaseModel(TestCase): def test_remote_id_with_user(self): user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.com', 'mouseword') + 'mouse', 'mouse@mouse.com', 'mouseword', local=True) instance = BookWyrmModel() instance.user = user instance.id = 1 diff --git a/bookwyrm/tests/models/test_book_model.py b/bookwyrm/tests/models/test_book_model.py index d01eea2e..12564992 100644 --- a/bookwyrm/tests/models/test_book_model.py +++ b/bookwyrm/tests/models/test_book_model.py @@ -24,7 +24,7 @@ class Book(TestCase): def test_remote_id(self): remote_id = 'https://%s/book/%d' % (settings.DOMAIN, self.work.id) self.assertEqual(self.work.get_remote_id(), remote_id) - self.assertEqual(self.work.remote_id, 'https://example.com/book/1') + self.assertEqual(self.work.remote_id, remote_id) def test_create_book(self): ''' you shouldn't be able to create Books (only editions and works) ''' @@ -59,7 +59,7 @@ class Book(TestCase): class Shelf(TestCase): def setUp(self): user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.mouse', 'mouseword') + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) models.Shelf.objects.create( name='Test Shelf', identifier='test-shelf', user=user) diff --git a/bookwyrm/tests/models/test_import_model.py b/bookwyrm/tests/models/test_import_model.py index b56d9a44..c703d08a 100644 --- a/bookwyrm/tests/models/test_import_model.py +++ b/bookwyrm/tests/models/test_import_model.py @@ -52,7 +52,7 @@ class ImportJob(TestCase): unknown_read_data['Date Read'] = '' user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.mouse', 'mouseword') + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) job = models.ImportJob.objects.create(user=user) models.ImportItem.objects.create( job=job, index=1, data=currently_reading_data) diff --git a/bookwyrm/tests/models/test_relationship_models.py b/bookwyrm/tests/models/test_relationship_models.py index 1e763f59..c5c619a0 100644 --- a/bookwyrm/tests/models/test_relationship_models.py +++ b/bookwyrm/tests/models/test_relationship_models.py @@ -1,4 +1,5 @@ ''' testing models ''' +from unittest.mock import patch from django.test import TestCase from bookwyrm import models @@ -6,15 +7,16 @@ from bookwyrm import models class Relationship(TestCase): def setUp(self): - self.remote_user = models.User.objects.create_user( - 'rat', 'rat@rat.com', 'ratword', - local=False, - remote_id='https://example.com/users/rat', - inbox='https://example.com/users/rat/inbox', - outbox='https://example.com/users/rat/outbox', - ) + with patch('bookwyrm.models.user.set_remote_server.delay'): + self.remote_user = models.User.objects.create_user( + 'rat', 'rat@rat.com', 'ratword', + local=False, + remote_id='https://example.com/users/rat', + inbox='https://example.com/users/rat/inbox', + outbox='https://example.com/users/rat/outbox', + ) self.local_user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.com', 'mouseword') + 'mouse', 'mouse@mouse.com', 'mouseword', local=True) self.local_user.remote_id = 'http://local.com/user/mouse' self.local_user.save() diff --git a/bookwyrm/tests/models/test_status_model.py b/bookwyrm/tests/models/test_status_model.py index 719adfe5..7c625197 100644 --- a/bookwyrm/tests/models/test_status_model.py +++ b/bookwyrm/tests/models/test_status_model.py @@ -7,7 +7,7 @@ from bookwyrm import models, settings class Status(TestCase): def setUp(self): user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.mouse', 'mouseword') + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) book = models.Edition.objects.create(title='Example Edition') models.Status.objects.create(user=user, content='Blah blah') @@ -40,13 +40,3 @@ class Status(TestCase): expected_id = 'https://%s/user/mouse/review/%d' % \ (settings.DOMAIN, review.id) self.assertEqual(review.remote_id, expected_id) - - -class Tag(TestCase): - def test_tag(self): - book = models.Edition.objects.create(title='Example Edition') - user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.mouse', 'mouseword') - tag = models.Tag.objects.create(user=user, book=book, name='t/est tag') - self.assertEqual(tag.identifier, 't%2Fest+tag') - diff --git a/bookwyrm/tests/models/test_user_model.py b/bookwyrm/tests/models/test_user_model.py index c648bf6b..dabf760c 100644 --- a/bookwyrm/tests/models/test_user_model.py +++ b/bookwyrm/tests/models/test_user_model.py @@ -8,7 +8,7 @@ from bookwyrm.settings import DOMAIN class User(TestCase): def setUp(self): self.user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.mouse', 'mouseword') + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) def test_computed_fields(self): ''' username instead of id here ''' @@ -53,7 +53,6 @@ class User(TestCase): self.assertEqual(activity['name'], self.user.name) self.assertEqual(activity['inbox'], self.user.inbox) self.assertEqual(activity['outbox'], self.user.outbox) - self.assertEqual(activity['followers'], self.user.ap_followers) self.assertEqual(activity['bookwyrmUser'], True) self.assertEqual(activity['discoverable'], True) self.assertEqual(activity['type'], 'Person') diff --git a/bookwyrm/tests/outgoing/test_follow.py b/bookwyrm/tests/outgoing/test_follow.py index 89e677ab..d27db876 100644 --- a/bookwyrm/tests/outgoing/test_follow.py +++ b/bookwyrm/tests/outgoing/test_follow.py @@ -2,17 +2,19 @@ from unittest.mock import patch from django.test import TestCase from bookwyrm import models, outgoing +from bookwyrm.settings import DOMAIN class Following(TestCase): def setUp(self): - self.remote_user = models.User.objects.create_user( - 'rat', 'rat@rat.com', 'ratword', - local=False, - remote_id='https://example.com/users/rat', - inbox='https://example.com/users/rat/inbox', - outbox='https://example.com/users/rat/outbox', - ) + with patch('bookwyrm.models.user.set_remote_server'): + self.remote_user = models.User.objects.create_user( + 'rat', 'rat@rat.com', 'ratword', + local=False, + remote_id='https://example.com/users/rat', + inbox='https://example.com/users/rat/inbox', + outbox='https://example.com/users/rat/outbox', + ) self.local_user = models.User.objects.create_user( 'mouse', 'mouse@mouse.com', 'mouseword', local=True, @@ -23,7 +25,7 @@ class Following(TestCase): def test_handle_follow(self): self.assertEqual(models.UserFollowRequest.objects.count(), 0) - with patch('bookwyrm.broadcast.broadcast_task.delay') as _: + with patch('bookwyrm.broadcast.broadcast_task.delay'): outgoing.handle_follow(self.local_user, self.remote_user) rel = models.UserFollowRequest.objects.get() @@ -36,7 +38,7 @@ class Following(TestCase): def test_handle_unfollow(self): self.remote_user.followers.add(self.local_user) self.assertEqual(self.remote_user.followers.count(), 1) - with patch('bookwyrm.broadcast.broadcast_task.delay') as _: + with patch('bookwyrm.broadcast.broadcast_task.delay'): outgoing.handle_unfollow(self.local_user, self.remote_user) self.assertEqual(self.remote_user.followers.count(), 0) @@ -49,7 +51,7 @@ class Following(TestCase): ) rel_id = rel.id - with patch('bookwyrm.broadcast.broadcast_task.delay') as _: + with patch('bookwyrm.broadcast.broadcast_task.delay'): outgoing.handle_accept(rel) # request should be deleted self.assertEqual( @@ -66,7 +68,7 @@ class Following(TestCase): ) rel_id = rel.id - with patch('bookwyrm.broadcast.broadcast_task.delay') as _: + with patch('bookwyrm.broadcast.broadcast_task.delay'): outgoing.handle_reject(rel) # request should be deleted self.assertEqual( From d92fb5333307bcc373438c842514dcf367035f24 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 3 Dec 2020 17:23:08 -0800 Subject: [PATCH 045/356] Handle absent remote ids it oughtent get to this state, but... --- bookwyrm/models/base_model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 520864c2..91724e62 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -87,6 +87,8 @@ class ActivitypubMixin: related_field = getattr(self, field_name) activity[field_name] = unfurl_related_field(related_field) + if not activity.get('id'): + activity['id'] = self.get_remote_id() return self.activity_serializer(**activity).serialize() From ae8d0e197443a78a30e67d86370f08dcf937de8e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 08:07:47 -0800 Subject: [PATCH 046/356] Adds sort order for outbox --- bookwyrm/models/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index ac729fd3..6395987b 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -102,7 +102,7 @@ class User(OrderedCollectionPageMixin, AbstractUser): queryset = Status.objects.filter( user=self, deleted=False, - ).select_subclasses() + ).select_subclasses().order_by('-published_date') return self.to_ordered_collection(queryset, \ remote_id=self.outbox, **kwargs) From de6147ecfa6b5da176bb6eeb20aa6f808583b4ad Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 08:29:54 -0800 Subject: [PATCH 047/356] Ignore many to many activitypub serialization of non-lists --- bookwyrm/activitypub/base_activity.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 786a2aab..ebc026b9 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -151,8 +151,9 @@ class ActivityObject: # add many to many fields for (model_key, values) in many_to_many_fields.items(): - # mention books, mention users - if values == MISSING: + # mention books, mention users, followers + if values == MISSING or not isinstance(values, list): + # user followers is a link to an orderedcollection, skip it continue model_field = getattr(instance, model_key) model = model_field.model From 1ae3830ae48c135b80155b409dc71293be1a2df9 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 08:42:34 -0800 Subject: [PATCH 048/356] Removes test state of signatures --- bookwyrm/signatures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py index a3e1fccc..ff281664 100644 --- a/bookwyrm/signatures.py +++ b/bookwyrm/signatures.py @@ -89,7 +89,7 @@ class Signature: def verify(self, public_key, request): ''' verify rsa signature ''' - if False:#http_date_age(request.headers['date']) > MAX_SIGNATURE_AGE: + if http_date_age(request.headers['date']) > MAX_SIGNATURE_AGE: raise ValueError( "Request too old: %s" % (request.headers['date'],)) public_key = RSA.import_key(public_key) From 9989641f4c6f947b7aca8e5642f72ba1d19af5df Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 09:46:40 -0800 Subject: [PATCH 049/356] fixes bug in update user public key --- bookwyrm/incoming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 9a7c6e63..29c0be98 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -104,7 +104,7 @@ def has_valid_signature(request, activity): except ValueError: old_key = remote_user.key_pair.public_key activitypub.resolve_remote_id( - models.User, remote_user, refresh=True + models.User, remote_user.remote_id, refresh=True ) if remote_user.key_pair.public_key == old_key: raise # Key unchanged. From 2e4aff90a36a021fa47b7d87ad7ecaeca3b4093e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 11:22:08 -0800 Subject: [PATCH 050/356] Fixes signing Create activities and some tests for the base_model --- bookwyrm/models/base_model.py | 2 +- bookwyrm/models/status.py | 5 --- bookwyrm/tests/models/test_base_model.py | 54 ++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 91724e62..121b5f75 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -96,7 +96,7 @@ class ActivitypubMixin: ''' returns the object wrapped in a Create activity ''' activity_object = self.to_activity() - signer = pkcs1_15.new(RSA.import_key(user.private_key)) + signer = pkcs1_15.new(RSA.import_key(user.key_pair.private_key)) content = activity_object['content'] signed_message = signer.sign(SHA256.new(content.encode('utf8'))) create_id = self.remote_id + '/activity' diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 6fa9f995..a2b873bb 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -44,11 +44,6 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): ) objects = InheritanceManager() - @property - def ap_replies(self): - ''' structured replies block ''' - return self.to_replies() - activity_serializer = activitypub.Note serialize_reverse_fields = ['attachments'] diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py index 7e383a46..9568ae6a 100644 --- a/bookwyrm/tests/models/test_base_model.py +++ b/bookwyrm/tests/models/test_base_model.py @@ -1,14 +1,15 @@ ''' testing models ''' +from collections import namedtuple from django.test import TestCase from bookwyrm import models -from bookwyrm.models.base_model import BookWyrmModel +from bookwyrm.models import base_model +from bookwyrm.models.base_model import ActivitypubMixin from bookwyrm.settings import DOMAIN - class BaseModel(TestCase): def test_remote_id(self): - instance = BookWyrmModel() + instance = base_model.BookWyrmModel() instance.id = 1 expected = instance.get_remote_id() self.assertEqual(expected, 'https://%s/bookwyrmmodel/1' % DOMAIN) @@ -16,10 +17,55 @@ class BaseModel(TestCase): def test_remote_id_with_user(self): user = models.User.objects.create_user( 'mouse', 'mouse@mouse.com', 'mouseword', local=True) - instance = BookWyrmModel() + instance = base_model.BookWyrmModel() instance.user = user instance.id = 1 expected = instance.get_remote_id() self.assertEqual( expected, 'https://%s/user/mouse/bookwyrmmodel/1' % DOMAIN) + + def test_execute_after_save(self): + ''' this function sets remote ids after creation ''' + # using Work because it BookWrymModel is abstract and this requires save + # Work is a relatively not-fancy model. + instance = models.Work.objects.create(title='work title') + instance.remote_id = None + base_model.execute_after_save(None, instance, True) + self.assertEqual( + instance.remote_id, + 'https://%s/book/%d' % (DOMAIN, instance.id) + ) + + # shouldn't set remote_id if it's not created + instance.remote_id = None + base_model.execute_after_save(None, instance, False) + self.assertIsNone(instance.remote_id) + + def test_to_create_activity(self): + user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.com', 'mouseword', local=True) + + object_activity = { + 'to': 'to field', 'cc': 'cc field', + 'content': 'hi', + 'published': '2020-12-04T17:52:22.623807+00:00', + } + MockSelf = namedtuple('Self', ('remote_id', 'to_activity')) + mock_self = MockSelf( + 'https://example.com/status/1', + lambda *args: object_activity + ) + activity = ActivitypubMixin.to_create_activity(mock_self, user) + self.assertEqual( + activity['id'], + 'https://example.com/status/1/activity' + ) + self.assertEqual(activity['actor'], user.remote_id) + self.assertEqual(activity['to'], 'to field') + self.assertEqual(activity['cc'], 'cc field') + self.assertEqual(activity['object'], object_activity) + self.assertEqual( + activity['signature'].creator, + '%s#main-key' % user.remote_id + ) From 39307ce1cdeda140637f9c4b69dfa059053043cf Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 11:46:16 -0800 Subject: [PATCH 051/356] Fixes remote_id on Update activities --- bookwyrm/models/base_model.py | 2 +- bookwyrm/tests/models/test_base_model.py | 49 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 121b5f75..94fbc015 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -130,7 +130,7 @@ class ActivitypubMixin: def to_update_activity(self, user): ''' wrapper for Updates to an activity ''' - activity_id = '%s#update/%s' % (user.remote_id, uuid4()) + activity_id = '%s#update/%s' % (self.remote_id, uuid4()) return activitypub.Update( id=activity_id, actor=user.remote_id, diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py index 9568ae6a..40306d91 100644 --- a/bookwyrm/tests/models/test_base_model.py +++ b/bookwyrm/tests/models/test_base_model.py @@ -1,5 +1,6 @@ ''' testing models ''' from collections import namedtuple +import re from django.test import TestCase from bookwyrm import models @@ -62,6 +63,7 @@ class BaseModel(TestCase): 'https://example.com/status/1/activity' ) self.assertEqual(activity['actor'], user.remote_id) + self.assertEqual(activity['type'], 'Create') self.assertEqual(activity['to'], 'to field') self.assertEqual(activity['cc'], 'cc field') self.assertEqual(activity['object'], object_activity) @@ -69,3 +71,50 @@ class BaseModel(TestCase): activity['signature'].creator, '%s#main-key' % user.remote_id ) + + def test_to_delete_activity(self): + user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.com', 'mouseword', local=True) + + MockSelf = namedtuple('Self', ('remote_id', 'to_activity')) + mock_self = MockSelf( + 'https://example.com/status/1', + lambda *args: {} + ) + activity = ActivitypubMixin.to_delete_activity(mock_self, user) + self.assertEqual( + activity['id'], + 'https://example.com/status/1/activity' + ) + self.assertEqual(activity['actor'], user.remote_id) + self.assertEqual(activity['type'], 'Delete') + self.assertEqual( + activity['to'], + ['%s/followers' % user.remote_id]) + self.assertEqual( + activity['cc'], + ['https://www.w3.org/ns/activitystreams#Public']) + + def test_to_update_activity(self): + user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.com', 'mouseword', local=True) + + MockSelf = namedtuple('Self', ('remote_id', 'to_activity')) + mock_self = MockSelf( + 'https://example.com/status/1', + lambda *args: {} + ) + activity = ActivitypubMixin.to_update_activity(mock_self, user) + print(activity['id']) + self.assertIsNotNone( + re.match( + r'^https:\/\/example\.com\/status\/1#update\/.*', + activity['id'] + ) + ) + self.assertEqual(activity['actor'], user.remote_id) + self.assertEqual(activity['type'], 'Update') + self.assertEqual( + activity['to'], + ['https://www.w3.org/ns/activitystreams#Public']) + self.assertEqual(activity['object'], {}) From 800ddf2a6b9144508a69807be32790fe7abc9e9d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 11:52:01 -0800 Subject: [PATCH 052/356] fixes inconsistency in to_undo activity helper --- bookwyrm/models/base_model.py | 4 ++-- bookwyrm/tests/models/test_base_model.py | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 94fbc015..d3c9471f 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -142,10 +142,10 @@ class ActivitypubMixin: def to_undo_activity(self, user): ''' undo an action ''' return activitypub.Undo( - id='%s#undo' % user.remote_id, + id='%s#undo' % self.remote_id, actor=user.remote_id, object=self.to_activity() - ) + ).serialize() class OrderedCollectionPageMixin(ActivitypubMixin): diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py index 40306d91..b187284a 100644 --- a/bookwyrm/tests/models/test_base_model.py +++ b/bookwyrm/tests/models/test_base_model.py @@ -105,7 +105,6 @@ class BaseModel(TestCase): lambda *args: {} ) activity = ActivitypubMixin.to_update_activity(mock_self, user) - print(activity['id']) self.assertIsNotNone( re.match( r'^https:\/\/example\.com\/status\/1#update\/.*', @@ -118,3 +117,21 @@ class BaseModel(TestCase): activity['to'], ['https://www.w3.org/ns/activitystreams#Public']) self.assertEqual(activity['object'], {}) + + def test_to_undo_activity(self): + user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.com', 'mouseword', local=True) + + MockSelf = namedtuple('Self', ('remote_id', 'to_activity')) + mock_self = MockSelf( + 'https://example.com/status/1', + lambda *args: {} + ) + activity = ActivitypubMixin.to_undo_activity(mock_self, user) + self.assertEqual( + activity['id'], + 'https://example.com/status/1#undo' + ) + self.assertEqual(activity['actor'], user.remote_id) + self.assertEqual(activity['type'], 'Undo') + self.assertEqual(activity['object'], {}) From 9c9da35d9a20baf7b3f9db50547a0aa3121c8e99 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 15:14:26 -0800 Subject: [PATCH 053/356] Tests base_model to_activity --- bookwyrm/tests/models/test_base_model.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py index b187284a..a807c47f 100644 --- a/bookwyrm/tests/models/test_base_model.py +++ b/bookwyrm/tests/models/test_base_model.py @@ -1,8 +1,10 @@ ''' testing models ''' from collections import namedtuple +from dataclasses import dataclass import re from django.test import TestCase +from bookwyrm.activitypub.base_activity import ActivityObject from bookwyrm import models from bookwyrm.models import base_model from bookwyrm.models.base_model import ActivitypubMixin @@ -135,3 +137,21 @@ class BaseModel(TestCase): self.assertEqual(activity['actor'], user.remote_id) self.assertEqual(activity['type'], 'Undo') self.assertEqual(activity['object'], {}) + + + def test_to_activity(self): + @dataclass(init=False) + class TestActivity(ActivityObject): + type: str = 'Test' + + class TestModel(ActivitypubMixin, base_model.BookWyrmModel): + pass + + instance = TestModel() + instance.remote_id = 'https://www.example.com/test' + instance.activity_serializer = TestActivity + + activity = instance.to_activity() + self.assertIsInstance(activity, dict) + self.assertEqual(activity['id'], 'https://www.example.com/test') + self.assertEqual(activity['type'], 'Test') From 142a39cf5586807a6794e02e85b3325cad69a59b Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 16:26:07 -0800 Subject: [PATCH 054/356] Updates remote user when refreshing key --- bookwyrm/activitypub/base_activity.py | 11 ++++++----- bookwyrm/incoming.py | 2 +- bookwyrm/tests/test_signing.py | 19 ++++++------------- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index ebc026b9..0a34d067 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -106,7 +106,7 @@ class ActivityObject: if isinstance(formatted_value, dict) and \ formatted_value.get('id'): # if the AP field is a serialized object (as in Add) - # or PublicKey + # or KeyPair (even though it's OneToOne) related_model = field.related_model related_activity = related_model.activity_serializer mapped_fields[field.name] = related_activity( @@ -131,6 +131,7 @@ class ActivityObject: formatted_value = None mapped_fields[field.name] = formatted_value + if instance: # updating an existing model instance for k, v in mapped_fields.items(): @@ -178,13 +179,13 @@ class ActivityObject: if values == MISSING: continue model_field = getattr(instance, model_key) - model = model_field.model + related_model = model_field.model for item in values: if isinstance(item, str): - item = resolve_remote_id(model, item) + item = resolve_remote_id(related_model, item) else: - item = model.activity_serializer(**item) - item = item.to_model(model) + item = related_model.activity_serializer(**item) + item = item.to_model(related_model) field_name = instance.__class__.__name__.lower() setattr(item, field_name, instance) item.save() diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 29c0be98..bbbebf0f 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -103,7 +103,7 @@ def has_valid_signature(request, activity): signature.verify(remote_user.key_pair.public_key, request) except ValueError: old_key = remote_user.key_pair.public_key - activitypub.resolve_remote_id( + remote_user = activitypub.resolve_remote_id( models.User, remote_user.remote_id, refresh=True ) if remote_user.key_pair.public_key == old_key: diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py index f2997ff9..bf252764 100644 --- a/bookwyrm/tests/test_signing.py +++ b/bookwyrm/tests/test_signing.py @@ -111,7 +111,7 @@ class Signature(TestCase): status=200 ) - with patch('bookwyrm.models.user.get_remote_reviews.delay') as _: + with patch('bookwyrm.models.user.get_remote_reviews.delay'): response = self.send_test_request(sender=self.fake_remote) self.assertEqual(response.status_code, 200) @@ -131,17 +131,10 @@ class Signature(TestCase): responses.GET, 'https://localhost/.well-known/nodeinfo', status=404) - responses.add( - responses.GET, - 'https://example.com/user/mouse/outbox?page=true', - json={'orderedItems': []}, - status=200 - ) # Second and subsequent fetches get a different key: key_pair = KeyPair(*create_key_pair()) - new_sender = Sender( - self.fake_remote.remote_id, key_pair) + new_sender = Sender(self.fake_remote.remote_id, key_pair) data['publicKey']['publicKeyPem'] = key_pair.public_key responses.add( responses.GET, @@ -149,7 +142,7 @@ class Signature(TestCase): json=data, status=200) - with patch('bookwyrm.models.user.get_remote_reviews.delay') as _: + with patch('bookwyrm.models.user.get_remote_reviews.delay'): # Key correct: response = self.send_test_request(sender=self.fake_remote) self.assertEqual(response.status_code, 200) @@ -181,7 +174,7 @@ class Signature(TestCase): @pytest.mark.integration def test_changed_data(self): '''Message data must match the digest header.''' - with patch('bookwyrm.activitypub.resolve_remote_id') as _: + with patch('bookwyrm.activitypub.resolve_remote_id'): response = self.send_test_request( self.mouse, send_data=get_follow_data(self.mouse, self.cat)) @@ -189,7 +182,7 @@ class Signature(TestCase): @pytest.mark.integration def test_invalid_digest(self): - with patch('bookwyrm.activitypub.resolve_remote_id') as _: + with patch('bookwyrm.activitypub.resolve_remote_id'): response = self.send_test_request( self.mouse, digest='SHA-256=AAAAAAAAAAAAAAAAAA') @@ -198,7 +191,7 @@ class Signature(TestCase): @pytest.mark.integration def test_old_message(self): '''Old messages should be rejected to prevent replay attacks.''' - with patch('bookwyrm.activitypub.resolve_remote_id') as _: + with patch('bookwyrm.activitypub.resolve_remote_id'): response = self.send_test_request( self.mouse, date=http_date(time.time() - 301) From aa6e312cfb03d4ef0685dc31a81df5640c93aa36 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 17:38:39 -0800 Subject: [PATCH 055/356] Starts adding tests for custom model fields --- bookwyrm/tests/models/test_fields.py | 77 ++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 bookwyrm/tests/models/test_fields.py diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py new file mode 100644 index 00000000..1e98c44d --- /dev/null +++ b/bookwyrm/tests/models/test_fields.py @@ -0,0 +1,77 @@ +''' testing models ''' +from django.core.exceptions import ValidationError +from django.db import models +from django.test import TestCase + +from bookwyrm.models import fields + +class ActivitypubFields(TestCase): + def test_validate_remote_id(self): + self.assertIsNone(fields.validate_remote_id( + 'http://www.example.com' + )) + self.assertIsNone(fields.validate_remote_id( + 'https://www.example.com' + )) + self.assertIsNone(fields.validate_remote_id( + 'http://example.com/dlfjg-23/x' + )) + self.assertRaises( + ValidationError, fields.validate_remote_id, + 'http:/example.com/dlfjg-23/x' + ) + self.assertRaises( + ValidationError, fields.validate_remote_id, + 'www.example.com/dlfjg-23/x' + ) + self.assertRaises( + ValidationError, fields.validate_remote_id, + 'http://www.example.com/dlfjg 23/x' + ) + + def test_activitypub_field_mixin(self): + instance = fields.ActivitypubFieldMixin() + self.assertEqual(instance.field_to_activity('fish'), 'fish') + self.assertEqual(instance.field_from_activity('fish'), 'fish') + + instance = fields.ActivitypubFieldMixin( + activitypub_wrapper='endpoints', activitypub_field='outbox' + ) + self.assertEqual( + instance.field_to_activity('fish'), + {'outbox': 'fish'} + ) + self.assertEqual( + instance.field_from_activity({'outbox': 'fish'}), + 'fish' + ) + self.assertEqual(instance.get_activitypub_field(), 'endpoints') + + instance = fields.ActivitypubFieldMixin() + instance.name = 'snake_case_name' + self.assertEqual(instance.get_activitypub_field(), 'snakeCaseName') + + def test_remote_id_field(self): + instance = fields.RemoteIdField() + self.assertEqual(instance.max_length, 255) + + with self.assertRaises(ValidationError): + instance.run_validators('http://www.example.com/dlfjg 23/x') + + def test_username_field(self): + instance = fields.UsernameField() + self.assertEqual(instance.activitypub_field, 'preferredUsername') + self.assertEqual(instance.max_length, 150) + self.assertEqual(instance.unique, True) + with self.assertRaises(ValidationError): + instance.run_validators('one two') + instance.run_validators('a*&') + instance.run_validators('trailingwhite ') + self.assertIsNone(instance.run_validators('aksdhf')) + + self.assertEqual(instance.field_to_activity('test@example.com'), 'test') + + def test_foreign_key(self): + instance = fields.ForeignKey('User', on_delete=models.CASCADE) + item = namedtuple('Serializable', ('to_activity'))(lambda: {'a': 'b'}) + self.assertEqual(instance.field_to_activity(item), {'a': 'b'}) From f116ce378d990849440948c148cc8874dec544b1 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 17:42:01 -0800 Subject: [PATCH 056/356] Fixes foreign key test --- bookwyrm/tests/models/test_fields.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index 1e98c44d..5e29a527 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -1,4 +1,6 @@ ''' testing models ''' +from collections import namedtuple + from django.core.exceptions import ValidationError from django.db import models from django.test import TestCase @@ -73,5 +75,6 @@ class ActivitypubFields(TestCase): def test_foreign_key(self): instance = fields.ForeignKey('User', on_delete=models.CASCADE) - item = namedtuple('Serializable', ('to_activity'))(lambda: {'a': 'b'}) - self.assertEqual(instance.field_to_activity(item), {'a': 'b'}) + Serializable = namedtuple('Serializable', ('to_activity', 'remote_id')) + item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c') + self.assertEqual(instance.field_to_activity(item), 'https://e.b/c') From 8a900689d3288bb1bbdf27bb8de21b8e6ad1f37c Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Dec 2020 17:57:14 -0800 Subject: [PATCH 057/356] Generalizes link format in many to many field --- bookwyrm/models/fields.py | 2 +- bookwyrm/tests/models/test_fields.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 2b0b3b47..b5e321ee 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -119,7 +119,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): def field_to_activity(self, value): if self.link_only: - return '%s/followers' % value.instance.remote_id + return '%s/%s' % (value.instance.remote_id, self.name) return [i.remote_id for i in value.all()] diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index 5e29a527..e7393096 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -78,3 +78,28 @@ class ActivitypubFields(TestCase): Serializable = namedtuple('Serializable', ('to_activity', 'remote_id')) item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c') self.assertEqual(instance.field_to_activity(item), 'https://e.b/c') + + def test_one_to_one_field(self): + instance = fields.OneToOneField('User', on_delete=models.CASCADE) + Serializable = namedtuple('Serializable', ('to_activity', 'remote_id')) + item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c') + self.assertEqual(instance.field_to_activity(item), {'a': 'b'}) + + def test_many_to_many_field(self): + instance = fields.ManyToManyField('User') + + Serializable = namedtuple('Serializable', ('to_activity', 'remote_id')) + Queryset = namedtuple('Queryset', ('all', 'instance')) + item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c') + another_item = Serializable(lambda: {}, 'example.com') + + items = Queryset(lambda: [item], another_item) + + self.assertEqual(instance.field_to_activity(items), ['https://e.b/c']) + + instance = fields.ManyToManyField('User', link_only=True) + instance.name = 'snake_case' + self.assertEqual( + instance.field_to_activity(items), + 'example.com/snake_case' + ) From 05cde33a0c3a962f04ad7d201b73a3d68b6b06d1 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 5 Dec 2020 14:40:23 -0800 Subject: [PATCH 058/356] Adds tests for remaining nontrivial model fields --- bookwyrm/tests/models/test_fields.py | 86 +++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index e7393096..2b2d3939 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -1,11 +1,20 @@ ''' testing models ''' +from io import BytesIO from collections import namedtuple +import pathlib +import re + +from PIL import Image +import responses from django.core.exceptions import ValidationError +from django.core.files.base import ContentFile from django.db import models from django.test import TestCase +from django.utils import timezone -from bookwyrm.models import fields +from bookwyrm.models import fields, User +from bookwyrm.settings import DOMAIN class ActivitypubFields(TestCase): def test_validate_remote_id(self): @@ -103,3 +112,78 @@ class ActivitypubFields(TestCase): instance.field_to_activity(items), 'example.com/snake_case' ) + + def test_tag_field(self): + instance = fields.TagField('User') + + Serializable = namedtuple( + 'Serializable', + ('to_activity', 'remote_id', 'name_field', 'name') + ) + Queryset = namedtuple('Queryset', ('all', 'instance')) + item = Serializable( + lambda: {'a': 'b'}, 'https://e.b/c', 'name', 'Name') + another_item = Serializable( + lambda: {}, 'example.com', '', '') + items = Queryset(lambda: [item], another_item) + + result = instance.field_to_activity(items) + self.assertIsInstance(result, list) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].href, 'https://e.b/c') + self.assertEqual(result[0].name, 'Name') + self.assertEqual(result[0].type, 'Serializable') + + + @responses.activate + def test_image_field(self): + user = User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) + image_file = pathlib.Path(__file__).parent.joinpath( + '../../static/images/default_avi.jpg') + image = Image.open(image_file) + output = BytesIO() + image.save(output, format=image.format) + user.avatar.save( + 'test.jpg', + ContentFile(output.getvalue()) + ) + + output = fields.image_serializer(user.avatar) + self.assertIsNotNone( + re.match( + r'https://%s/images/avatars/test_[A-z0-9]+\.jpg' % DOMAIN, + output.url, + ) + ) + self.assertEqual(output.type, 'Image') + + instance = fields.ImageField() + + self.assertEqual(instance.field_to_activity(user.avatar), output) + + responses.add( + responses.GET, + 'http://www.example.com/image.jpg', + body=user.avatar.file.read(), + status=200) + loaded_image = instance.field_from_activity( + 'http://www.example.com/image.jpg') + self.assertIsInstance(loaded_image, list) + self.assertIsNotNone(re.match(r'.*\.jpg', loaded_image[0])) + self.assertIsInstance(loaded_image[1], ContentFile) + + + def test_datetime_field(self): + instance = fields.DateTimeField() + now = timezone.now() + self.assertEqual(instance.field_to_activity(now), now.isoformat()) + self.assertEqual( + instance.field_from_activity(now.isoformat()), now + ) + self.assertEqual(instance.field_from_activity('bip'), None) + + + def test_array_field(self): + instance = fields.ArrayField(fields.IntegerField) + self.assertEqual(instance.field_to_activity([0, 1]), ['0', '1']) From ef1558628f7041be2c6f293bba787a898f6176df Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 5 Dec 2020 14:48:47 -0800 Subject: [PATCH 059/356] Fixes transient failure in image field test --- bookwyrm/tests/models/test_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index 2b2d3939..a85f3234 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -152,7 +152,7 @@ class ActivitypubFields(TestCase): output = fields.image_serializer(user.avatar) self.assertIsNotNone( re.match( - r'https://%s/images/avatars/test_[A-z0-9]+\.jpg' % DOMAIN, + r'https://%s/images/avatars/test_.*\.jpg' % DOMAIN, output.url, ) ) From 8500a7cfe12b432dd97d4ae7df2e153089d2c3a1 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 5 Dec 2020 15:28:41 -0800 Subject: [PATCH 060/356] Unit test fails in CI but not local --- bookwyrm/tests/models/test_fields.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index a85f3234..b157aab3 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -152,7 +152,7 @@ class ActivitypubFields(TestCase): output = fields.image_serializer(user.avatar) self.assertIsNotNone( re.match( - r'https://%s/images/avatars/test_.*\.jpg' % DOMAIN, + r'.*\.jpg', output.url, ) ) @@ -170,7 +170,6 @@ class ActivitypubFields(TestCase): loaded_image = instance.field_from_activity( 'http://www.example.com/image.jpg') self.assertIsInstance(loaded_image, list) - self.assertIsNotNone(re.match(r'.*\.jpg', loaded_image[0])) self.assertIsInstance(loaded_image[1], ContentFile) From 7a90aa8f6c95e14d91e685d9567ddd581f381ac0 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 5 Dec 2020 21:33:48 -0800 Subject: [PATCH 061/356] Start moving serializing from to_model to fields --- bookwyrm/activitypub/base_activity.py | 63 +++++---------------------- bookwyrm/models/fields.py | 43 +++++++++++++++++- bookwyrm/tests/models/test_fields.py | 17 +++++++- 3 files changed, 69 insertions(+), 54 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 0a34d067..4de4dc6b 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -92,44 +92,24 @@ class ActivityObject: if value is None: continue - # value is None if there's a default that isn't supplied - # in the activity but is supplied in the formatter - value = getattr(self, activitypub_field) model_field = getattr(model, field.name) - formatted_value = field.field_from_activity(value) if isinstance(model_field, ForwardManyToOneDescriptor): - if not formatted_value: - continue - # foreign key remote id reolver (work on Edition, for example) - fk_model = model_field.field.related_model - if isinstance(formatted_value, dict) and \ - formatted_value.get('id'): - # if the AP field is a serialized object (as in Add) - # or KeyPair (even though it's OneToOne) - related_model = field.related_model - related_activity = related_model.activity_serializer - mapped_fields[field.name] = related_activity( - **formatted_value - ).to_model(related_model) - else: - # if the field is just a remote_id (as in every other case) - remote_id = formatted_value - reference = resolve_remote_id(fk_model, remote_id) - mapped_fields[field.name] = reference - elif isinstance(model_field, ManyToManyDescriptor): + mapped_fields[field.name] = value + if isinstance(model_field, ManyToManyDescriptor): # status mentions book/users - many_to_many_fields[field.name] = formatted_value + many_to_many_fields[field.name] = value elif isinstance(model_field, ReverseManyToOneDescriptor): # attachments on Status, for example - one_to_many_fields[field.name] = formatted_value + one_to_many_fields[field.name] = value elif isinstance(model_field, ImageFileDescriptor): # image fields need custom handling - image_fields[field.name] = formatted_value + print(model_field, field.name, value) + image_fields[field.name] = value else: - if formatted_value == MISSING: - formatted_value = None - mapped_fields[field.name] = formatted_value + if value == MISSING: + value = None + mapped_fields[field.name] = value if instance: @@ -146,33 +126,14 @@ class ActivityObject: # add images for (model_key, value) in image_fields.items(): - if not formatted_value: + if not value: continue - getattr(instance, model_key).save(*formatted_value, save=True) + getattr(instance, model_key).save(*value, save=True) # add many to many fields for (model_key, values) in many_to_many_fields.items(): # mention books, mention users, followers - if values == MISSING or not isinstance(values, list): - # user followers is a link to an orderedcollection, skip it - continue - model_field = getattr(instance, model_key) - model = model_field.model - items = [] - for link in values: - if isinstance(link, dict): - # check that the Type matches the model (Status - # tags contain both user mentions and book tags) - if not model.activity_serializer.type == \ - link.get('type'): - continue - remote_id = link.get('href') - else: - remote_id = link - items.append( - resolve_remote_id(model, remote_id) - ) - getattr(instance, model_key).set(items) + getattr(instance, model_key).set(values) # add one to many fields for (model_key, values) in one_to_many_fields.items(): diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index b5e321ee..e1636f84 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -57,6 +57,27 @@ class ActivitypubFieldMixin: return components[0] + ''.join(x.title() for x in components[1:]) +class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin): + ''' default (de)serialization for foreign key and one to one ''' + def field_from_activity(self, value): + if not value: + return None + + related_model = self.related_model + if isinstance(value, dict) and value.get('id'): + # this is an activitypub object, which we can deserialize + activity_serializer = related_model.activity_serializer + return activity_serializer(**value).to_model(related_model) + try: + # make sure the value looks like a remote id + validate_remote_id(value) + except ValidationError: + # we don't know what this is, ignore it + return None + # gets or creates the model field from the remote id + return activitypub.resolve_remote_id(related_model, value) + + class RemoteIdField(ActivitypubFieldMixin, models.CharField): ''' a url that serves as a unique identifier ''' def __init__(self, *args, max_length=255, validators=None, **kwargs): @@ -95,7 +116,7 @@ class UsernameField(ActivitypubFieldMixin, models.CharField): return value.split('@')[0] -class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): +class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey): ''' activitypub-aware foreign key field ''' def field_to_activity(self, value): if not value: @@ -103,7 +124,7 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey): return value.remote_id -class OneToOneField(ActivitypubFieldMixin, models.OneToOneField): +class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField): ''' activitypub-aware foreign key field ''' def field_to_activity(self, value): if not value: @@ -122,6 +143,15 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): return '%s/%s' % (value.instance.remote_id, self.name) return [i.remote_id for i in value.all()] + def field_from_activity(self, value): + items = [] + for remote_id in value: + validate_remote_id(remote_id) + items.append( + activitypub.resolve_remote_id(self.related_model, remote_id) + ) + return items + class TagField(ManyToManyField): ''' special case of many to many that uses Tags ''' @@ -142,6 +172,15 @@ class TagField(ManyToManyField): )) return tags + def field_from_activity(self, value): + items = [] + for link_json in value: + link = activitypub.Link(**link_json) + items.append( + activitypub.resolve_remote_id(self.related_model, link.href) + ) + return items + def image_serializer(value): ''' helper for serializing images ''' diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index b157aab3..06ff01e0 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -14,7 +14,7 @@ from django.test import TestCase from django.utils import timezone from bookwyrm.models import fields, User -from bookwyrm.settings import DOMAIN +from bookwyrm import activitypub class ActivitypubFields(TestCase): def test_validate_remote_id(self): @@ -86,8 +86,23 @@ class ActivitypubFields(TestCase): instance = fields.ForeignKey('User', on_delete=models.CASCADE) Serializable = namedtuple('Serializable', ('to_activity', 'remote_id')) item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c') + # returns the remote_id field of the related object self.assertEqual(instance.field_to_activity(item), 'https://e.b/c') + def test_foreign_key_from_activity(self): + instance = fields.ForeignKey(User, on_delete=models.CASCADE) + + # test receiving an unknown remote id and loading data TODO + + # test recieving activity json TODO + + # test receiving a remote id of an object in the db + user = User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) + value = instance.field_from_activity(user.remote_id) + self.assertEqual(value, user) + + def test_one_to_one_field(self): instance = fields.OneToOneField('User', on_delete=models.CASCADE) Serializable = namedtuple('Serializable', ('to_activity', 'remote_id')) From 9eba60b0c8e24ef9299ae0a0642e20a2e3593271 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 6 Dec 2020 11:15:40 -0800 Subject: [PATCH 062/356] Fix typo in readme, update to bw-dev --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5e16597c..5a2378ef 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ This project is still young and isn't, at the momoment, very stable, so please p `cp .env.example .env` - Add your domain, email address, mailgun credentials - Set a secure redis password and secret key - - Update your nginx configuration in `nginx/defautl.conf` + - Update your nginx configuration in `nginx/default.conf` - Replace `your-domain.com` with your domain name - Run the application (this should also set up a Certbot ssl cert for your domain) `docker-compose up --build` @@ -124,13 +124,13 @@ This project is still young and isn't, at the momoment, very stable, so please p - Run docker-compose in the background `docker-compose up -d` - Initialize the database - `./fr-dev initdb` + `./bw-dev initdb` - Congrats! You did it, go to your domain and enjoy the fruits of your labors ### Configure your instance - Register a user account in the applcation UI - Make your account a superuser (warning: do *not* use django's `createsuperuser` command) - On your server, open the django shell - `./fr-dev shell` + `./bw-dev shell` - Load your user and make it a superuser ```python from bookwyrm import models From 69bb3f27516374b003c55da7ae624c211bc8ed3f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 7 Dec 2020 09:14:30 -0800 Subject: [PATCH 063/356] Fixes validation error in many to many field deserializer --- bookwyrm/models/fields.py | 15 +++++++++--- bookwyrm/tests/models/test_fields.py | 36 ++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index e1636f84..0d6c678a 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -92,7 +92,10 @@ class UsernameField(ActivitypubFieldMixin, models.CharField): ''' activitypub-aware username field ''' def __init__(self, activitypub_field='preferredUsername'): self.activitypub_field = activitypub_field - super(ActivitypubFieldMixin, self).__init__( + # I don't totally know why pylint is mad at this, but it makes it work + super( #pylint: disable=bad-super-call + ActivitypubFieldMixin, self + ).__init__( _('username'), max_length=150, unique=True, @@ -146,7 +149,10 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): def field_from_activity(self, value): items = [] for remote_id in value: - validate_remote_id(remote_id) + try: + validate_remote_id(remote_id) + except ValidationError: + return None items.append( activitypub.resolve_remote_id(self.related_model, remote_id) ) @@ -207,7 +213,10 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): url = image_slug else: return None - if not url: + + try: + validate_remote_id(url) + except ValidationError: return None response = get_image(url) diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index 06ff01e0..8fdb0e2b 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -1,6 +1,7 @@ ''' testing models ''' from io import BytesIO from collections import namedtuple +import json import pathlib import re @@ -14,10 +15,11 @@ from django.test import TestCase from django.utils import timezone from bookwyrm.models import fields, User -from bookwyrm import activitypub class ActivitypubFields(TestCase): + ''' overwrites standard model feilds to work with activitypub ''' def test_validate_remote_id(self): + ''' should look like a url ''' self.assertIsNone(fields.validate_remote_id( 'http://www.example.com' )) @@ -41,6 +43,7 @@ class ActivitypubFields(TestCase): ) def test_activitypub_field_mixin(self): + ''' generic mixin with super basic to and from functionality ''' instance = fields.ActivitypubFieldMixin() self.assertEqual(instance.field_to_activity('fish'), 'fish') self.assertEqual(instance.field_from_activity('fish'), 'fish') @@ -63,6 +66,7 @@ class ActivitypubFields(TestCase): self.assertEqual(instance.get_activitypub_field(), 'snakeCaseName') def test_remote_id_field(self): + ''' just sets some defaults on charfield ''' instance = fields.RemoteIdField() self.assertEqual(instance.max_length, 255) @@ -70,6 +74,7 @@ class ActivitypubFields(TestCase): instance.run_validators('http://www.example.com/dlfjg 23/x') def test_username_field(self): + ''' again, just setting defaults on username field ''' instance = fields.UsernameField() self.assertEqual(instance.activitypub_field, 'preferredUsername') self.assertEqual(instance.max_length, 150) @@ -83,6 +88,7 @@ class ActivitypubFields(TestCase): self.assertEqual(instance.field_to_activity('test@example.com'), 'test') def test_foreign_key(self): + ''' should be able to format a related model ''' instance = fields.ForeignKey('User', on_delete=models.CASCADE) Serializable = namedtuple('Serializable', ('to_activity', 'remote_id')) item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c') @@ -90,11 +96,22 @@ class ActivitypubFields(TestCase): self.assertEqual(instance.field_to_activity(item), 'https://e.b/c') def test_foreign_key_from_activity(self): + ''' this is the important stuff ''' instance = fields.ForeignKey(User, on_delete=models.CASCADE) + datafile = pathlib.Path(__file__).parent.joinpath( + '../data/ap_user.json' + ) + userdata = json.loads(datafile.read_bytes()) + del userdata['icon'] # test receiving an unknown remote id and loading data TODO - # test recieving activity json TODO + # test recieving activity json + value = instance.field_from_activity(userdata) + self.assertIsInstance(value, User) + self.assertEqual(value.remote_id, 'https://example.com/user/mouse') + self.assertEqual(value.name, 'MOUSE?? MOUSE!!') + # et cetera but we're not testing serializing user json # test receiving a remote id of an object in the db user = User.objects.create_user( @@ -104,12 +121,14 @@ class ActivitypubFields(TestCase): def test_one_to_one_field(self): + ''' a gussied up foreign key ''' instance = fields.OneToOneField('User', on_delete=models.CASCADE) Serializable = namedtuple('Serializable', ('to_activity', 'remote_id')) item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c') self.assertEqual(instance.field_to_activity(item), {'a': 'b'}) def test_many_to_many_field(self): + ''' lists! ''' instance = fields.ManyToManyField('User') Serializable = namedtuple('Serializable', ('to_activity', 'remote_id')) @@ -128,7 +147,12 @@ class ActivitypubFields(TestCase): 'example.com/snake_case' ) + def test_many_to_many_field_from_activity(self): + ''' resolve related fields for a list ''' + # TODO + def test_tag_field(self): + ''' a special type of many to many field ''' instance = fields.TagField('User') Serializable = namedtuple( @@ -150,8 +174,14 @@ class ActivitypubFields(TestCase): self.assertEqual(result[0].type, 'Serializable') + def test_tag_field_from_activity(self): + ''' loadin' a list of items from Links ''' + # TODO + + @responses.activate def test_image_field(self): + ''' storing images ''' user = User.objects.create_user( 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) image_file = pathlib.Path(__file__).parent.joinpath( @@ -189,6 +219,7 @@ class ActivitypubFields(TestCase): def test_datetime_field(self): + ''' this one is pretty simple, it just has to use isoformat ''' instance = fields.DateTimeField() now = timezone.now() self.assertEqual(instance.field_to_activity(now), now.isoformat()) @@ -199,5 +230,6 @@ class ActivitypubFields(TestCase): def test_array_field(self): + ''' idk why it makes them strings but probably for a good reason ''' instance = fields.ArrayField(fields.IntegerField) self.assertEqual(instance.field_to_activity([0, 1]), ['0', '1']) From 4599df752d96c1c71c67e590bcd953c89e86af87 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 7 Dec 2020 09:31:12 -0800 Subject: [PATCH 064/356] Adds tests for many to many field deserialization --- bookwyrm/models/fields.py | 2 +- bookwyrm/tests/models/test_fields.py | 35 +++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 0d6c678a..41e34b36 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -152,7 +152,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): try: validate_remote_id(remote_id) except ValidationError: - return None + continue items.append( activitypub.resolve_remote_id(self.related_model, remote_id) ) diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index 8fdb0e2b..dec602da 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -95,6 +95,7 @@ class ActivitypubFields(TestCase): # returns the remote_id field of the related object self.assertEqual(instance.field_to_activity(item), 'https://e.b/c') + @responses.activate def test_foreign_key_from_activity(self): ''' this is the important stuff ''' instance = fields.ForeignKey(User, on_delete=models.CASCADE) @@ -103,8 +104,16 @@ class ActivitypubFields(TestCase): '../data/ap_user.json' ) userdata = json.loads(datafile.read_bytes()) + # don't try to load the user icon del userdata['icon'] - # test receiving an unknown remote id and loading data TODO + + # test receiving an unknown remote id and loading data + responses.add( + responses.GET, + 'https://example.com/user/mouse', + json=userdata, + status=200) + value = instance.field_from_activity('https://example.com/user/mouse') # test recieving activity json value = instance.field_from_activity(userdata) @@ -147,9 +156,29 @@ class ActivitypubFields(TestCase): 'example.com/snake_case' ) + @responses.activate def test_many_to_many_field_from_activity(self): - ''' resolve related fields for a list ''' - # TODO + ''' resolve related fields for a list, takes a list of remote ids ''' + instance = fields.ManyToManyField(User) + datafile = pathlib.Path(__file__).parent.joinpath( + '../data/ap_user.json' + ) + userdata = json.loads(datafile.read_bytes()) + # don't try to load the user icon + del userdata['icon'] + + # test receiving an unknown remote id and loading data + responses.add( + responses.GET, + 'https://example.com/user/mouse', + json=userdata, + status=200) + value = instance.field_from_activity( + ['https://example.com/user/mouse', 'bleh'] + ) + self.assertIsInstance(value, list) + self.assertEqual(len(value), 1) + self.assertIsInstance(value[0], User) def test_tag_field(self): ''' a special type of many to many field ''' From 74ac8d60f84d5c91271e10185ff5196fba1e1c09 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 7 Dec 2020 10:32:31 -0800 Subject: [PATCH 065/356] Starts adding tests for base_activity includes init and find_existing_by_remote_id --- .../tests/activitypub/test_base_activity.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 bookwyrm/tests/activitypub/test_base_activity.py diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py new file mode 100644 index 00000000..865e1638 --- /dev/null +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -0,0 +1,81 @@ +''' tests the base functionality for activitypub dataclasses ''' +from dataclasses import dataclass +from django.test import TestCase + +from bookwyrm.activitypub.base_activity import ActivityObject, \ + find_existing_by_remote_id +from bookwyrm.activitypub import ActivitySerializerError +from bookwyrm import models + +class BaseActivity(TestCase): + ''' the super class for model-linked activitypub dataclasses ''' + def test_init(self): + ''' simple successfuly init ''' + instance = ActivityObject(id='a', type='b') + self.assertTrue(hasattr(instance, 'id')) + self.assertTrue(hasattr(instance, 'type')) + + def test_init_missing(self): + ''' init with missing required params ''' + with self.assertRaises(ActivitySerializerError): + ActivityObject() + + def test_init_extra_fields(self): + ''' init ignoring additional fields ''' + instance = ActivityObject(id='a', type='b', fish='c') + self.assertTrue(hasattr(instance, 'id')) + self.assertTrue(hasattr(instance, 'type')) + + def test_init_default_field(self): + ''' replace an existing required field with a default field ''' + @dataclass(init=False) + class TestClass(ActivityObject): + ''' test class with default field ''' + type: str = 'TestObject' + + instance = TestClass(id='a') + self.assertEqual(instance.id, 'a') + self.assertEqual(instance.type, 'TestObject') + + def test_serialize(self): + ''' simple function for converting dataclass to dict ''' + instance = ActivityObject(id='a', type='b') + serialized = instance.serialize() + self.assertIsInstance(serialized, dict) + self.assertEqual(serialized['id'], 'a') + self.assertEqual(serialized['type'], 'b') + + def test_find_existing_by_remote_id(self): + ''' attempt to match a remote id to an object in the db ''' + user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) + user.remote_id = 'http://example.com/a/b' + user.save() + + # uses a different remote id scheme + book = models.Edition.objects.create( + title='Test Edition', remote_id='http://book.com/book') + # this isn't really part of this test directly but it's helpful to state + self.assertEqual(book.origin_id, 'http://book.com/book') + self.assertNotEqual(book.remote_id, 'http://book.com/book') + + # uses subclasses + models.Comment.objects.create( + user=user, content='test status', book=book, \ + remote_id='https://comment.net') + + result = find_existing_by_remote_id(models.User, 'hi') + self.assertIsNone(result) + + result = find_existing_by_remote_id( + models.User, 'http://example.com/a/b') + self.assertEqual(result, user) + + # test using origin id + result = find_existing_by_remote_id( + models.Edition, 'http://book.com/book') + self.assertEqual(result, book) + + # test subclass match + result = find_existing_by_remote_id( + models.Status, 'https://comment.net') From f61fcb1261b63a8d7e05605408e2f2d5b8fd5d48 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 7 Dec 2020 10:46:41 -0800 Subject: [PATCH 066/356] Adds tests for resolve_remote_id --- .../tests/activitypub/test_base_activity.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py index 865e1638..e30db281 100644 --- a/bookwyrm/tests/activitypub/test_base_activity.py +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -1,9 +1,14 @@ ''' tests the base functionality for activitypub dataclasses ''' +import json +import pathlib +from unittest.mock import patch + from dataclasses import dataclass from django.test import TestCase +import responses from bookwyrm.activitypub.base_activity import ActivityObject, \ - find_existing_by_remote_id + find_existing_by_remote_id, resolve_remote_id from bookwyrm.activitypub import ActivitySerializerError from bookwyrm import models @@ -79,3 +84,35 @@ class BaseActivity(TestCase): # test subclass match result = find_existing_by_remote_id( models.Status, 'https://comment.net') + + @responses.activate + def test_resolve_remote_id(self): + ''' look up or load remote data ''' + user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) + user.remote_id = 'http://example.com/a/b' + user.save() + + # existing item + result = resolve_remote_id(models.User, 'http://example.com/a/b') + self.assertEqual(result, user) + + # remote item + datafile = pathlib.Path(__file__).parent.joinpath( + '../data/ap_user.json' + ) + userdata = json.loads(datafile.read_bytes()) + # don't try to load the user icon + del userdata['icon'] + responses.add( + responses.GET, + 'https://example.com/user/mouse', + json=userdata, + status=200) + + with patch('bookwyrm.models.user.set_remote_server.delay'): + result = resolve_remote_id( + models.User, 'https://example.com/user/mouse') + self.assertIsInstance(result, models.User) + self.assertEqual(result.remote_id, 'https://example.com/user/mouse') + self.assertEqual(result.name, 'MOUSE?? MOUSE!!') From 6817babf3c4fa7753a2efe8a31d37b49549b8a98 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 7 Dec 2020 12:19:15 -0800 Subject: [PATCH 067/356] adds some tests for to_model --- bookwyrm/activitypub/base_activity.py | 2 - .../tests/activitypub/test_base_activity.py | 89 ++++++++++++++----- 2 files changed, 69 insertions(+), 22 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 4de4dc6b..169d7ee3 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -104,14 +104,12 @@ class ActivityObject: one_to_many_fields[field.name] = value elif isinstance(model_field, ImageFileDescriptor): # image fields need custom handling - print(model_field, field.name, value) image_fields[field.name] = value else: if value == MISSING: value = None mapped_fields[field.name] = value - if instance: # updating an existing model instance for k, v in mapped_fields.items(): diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py index e30db281..7f915aa3 100644 --- a/bookwyrm/tests/activitypub/test_base_activity.py +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -1,12 +1,15 @@ ''' tests the base functionality for activitypub dataclasses ''' +from io import BytesIO import json import pathlib from unittest.mock import patch from dataclasses import dataclass from django.test import TestCase +from PIL import Image import responses +from bookwyrm import activitypub from bookwyrm.activitypub.base_activity import ActivityObject, \ find_existing_by_remote_id, resolve_remote_id from bookwyrm.activitypub import ActivitySerializerError @@ -14,6 +17,20 @@ from bookwyrm import models class BaseActivity(TestCase): ''' the super class for model-linked activitypub dataclasses ''' + def setUp(self): + ''' we're probably going to re-use this so why copy/paste ''' + self.user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) + self.user.remote_id = 'http://example.com/a/b' + self.user.save() + + datafile = pathlib.Path(__file__).parent.joinpath( + '../data/ap_user.json' + ) + self.userdata = json.loads(datafile.read_bytes()) + # don't try to load the user icon + del self.userdata['icon'] + def test_init(self): ''' simple successfuly init ''' instance = ActivityObject(id='a', type='b') @@ -52,11 +69,6 @@ class BaseActivity(TestCase): def test_find_existing_by_remote_id(self): ''' attempt to match a remote id to an object in the db ''' - user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) - user.remote_id = 'http://example.com/a/b' - user.save() - # uses a different remote id scheme book = models.Edition.objects.create( title='Test Edition', remote_id='http://book.com/book') @@ -66,7 +78,7 @@ class BaseActivity(TestCase): # uses subclasses models.Comment.objects.create( - user=user, content='test status', book=book, \ + user=self.user, content='test status', book=book, \ remote_id='https://comment.net') result = find_existing_by_remote_id(models.User, 'hi') @@ -74,7 +86,7 @@ class BaseActivity(TestCase): result = find_existing_by_remote_id( models.User, 'http://example.com/a/b') - self.assertEqual(result, user) + self.assertEqual(result, self.user) # test using origin id result = find_existing_by_remote_id( @@ -88,26 +100,15 @@ class BaseActivity(TestCase): @responses.activate def test_resolve_remote_id(self): ''' look up or load remote data ''' - user = models.User.objects.create_user( - 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) - user.remote_id = 'http://example.com/a/b' - user.save() - # existing item result = resolve_remote_id(models.User, 'http://example.com/a/b') - self.assertEqual(result, user) + self.assertEqual(result, self.user) # remote item - datafile = pathlib.Path(__file__).parent.joinpath( - '../data/ap_user.json' - ) - userdata = json.loads(datafile.read_bytes()) - # don't try to load the user icon - del userdata['icon'] responses.add( responses.GET, 'https://example.com/user/mouse', - json=userdata, + json=self.userdata, status=200) with patch('bookwyrm.models.user.set_remote_server.delay'): @@ -116,3 +117,51 @@ class BaseActivity(TestCase): self.assertIsInstance(result, models.User) self.assertEqual(result.remote_id, 'https://example.com/user/mouse') self.assertEqual(result.name, 'MOUSE?? MOUSE!!') + + def test_to_model(self): + ''' the big boy of this module. it feels janky to test this with actual + models rather than a test model, but I don't know how to make a test + model so here we are. ''' + instance = ActivityObject(id='a', type='b') + with self.assertRaises(ActivitySerializerError): + instance.to_model(models.User) + + # test setting simple fields + self.assertEqual(self.user.name, '') + update_data = activitypub.Person(**self.user.to_activity()) + update_data.name = 'New Name' + update_data.to_model(models.User, self.user) + + self.assertEqual(self.user.name, 'New Name') + + def test_to_model_foreign_key(self): + ''' test setting one to one/foreign key ''' + update_data = activitypub.Person(**self.user.to_activity()) + update_data.publicKey['publicKeyPem'] = 'hi im secure' + update_data.to_model(models.User, self.user) + self.assertEqual(self.user.key_pair.public_key, 'hi im secure') + + @responses.activate + def test_to_model_image(self): + ''' update an image field ''' + update_data = activitypub.Person(**self.user.to_activity()) + update_data.icon = {'url': 'http://www.example.com/image.jpg'} + image_file = pathlib.Path(__file__).parent.joinpath( + '../../static/images/default_avi.jpg') + image = Image.open(image_file) + output = BytesIO() + image.save(output, format=image.format) + image_data = output.getvalue() + responses.add( + responses.GET, + 'http://www.example.com/image.jpg', + body=image_data, + status=200) + + self.assertIsNone(self.user.avatar.name) + with self.assertRaises(ValueError): + self.user.avatar.file #pylint: disable=pointless-statement + + update_data.to_model(models.User, self.user) + self.assertIsNotNone(self.user.avatar.name) + self.assertIsNotNone(self.user.avatar.file) From 0a576c325cc70e291f5ab0295adbe88601f995dc Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 7 Dec 2020 13:16:42 -0800 Subject: [PATCH 068/356] Fixes deserializing tags of varied types --- bookwyrm/models/fields.py | 4 +++ .../tests/activitypub/test_base_activity.py | 36 +++++++++++++++---- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 41e34b36..1a6138fd 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -182,6 +182,10 @@ class TagField(ManyToManyField): items = [] for link_json in value: link = activitypub.Link(**link_json) + tag_type = link.type if link.type != 'Mention' else 'Person' + if tag_type != self.related_model.activity_serializer.type: + # tags can contain multiple types + continue items.append( activitypub.resolve_remote_id(self.related_model, link.href) ) diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py index 7f915aa3..44581144 100644 --- a/bookwyrm/tests/activitypub/test_base_activity.py +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -31,6 +31,9 @@ class BaseActivity(TestCase): # don't try to load the user icon del self.userdata['icon'] + self.book = models.Edition.objects.create( + title='Test Edition', remote_id='http://book.com/book') + def test_init(self): ''' simple successfuly init ''' instance = ActivityObject(id='a', type='b') @@ -70,15 +73,13 @@ class BaseActivity(TestCase): def test_find_existing_by_remote_id(self): ''' attempt to match a remote id to an object in the db ''' # uses a different remote id scheme - book = models.Edition.objects.create( - title='Test Edition', remote_id='http://book.com/book') # this isn't really part of this test directly but it's helpful to state - self.assertEqual(book.origin_id, 'http://book.com/book') - self.assertNotEqual(book.remote_id, 'http://book.com/book') + self.assertEqual(self.book.origin_id, 'http://book.com/book') + self.assertNotEqual(self.book.remote_id, 'http://book.com/book') # uses subclasses models.Comment.objects.create( - user=self.user, content='test status', book=book, \ + user=self.user, content='test status', book=self.book, \ remote_id='https://comment.net') result = find_existing_by_remote_id(models.User, 'hi') @@ -91,7 +92,7 @@ class BaseActivity(TestCase): # test using origin id result = find_existing_by_remote_id( models.Edition, 'http://book.com/book') - self.assertEqual(result, book) + self.assertEqual(result, self.book) # test subclass match result = find_existing_by_remote_id( @@ -165,3 +166,26 @@ class BaseActivity(TestCase): update_data.to_model(models.User, self.user) self.assertIsNotNone(self.user.avatar.name) self.assertIsNotNone(self.user.avatar.file) + + def test_to_model_many_to_many(self): + ''' annoying that these all need special handling ''' + status = models.Status.objects.create( + content='test status', + user=self.user, + ) + update_data = activitypub.Note(**status.to_activity()) + update_data.tag = [ + { + 'type': 'Mention', + 'name': 'gerald', + 'href': 'http://example.com/a/b' + }, + { + 'type': 'Edition', + 'name': 'gerald j. books', + 'href': 'http://book.com/book' + }, + ] + update_data.to_model(models.Status, instance=status) + self.assertEqual(status.mention_users.first(), self.user) + self.assertEqual(status.mention_books.first(), self.book) From d0c1a68df61f628618a3099ef43fc0f0439aec10 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 7 Dec 2020 13:39:05 -0800 Subject: [PATCH 069/356] Patches celery call in field tests and fixes tag field --- bookwyrm/models/fields.py | 2 ++ bookwyrm/tests/models/test_fields.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 1a6138fd..b4d08b04 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -179,6 +179,8 @@ class TagField(ManyToManyField): return tags def field_from_activity(self, value): + if not isinstance(value, list): + return None items = [] for link_json in value: link = activitypub.Link(**link_json) diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index dec602da..58d9e08c 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -4,6 +4,7 @@ from collections import namedtuple import json import pathlib import re +from unittest.mock import patch from PIL import Image import responses @@ -113,7 +114,9 @@ class ActivitypubFields(TestCase): 'https://example.com/user/mouse', json=userdata, status=200) - value = instance.field_from_activity('https://example.com/user/mouse') + with patch('bookwyrm.models.user.set_remote_server.delay'): + value = instance.field_from_activity( + 'https://example.com/user/mouse') # test recieving activity json value = instance.field_from_activity(userdata) @@ -173,9 +176,10 @@ class ActivitypubFields(TestCase): 'https://example.com/user/mouse', json=userdata, status=200) - value = instance.field_from_activity( - ['https://example.com/user/mouse', 'bleh'] - ) + with patch('bookwyrm.models.user.set_remote_server.delay'): + value = instance.field_from_activity( + ['https://example.com/user/mouse', 'bleh'] + ) self.assertIsInstance(value, list) self.assertEqual(len(value), 1) self.assertIsInstance(value[0], User) From 4d4ee8b8c3db1c396c18a1a21dd6e642427e7650 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 7 Dec 2020 18:28:42 -0800 Subject: [PATCH 070/356] Starts getting reverse fields working for deserialization also fixes the fields on the image model and runs a long overdue migration --- bookwyrm/activitypub/base_activity.py | 77 ++-- .../migrations/0020_auto_20201208_0213.py | 353 ++++++++++++++++++ bookwyrm/models/attachment.py | 7 +- bookwyrm/models/base_model.py | 8 +- bookwyrm/models/book.py | 3 +- bookwyrm/models/status.py | 3 +- bookwyrm/models/user.py | 2 +- .../tests/activitypub/test_base_activity.py | 40 +- 8 files changed, 434 insertions(+), 59 deletions(-) create mode 100644 bookwyrm/migrations/0020_auto_20201208_0213.py diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 169d7ee3..a478a41c 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -76,77 +76,68 @@ class ActivityObject: # check for an existing instance, if we're not updating a known obj if not instance: - instance = find_existing_by_remote_id(model, self.id) + instance = find_existing_by_remote_id(model, self.id) or model() # TODO: deduplicate books by identifiers - mapped_fields = {} many_to_many_fields = {} - one_to_many_fields = {} - image_fields = {} - for field in model._meta.get_fields(): if not hasattr(field, 'field_to_activity'): continue - activitypub_field = field.get_activitypub_field() - value = field.field_from_activity(getattr(self, activitypub_field)) - if value is None: + # call the formatter associated with the model field class + value = field.field_from_activity( + getattr(self, field.get_activitypub_field()) + ) + if value is None or value is MISSING: continue model_field = getattr(model, field.name) - if isinstance(model_field, ForwardManyToOneDescriptor): - mapped_fields[field.name] = value if isinstance(model_field, ManyToManyDescriptor): - # status mentions book/users + # status mentions book/users for example, stash this for later many_to_many_fields[field.name] = value - elif isinstance(model_field, ReverseManyToOneDescriptor): - # attachments on Status, for example - one_to_many_fields[field.name] = value elif isinstance(model_field, ImageFileDescriptor): # image fields need custom handling - image_fields[field.name] = value + getattr(instance, field.name).save(*value) else: - if value == MISSING: - value = None - mapped_fields[field.name] = value + # just a good old fashioned model.field = value + setattr(instance, field.name, value) - if instance: - # updating an existing model instance - for k, v in mapped_fields.items(): - setattr(instance, k, v) - instance.save() - else: - # creating a new model instance - instance = model.objects.create(**mapped_fields) + instance.save() - # --- these are all fields that can't be saved until after the - # instance has an id (after it's been saved). ---------------# - - # add images - for (model_key, value) in image_fields.items(): - if not value: - continue - getattr(instance, model_key).save(*value, save=True) - - # add many to many fields + # add many to many fields, which have to be set post-save for (model_key, values) in many_to_many_fields.items(): # mention books, mention users, followers getattr(instance, model_key).set(values) - # add one to many fields - for (model_key, values) in one_to_many_fields.items(): - if values == MISSING: + if not hasattr(model, 'deserialize_reverse_fields'): + return instance + + # reversed relationships in the models + for (model_field_name, activity_field_name) in \ + model.deserialize_reverse_fields: + if not activity_field_name: continue - model_field = getattr(instance, model_key) - related_model = model_field.model + # attachments on Status, for example + values = getattr(self, activity_field_name) + if values is None or values is MISSING: + continue + try: + # this is for one to many + related_model = getattr(model, model_field_name).field.model + except AttributeError: + # it's a one to one or foreign key + related_model = getattr(model, model_field_name)\ + .related.related_model + values = [values] + for item in values: if isinstance(item, str): item = resolve_remote_id(related_model, item) else: item = related_model.activity_serializer(**item) item = item.to_model(related_model) - field_name = instance.__class__.__name__.lower() - setattr(item, field_name, instance) + related_name = instance.__class__.__name__.lower() + setattr(item, related_name, instance) item.save() return instance diff --git a/bookwyrm/migrations/0020_auto_20201208_0213.py b/bookwyrm/migrations/0020_auto_20201208_0213.py new file mode 100644 index 00000000..9c5345c7 --- /dev/null +++ b/bookwyrm/migrations/0020_auto_20201208_0213.py @@ -0,0 +1,353 @@ +# Generated by Django 3.0.7 on 2020-12-08 02:13 + +import bookwyrm.models.fields +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0019_auto_20201130_1939'), + ] + + operations = [ + migrations.AlterField( + model_name='author', + name='aliases', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + ), + migrations.AlterField( + model_name='author', + name='bio', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='author', + name='born', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='author', + name='died', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='author', + name='name', + field=bookwyrm.models.fields.CharField(max_length=255), + ), + migrations.AlterField( + model_name='author', + name='openlibrary_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='author', + name='wikipedia_link', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='authors', + field=bookwyrm.models.fields.ManyToManyField(to='bookwyrm.Author'), + ), + migrations.AlterField( + model_name='book', + name='cover', + field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='covers/'), + ), + migrations.AlterField( + model_name='book', + name='description', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='first_published_date', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='goodreads_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='languages', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + ), + migrations.AlterField( + model_name='book', + name='librarything_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='openlibrary_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='published_date', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='series', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='series_number', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='sort_title', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='subject_places', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='book', + name='subjects', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='book', + name='subtitle', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='title', + field=bookwyrm.models.fields.CharField(max_length=255), + ), + migrations.AlterField( + model_name='boost', + name='boosted_status', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='boosters', to='bookwyrm.Status'), + ), + migrations.AlterField( + model_name='comment', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='edition', + name='asin', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='isbn_10', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='isbn_13', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='oclc_number', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='pages', + field=bookwyrm.models.fields.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='edition', + name='parent_work', + field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'), + ), + migrations.AlterField( + model_name='edition', + name='physical_format', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='publishers', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + ), + migrations.AlterField( + model_name='favorite', + name='status', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'), + ), + migrations.AlterField( + model_name='favorite', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='image', + name='caption', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='image', + name='image', + field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='status/'), + ), + migrations.AlterField( + model_name='quotation', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='quotation', + name='quote', + field=bookwyrm.models.fields.TextField(), + ), + migrations.AlterField( + model_name='review', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='review', + name='name', + field=bookwyrm.models.fields.CharField(max_length=255, null=True), + ), + migrations.AlterField( + model_name='review', + name='rating', + field=bookwyrm.models.fields.IntegerField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(5)]), + ), + migrations.AlterField( + model_name='shelf', + name='name', + field=bookwyrm.models.fields.CharField(max_length=100), + ), + migrations.AlterField( + model_name='shelf', + name='privacy', + field=bookwyrm.models.fields.CharField(choices=[('public', 'Public'), ('unlisted', 'Unlisted'), ('followers', 'Followers'), ('direct', 'Direct')], default='public', max_length=255), + ), + migrations.AlterField( + model_name='shelf', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='shelfbook', + name='added_by', + field=bookwyrm.models.fields.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='shelfbook', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='shelfbook', + name='shelf', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Shelf'), + ), + migrations.AlterField( + model_name='status', + name='content', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='status', + name='mention_books', + field=bookwyrm.models.fields.TagField(related_name='mention_book', to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='status', + name='mention_users', + field=bookwyrm.models.fields.TagField(related_name='mention_user', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='status', + name='published_date', + field=bookwyrm.models.fields.DateTimeField(default=django.utils.timezone.now), + ), + migrations.AlterField( + model_name='status', + name='reply_parent', + field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'), + ), + migrations.AlterField( + model_name='status', + name='sensitive', + field=bookwyrm.models.fields.BooleanField(default=False), + ), + migrations.AlterField( + model_name='status', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='tag', + name='name', + field=bookwyrm.models.fields.CharField(max_length=100, unique=True), + ), + migrations.AlterField( + model_name='userblocks', + name='user_object', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_object', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userblocks', + name='user_subject', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_subject', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollowrequest', + name='user_object', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_object', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollowrequest', + name='user_subject', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_subject', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollows', + name='user_object', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_object', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollows', + name='user_subject', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_subject', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='usertag', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='usertag', + name='tag', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag'), + ), + migrations.AlterField( + model_name='usertag', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='work', + name='default_edition', + field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='work', + name='lccn', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/bookwyrm/models/attachment.py b/bookwyrm/models/attachment.py index 6a92240d..b3337e15 100644 --- a/bookwyrm/models/attachment.py +++ b/bookwyrm/models/attachment.py @@ -9,7 +9,7 @@ from . import fields class Attachment(ActivitypubMixin, BookWyrmModel): ''' an image (or, in the future, video etc) associated with a status ''' - status = fields.ForeignKey( + status = models.ForeignKey( 'Status', on_delete=models.CASCADE, related_name='attachments', @@ -23,7 +23,8 @@ class Attachment(ActivitypubMixin, BookWyrmModel): class Image(Attachment): ''' an image attachment ''' - image = fields.ImageField(upload_to='status/', null=True, blank=True) - caption = fields.TextField(null=True, blank=True) + image = fields.ImageField( + upload_to='status/', null=True, blank=True, activitypub_field='url') + caption = fields.TextField(null=True, blank=True, activitypub_field='name') activity_serializer = activitypub.Image diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index d3c9471f..1e437152 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -83,9 +83,11 @@ class ActivitypubMixin: if hasattr(self, 'serialize_reverse_fields'): # for example, editions of a work - for field_name in self.serialize_reverse_fields: - related_field = getattr(self, field_name) - activity[field_name] = unfurl_related_field(related_field) + for model_field_name, activity_field_name in \ + self.serialize_reverse_fields: + related_field = getattr(self, model_field_name) + activity[activity_field_name] = \ + unfurl_related_field(related_field) if not activity.get('id'): activity['id'] = self.get_remote_id() diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index da532561..47b8b99e 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -96,7 +96,8 @@ class Work(OrderedCollectionPageMixin, Book): return self.default_edition or self.editions.first() activity_serializer = activitypub.Work - serialize_reverse_fields = ['editions'] + serialize_reverse_fields = [('editions', 'editions')] + deserialize_reverse_fields = [('editions', 'editions')] class Edition(Book): diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index a2b873bb..55036f2c 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -45,7 +45,8 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): objects = InheritanceManager() activity_serializer = activitypub.Note - serialize_reverse_fields = ['attachments'] + serialize_reverse_fields = [('attachments', 'attachment')] + deserialize_reverse_fields = [('attachments', 'attachment')] #----- replies collection activitypub ----# @classmethod diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 6395987b..531b0da2 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -166,7 +166,7 @@ class KeyPair(ActivitypubMixin, BookWyrmModel): blank=True, null=True, activitypub_field='publicKeyPem') activity_serializer = activitypub.PublicKey - serialize_reverse_fields = ['owner'] + serialize_reverse_fields = [('owner', 'owner')] def get_remote_id(self): # self.owner is set by the OneToOneField on User diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py index 44581144..6892f0e4 100644 --- a/bookwyrm/tests/activitypub/test_base_activity.py +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -34,6 +34,13 @@ class BaseActivity(TestCase): self.book = models.Edition.objects.create( title='Test Edition', remote_id='http://book.com/book') + image_file = pathlib.Path(__file__).parent.joinpath( + '../../static/images/default_avi.jpg') + image = Image.open(image_file) + output = BytesIO() + image.save(output, format=image.format) + self.image_data = output.getvalue() + def test_init(self): ''' simple successfuly init ''' instance = ActivityObject(id='a', type='b') @@ -147,16 +154,10 @@ class BaseActivity(TestCase): ''' update an image field ''' update_data = activitypub.Person(**self.user.to_activity()) update_data.icon = {'url': 'http://www.example.com/image.jpg'} - image_file = pathlib.Path(__file__).parent.joinpath( - '../../static/images/default_avi.jpg') - image = Image.open(image_file) - output = BytesIO() - image.save(output, format=image.format) - image_data = output.getvalue() responses.add( responses.GET, 'http://www.example.com/image.jpg', - body=image_data, + body=self.image_data, status=200) self.assertIsNone(self.user.avatar.name) @@ -189,3 +190,28 @@ class BaseActivity(TestCase): update_data.to_model(models.Status, instance=status) self.assertEqual(status.mention_users.first(), self.user) self.assertEqual(status.mention_books.first(), self.book) + + + @responses.activate + def test_to_model_one_to_many(self): + ''' these are reversed relationships, where the secondary object + keys the primary object but not vice versa ''' + status = models.Status.objects.create( + content='test status', + user=self.user, + ) + update_data = activitypub.Note(**status.to_activity()) + update_data.attachment = [{ + 'url': 'http://www.example.com/image.jpg', + 'name': 'alt text', + 'type': 'Image', + }] + + responses.add( + responses.GET, + 'http://www.example.com/image.jpg', + body=self.image_data, + status=200) + + update_data.to_model(models.Status, instance=status) + self.assertIsInstance(status.attachments.first(), models.Image) From cc42e9d149cdbc5dcf0825bc600e7f58b9919689 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 8 Dec 2020 09:43:12 -0800 Subject: [PATCH 071/356] Asyncronously set related fields --- bookwyrm/activitypub/base_activity.py | 64 ++++++++++++++++++--------- bookwyrm/models/fields.py | 9 ++++ bookwyrm/templates/book.html | 4 +- celerywyrm/celery.py | 3 +- 4 files changed, 57 insertions(+), 23 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index a478a41c..c784c3c1 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -2,12 +2,13 @@ from dataclasses import dataclass, fields, MISSING from json import JSONEncoder -from django.db.models.fields.related_descriptors \ - import ForwardManyToOneDescriptor, ManyToManyDescriptor, \ - ReverseManyToOneDescriptor +from django.apps import apps +from django.db import transaction from django.db.models.fields.files import ImageFileDescriptor +from django.db.models.fields.related_descriptors import ManyToManyDescriptor from bookwyrm.connectors import ConnectorException, get_data +from bookwyrm.tasks import app class ActivitySerializerError(ValueError): ''' routine problems serializing activitypub json ''' @@ -64,7 +65,8 @@ class ActivityObject: setattr(self, field.name, value) - def to_model(self, model, instance=None): + @transaction.atomic + def to_model(self, model, instance=None, save=True): ''' convert from an activity to a model instance ''' if not isinstance(self, model.activity_serializer): raise ActivitySerializerError( @@ -97,26 +99,28 @@ class ActivityObject: many_to_many_fields[field.name] = value elif isinstance(model_field, ImageFileDescriptor): # image fields need custom handling - getattr(instance, field.name).save(*value) + getattr(instance, field.name).save(*value, save=save) else: # just a good old fashioned model.field = value setattr(instance, field.name, value) + if not save: + # we can't set many to many and reverse fields on an unsaved object + return instance + instance.save() # add many to many fields, which have to be set post-save for (model_key, values) in many_to_many_fields.items(): - # mention books, mention users, followers + # mention books/users, for example getattr(instance, model_key).set(values) - if not hasattr(model, 'deserialize_reverse_fields'): + if not save or not hasattr(model, 'deserialize_reverse_fields'): return instance # reversed relationships in the models for (model_field_name, activity_field_name) in \ model.deserialize_reverse_fields: - if not activity_field_name: - continue # attachments on Status, for example values = getattr(self, activity_field_name) if values is None or values is MISSING: @@ -131,15 +135,13 @@ class ActivityObject: values = [values] for item in values: - if isinstance(item, str): - item = resolve_remote_id(related_model, item) - else: - item = related_model.activity_serializer(**item) - item = item.to_model(related_model) - related_name = instance.__class__.__name__.lower() - setattr(item, related_name, instance) - item.save() - + set_related_field.delay( + related_model.__name__, + instance.__class__.__name__, + instance.__class__.__name__.lower(), + instance.remote_id, + item + ) return instance @@ -150,6 +152,28 @@ class ActivityObject: return data +@app.task +@transaction.atomic +def set_related_field( + model_name, origin_model_name, + related_field_name, related_remote_id, data): + ''' load reverse related fields (editions, attachments) without blocking ''' + model = apps.get_model('bookwyrm.%s' % model_name, require_ready=True) + origin_model = apps.get_model( + 'bookwyrm.%s' % origin_model_name, + require_ready=True + ) + + if isinstance(data, str): + item = resolve_remote_id(model, data, save=False) + else: + item = model.activity_serializer(**data) + item = item.to_model(model, save=False) + instance = find_existing_by_remote_id(origin_model, related_remote_id) + setattr(item, related_field_name, instance) + item.save() + + def find_existing_by_remote_id(model, remote_id): ''' check for an existing instance of this id in the db ''' objects = model.objects @@ -168,7 +192,7 @@ def find_existing_by_remote_id(model, remote_id): return result -def resolve_remote_id(model, remote_id, refresh=False): +def resolve_remote_id(model, remote_id, refresh=False, save=True): ''' look up the remote_id in the database or load it remotely ''' result = find_existing_by_remote_id(model, remote_id) if result and not refresh: @@ -184,4 +208,4 @@ def resolve_remote_id(model, remote_id, refresh=False): item = model.activity_serializer(**data) # if we're refreshing, "result" will be set and we'll update it - return item.to_model(model, instance=result) + return item.to_model(model, instance=result, save=save) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index b4d08b04..f480ab89 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -126,6 +126,15 @@ class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey): return None return value.remote_id + def field_from_activity(self, value): + print(value) + try: + validate_remote_id(value) + except ValidationError: + return None + return activitypub.resolve_remote_id(self.related_model, value) + + class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField): ''' activitypub-aware foreign key field ''' diff --git a/bookwyrm/templates/book.html b/bookwyrm/templates/book.html index 51fbdafc..8b21b88c 100644 --- a/bookwyrm/templates/book.html +++ b/bookwyrm/templates/book.html @@ -86,8 +86,8 @@ {% endif %} - {% if book.parent_work.edition_set.count > 1 %} -

{{ book.parent_work.edition_set.count }} editions

+ {% if book.parent_work.editions.count > 1 %} +

{{ book.parent_work.editions.count }} editions

{% endif %} diff --git a/celerywyrm/celery.py b/celerywyrm/celery.py index a47aad32..efa081ee 100644 --- a/celerywyrm/celery.py +++ b/celerywyrm/celery.py @@ -19,8 +19,9 @@ app.config_from_object('django.conf:settings', namespace='CELERY') # Load task modules from all registered Django app configs. app.autodiscover_tasks() -app.autodiscover_tasks(['bookwyrm'], related_name='broadcast') +app.autodiscover_tasks(['bookwyrm'], related_name='activitypub.base_activity') app.autodiscover_tasks(['bookwyrm'], related_name='books_manager') +app.autodiscover_tasks(['bookwyrm'], related_name='broadcast') app.autodiscover_tasks(['bookwyrm'], related_name='emailing') app.autodiscover_tasks(['bookwyrm'], related_name='goodreads_import') app.autodiscover_tasks(['bookwyrm'], related_name='incoming') From ef2a07884fb3699c3c2b0e807354d50231f7df8a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 9 Dec 2020 11:57:29 -0800 Subject: [PATCH 072/356] Throws validation error when remote_id is None --- bookwyrm/models/fields.py | 3 +-- bookwyrm/tests/activitypub/test_quotation.py | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index f480ab89..fae3cb57 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -18,7 +18,7 @@ from bookwyrm.connectors import get_image def validate_remote_id(value): ''' make sure the remote_id looks like a url ''' - if not re.match(r'^http.?:\/\/[^\s]+$', value): + if not value or not re.match(r'^http.?:\/\/[^\s]+$', value): raise ValidationError( _('%(value)s is not a valid remote_id'), params={'value': value}, @@ -127,7 +127,6 @@ class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey): return value.remote_id def field_from_activity(self, value): - print(value) try: validate_remote_id(value) except ValidationError: diff --git a/bookwyrm/tests/activitypub/test_quotation.py b/bookwyrm/tests/activitypub/test_quotation.py index b0699571..60920889 100644 --- a/bookwyrm/tests/activitypub/test_quotation.py +++ b/bookwyrm/tests/activitypub/test_quotation.py @@ -1,3 +1,4 @@ +''' quotation activty object serializer class ''' import json import pathlib from unittest.mock import patch @@ -9,6 +10,7 @@ from bookwyrm import activitypub, models class Quotation(TestCase): ''' we have hecka ways to create statuses ''' def setUp(self): + ''' model objects we'll need ''' with patch('bookwyrm.models.user.set_remote_server.delay'): self.user = models.User.objects.create_user( 'mouse', 'mouse@mouse.mouse', 'mouseword', @@ -28,6 +30,7 @@ class Quotation(TestCase): def test_quotation_activity(self): + ''' create a Quoteation ap object from json ''' quotation = activitypub.Quotation(**self.status_data) self.assertEqual(quotation.type, 'Quotation') @@ -41,6 +44,7 @@ class Quotation(TestCase): def test_activity_to_model(self): + ''' create a model instance from an activity object ''' activity = activitypub.Quotation(**self.status_data) quotation = activity.to_model(models.Quotation) From 6b9db97ab86bc008298503ea13729d00ad352621 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 9 Dec 2020 13:11:42 -0800 Subject: [PATCH 073/356] tests set_related_field --- .../tests/activitypub/test_base_activity.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py index 6892f0e4..26d63b61 100644 --- a/bookwyrm/tests/activitypub/test_base_activity.py +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -11,7 +11,7 @@ import responses from bookwyrm import activitypub from bookwyrm.activitypub.base_activity import ActivityObject, \ - find_existing_by_remote_id, resolve_remote_id + find_existing_by_remote_id, resolve_remote_id, set_related_field from bookwyrm.activitypub import ActivitySerializerError from bookwyrm import models @@ -213,5 +213,32 @@ class BaseActivity(TestCase): body=self.image_data, status=200) - update_data.to_model(models.Status, instance=status) + # sets the celery task call to the function call + with patch( + 'bookwyrm.activitypub.base_activity.set_related_field.delay'): + update_data.to_model(models.Status, instance=status) + self.assertIsNone(status.attachments.first()) + + + @responses.activate + def test_set_related_field(self): + ''' celery task to add back-references to created objects ''' + status = models.Status.objects.create( + content='test status', + user=self.user, + ) + data = { + 'url': 'http://www.example.com/image.jpg', + 'name': 'alt text', + 'type': 'Image', + } + responses.add( + responses.GET, + 'http://www.example.com/image.jpg', + body=self.image_data, + status=200) + set_related_field( + 'Image', 'Status', 'status', status.remote_id, data) + self.assertIsInstance(status.attachments.first(), models.Image) + self.assertIsNotNone(status.attachments.first().image) From 7204068d2ae60004ae9e8322bd4857f3ea129f23 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 9 Dec 2020 13:35:36 -0800 Subject: [PATCH 074/356] Removes unnecessary override of field_from_activity on foreign key --- bookwyrm/models/fields.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index fae3cb57..fb94e1a6 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -126,14 +126,6 @@ class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey): return None return value.remote_id - def field_from_activity(self, value): - try: - validate_remote_id(value) - except ValidationError: - return None - return activitypub.resolve_remote_id(self.related_model, value) - - class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField): ''' activitypub-aware foreign key field ''' From c9f7cbb9f8757819ce2d79dbfbba4483820921a7 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 9 Dec 2020 14:49:47 -0800 Subject: [PATCH 075/356] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5a2378ef..51e39541 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Social reading and reviewing, decentralized with ActivityPub - [The role of federation](#the-role-of-federation) - [Features](#features) - [Setting up the developer environment](#setting-up-the-developer-environment) + - [Installing in Production](#installing-in-production) - [Project structure](#project-structure) - [Book data](#book-data) - [Contributing](#contributing) From 3aba3c8052fce2ef09e69e2669fbb9de460073c6 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 9 Dec 2020 15:08:53 -0800 Subject: [PATCH 076/356] Create FUNDING.yml --- .github/FUNDING.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..cfbe0524 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: bookwrym +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From adbc8c3f4cdc7fa286e87bd33cb68ac267722fd1 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 11 Dec 2020 11:26:34 -0800 Subject: [PATCH 077/356] Preview statuses in notifications Fixes #363 --- bookwyrm/templates/notifications.html | 31 +++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/bookwyrm/templates/notifications.html b/bookwyrm/templates/notifications.html index bc96f307..f9494c31 100644 --- a/bookwyrm/templates/notifications.html +++ b/bookwyrm/templates/notifications.html @@ -1,5 +1,6 @@ {% extends 'layout.html' %} -{% load humanize %}l +{% load humanize %} +{% load fr_display %} {% block content %}

Notifications

@@ -12,9 +13,10 @@
{% for notification in notifications %} -
-
+
+

+ {# DESCRIPTION #} {% if notification.related_user %} {% include 'snippets/avatar.html' with user=notification.related_user %} {% include 'snippets/username.html' with user=notification.related_user %} @@ -30,10 +32,8 @@ replied to your status - {% elif notification.notification_type == 'FOLLOW' %} followed you - {% elif notification.notification_type == 'FOLLOW_REQUEST' %} sent you a follow request

@@ -45,17 +45,30 @@ {% endif %} {% else %} your import completed. - {% endif %}

- -

{{ notification.created_date | naturaltime }}

+ {% if notification.related_status %} +
+ {# PREVIEW #} +
+
+
+ {{ notification.related_status.content | truncatewords_html:10 }} +
+
+ {{ notification.related_status.published_date | post_date }} + {% include 'snippets/privacy-icons.html' with item=notification.related_status %} +
+
+
+
+ {% endif %}
{% endfor %} + {% if not notifications %}

You're all caught up!

{% endif %}
- {% endblock %} From b33ea40da278649a56fba91cf0fce87b6f1a3751 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 11 Dec 2020 12:07:37 -0800 Subject: [PATCH 078/356] Use context processor to include site settings in templates --- bookwyrm/settings.py | 1 + bookwyrm/templates/about.html | 4 ++-- bookwyrm/templates/invite.html | 2 +- bookwyrm/templates/layout.html | 17 ++++++++++++----- bookwyrm/templates/login.html | 4 ++-- bookwyrm/templates/password_reset.html | 2 +- bookwyrm/templates/snippets/about.html | 4 ++-- bookwyrm/view_actions.py | 2 -- bookwyrm/views.py | 3 --- 9 files changed, 21 insertions(+), 18 deletions(-) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index ecabb8d7..3784158c 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -75,6 +75,7 @@ TEMPLATES = [ 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', + 'bookwyrm.context_processors.site_settings', ], }, }, diff --git a/bookwyrm/templates/about.html b/bookwyrm/templates/about.html index 25caf5e2..aa7426ca 100644 --- a/bookwyrm/templates/about.html +++ b/bookwyrm/templates/about.html @@ -3,13 +3,13 @@
- {% include 'snippets/about.html' with site_settings=site_settings %} + {% include 'snippets/about.html' %}

Code of Conduct

- {{ site_settings.code_of_conduct | safe }} + {{ site.code_of_conduct | safe }}
diff --git a/bookwyrm/templates/invite.html b/bookwyrm/templates/invite.html index 332a5ed2..42f564d6 100644 --- a/bookwyrm/templates/invite.html +++ b/bookwyrm/templates/invite.html @@ -15,7 +15,7 @@
- {% include 'snippets/about.html' with site_settings=site_settings %} + {% include 'snippets/about.html' %}
diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html index 4ae70a4b..a363debd 100644 --- a/bookwyrm/templates/layout.html +++ b/bookwyrm/templates/layout.html @@ -2,7 +2,7 @@ - {% if title %}{{ title }} | {% endif %}BookWyrm + {% if title %}{{ title }} | {% endif %}{{ site.name }} @@ -11,12 +11,10 @@ - - + + - - @@ -123,6 +121,15 @@ {% endblock %}
+
+
+
+ About this server + +
+
+
+ diff --git a/bookwyrm/templates/login.html b/bookwyrm/templates/login.html index 0142d390..ce5d9e41 100644 --- a/bookwyrm/templates/login.html +++ b/bookwyrm/templates/login.html @@ -36,7 +36,7 @@
- {% if site_settings.allow_registration %} + {% if site.allow_registration %}

Create an Account

{% include 'snippets/register_form.html' %} @@ -50,7 +50,7 @@
- {% include 'snippets/about.html' with site_settings=site_settings %} + {% include 'snippets/about.html' %}

More about this site diff --git a/bookwyrm/templates/password_reset.html b/bookwyrm/templates/password_reset.html index ac2ba7bd..99d114e4 100644 --- a/bookwyrm/templates/password_reset.html +++ b/bookwyrm/templates/password_reset.html @@ -34,7 +34,7 @@

- {% include 'snippets/about.html' with site_settings=site_settings %} + {% include 'snippets/about.html' %}
diff --git a/bookwyrm/templates/snippets/about.html b/bookwyrm/templates/snippets/about.html index 35b39f82..9660f521 100644 --- a/bookwyrm/templates/snippets/about.html +++ b/bookwyrm/templates/snippets/about.html @@ -1,7 +1,7 @@ -

About {{ site_settings.name }}

+

About {{ site.name }}

BookWyrm

- {{ site_settings.instance_description }} + {{ site.instance_description }}

diff --git a/bookwyrm/view_actions.py b/bookwyrm/view_actions.py index bb862007..c3b48b3b 100644 --- a/bookwyrm/view_actions.py +++ b/bookwyrm/view_actions.py @@ -42,7 +42,6 @@ def user_login(request): login_form.non_field_errors = 'Username or password are incorrect' register_form = forms.RegisterForm() data = { - 'site_settings': models.SiteSettings.get(), 'login_form': login_form, 'register_form': register_form } @@ -78,7 +77,6 @@ def register(request): if errors: data = { - 'site_settings': models.SiteSettings.get(), 'login_form': forms.LoginForm(), 'register_form': form } diff --git a/bookwyrm/views.py b/bookwyrm/views.py index e0feaee7..ab42557c 100644 --- a/bookwyrm/views.py +++ b/bookwyrm/views.py @@ -235,7 +235,6 @@ def login_page(request): # send user to the login page data = { 'title': 'Login', - 'site_settings': models.SiteSettings.get(), 'login_form': forms.LoginForm(), 'register_form': forms.RegisterForm(), } @@ -247,7 +246,6 @@ def about_page(request): ''' more information about the instance ''' data = { 'title': 'About', - 'site_settings': models.SiteSettings.get(), } return TemplateResponse(request, 'about.html', data) @@ -295,7 +293,6 @@ def invite_page(request, code): data = { 'title': 'Join', - 'site_settings': models.SiteSettings.get(), 'register_form': forms.RegisterForm(), 'invite': invite, } From 117d2435774ff9521f9028e472193e7253413106 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 11 Dec 2020 12:31:02 -0800 Subject: [PATCH 079/356] Expand site settings and add footer --- bookwyrm/context_processors.py | 8 ++++++ .../migrations/0016_auto_20201211_2026.py | 28 +++++++++++++++++++ bookwyrm/models/site.py | 3 ++ bookwyrm/templates/layout.html | 19 +++++++++++-- 4 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 bookwyrm/context_processors.py create mode 100644 bookwyrm/migrations/0016_auto_20201211_2026.py diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py new file mode 100644 index 00000000..72839dce --- /dev/null +++ b/bookwyrm/context_processors.py @@ -0,0 +1,8 @@ +''' customize the info available in context for rendering templates ''' +from bookwyrm import models + +def site_settings(request): + ''' include the custom info about the site ''' + return { + 'site': models.SiteSettings.objects.get() + } diff --git a/bookwyrm/migrations/0016_auto_20201211_2026.py b/bookwyrm/migrations/0016_auto_20201211_2026.py new file mode 100644 index 00000000..46b6140c --- /dev/null +++ b/bookwyrm/migrations/0016_auto_20201211_2026.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.7 on 2020-12-11 20:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0015_auto_20201128_0349'), + ] + + operations = [ + migrations.AddField( + model_name='sitesettings', + name='admin_email', + field=models.EmailField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='sitesettings', + name='support_link', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='sitesettings', + name='support_title', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index aa2e2a67..1577e369 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -17,6 +17,9 @@ class SiteSettings(models.Model): code_of_conduct = models.TextField( default="Add a code of conduct here.") allow_registration = models.BooleanField(default=True) + support_link = models.CharField(max_length=255, null=True, blank=True) + support_title = models.CharField(max_length=100, null=True, blank=True) + admin_email = models.EmailField(max_length=255, null=True, blank=True) @classmethod def get(cls): diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html index a363debd..b37c9cda 100644 --- a/bookwyrm/templates/layout.html +++ b/bookwyrm/templates/layout.html @@ -124,8 +124,23 @@
- About this server - +

+ About this server +

+ {% if site.admin_email %} +

+ Contact site admin +

+ {% endif %} +
+ {% if site.support_link %} +
+ + Support {{ site.name }} on {{ site.support_title }} +
+ {% endif %} +
+ BookWyrm is open source software. You can contribute or report issues on GitHub.
From 72eb94315a3636e88dddc825c1705c798cd7bc23 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 11 Dec 2020 16:39:58 -0800 Subject: [PATCH 080/356] Adds shelf info to book page - includes change shelf button - WIP button for switching to the current edition --- bookwyrm/templates/book.html | 97 ++++--------------- bookwyrm/templates/snippets/readthrough.html | 80 +++++++++++++++ .../snippets/switch_edition_button.html | 3 + bookwyrm/views.py | 21 ++-- 4 files changed, 116 insertions(+), 85 deletions(-) create mode 100644 bookwyrm/templates/snippets/readthrough.html create mode 100644 bookwyrm/templates/snippets/switch_edition_button.html diff --git a/bookwyrm/templates/book.html b/bookwyrm/templates/book.html index 51fbdafc..eee22898 100644 --- a/bookwyrm/templates/book.html +++ b/bookwyrm/templates/book.html @@ -91,87 +91,26 @@ {% endif %}
- {% for readthrough in readthroughs %} -
- - -
- -
- - -
- + {# user's relationship to the book #}
- - + {% for shelf in user_shelves %} +

+ This edition is on your {{ shelf.shelf.name }} shelf. + {% include 'snippets/shelf_selector.html' with current=shelf.shelf %} +

+ {% endfor %} + + {% for shelf in other_edition_shelves %} +

+ A different edition of this book is on your {{ shelf.shelf.name }} shelf. + {% include 'snippets/switch_edition_button.html' with desired_edition=book %} +

+ {% endfor %} + + {% for readthrough in readthroughs %} + {% include 'snippets/readthrough.html' with readthrough=readthrough %} + {% endfor %}
- {% endfor %} {% if request.user.is_authenticated %}
diff --git a/bookwyrm/templates/snippets/readthrough.html b/bookwyrm/templates/snippets/readthrough.html new file mode 100644 index 00000000..4d6ca03a --- /dev/null +++ b/bookwyrm/templates/snippets/readthrough.html @@ -0,0 +1,80 @@ +{% load humanize %} +
+ + +
+ +
+ + +
+ +
+ + +
diff --git a/bookwyrm/templates/snippets/switch_edition_button.html b/bookwyrm/templates/snippets/switch_edition_button.html new file mode 100644 index 00000000..3fa09f4f --- /dev/null +++ b/bookwyrm/templates/snippets/switch_edition_button.html @@ -0,0 +1,3 @@ +
+ +
diff --git a/bookwyrm/views.py b/bookwyrm/views.py index e0feaee7..7c27eade 100644 --- a/bookwyrm/views.py +++ b/bookwyrm/views.py @@ -5,8 +5,7 @@ from django.contrib.auth.decorators import login_required, permission_required from django.contrib.postgres.search import TrigramSimilarity from django.core.paginator import Paginator from django.db.models import Avg, Q -from django.http import HttpResponseBadRequest, HttpResponseNotFound,\ - JsonResponse +from django.http import HttpResponseNotFound, JsonResponse from django.core.exceptions import PermissionDenied from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse @@ -558,8 +557,7 @@ def book_page(request, book_id): prev_page = '/book/%s/?page=%d' % \ (book_id, reviews_page.previous_page_number()) - user_tags = [] - readthroughs = [] + user_tags = readthroughs = user_shelves = other_edition_shelves = [] if request.user.is_authenticated: user_tags = models.Tag.objects.filter( book=book, user=request.user @@ -570,6 +568,16 @@ def book_page(request, book_id): book=book, ).order_by('start_date') + user_shelves = models.ShelfBook.objects.filter( + added_by=request.user, book=book + ) + + other_edition_shelves = models.ShelfBook.objects.filter( + ~Q(book=book), + added_by=request.user, + book__parent_work=book.parent_work, + ) + rating = reviews.aggregate(Avg('rating')) tags = models.Tag.objects.filter( book=book @@ -585,6 +593,8 @@ def book_page(request, book_id): 'rating': rating['rating__avg'], 'tags': tags, 'user_tags': user_tags, + 'user_shelves': user_shelves, + 'other_edition_shelves': other_edition_shelves, 'readthroughs': readthroughs, 'path': '/book/%s' % book_id, 'info_fields': [ @@ -628,10 +638,9 @@ def editions_page(request, book_id): encoder=ActivityEncoder ) - editions = models.Edition.objects.filter(parent_work=work).all() data = { 'title': 'Editions of %s' % work.title, - 'editions': editions, + 'editions': work.edition_set.all(), 'work': work, } return TemplateResponse(request, 'editions.html', data) From 2d7f8ada6139a39a6962840908b5473f778b5e6e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 11 Dec 2020 16:57:38 -0800 Subject: [PATCH 081/356] Functional switch editions button --- bookwyrm/models/status.py | 2 +- bookwyrm/templates/book.html | 2 +- .../snippets/switch_edition_button.html | 8 +++--- bookwyrm/urls.py | 17 ++++++------ bookwyrm/view_actions.py | 27 +++++++++++++++++++ 5 files changed, 43 insertions(+), 13 deletions(-) diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 9d45379c..07e25119 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -283,7 +283,7 @@ class Boost(Status): class ReadThrough(BookWyrmModel): ''' Store progress through a book in the database. ''' user = models.ForeignKey('User', on_delete=models.PROTECT) - book = models.ForeignKey('Book', on_delete=models.PROTECT) + book = models.ForeignKey('Edition', on_delete=models.PROTECT) pages_read = models.IntegerField( null=True, blank=True) diff --git a/bookwyrm/templates/book.html b/bookwyrm/templates/book.html index eee22898..c7112016 100644 --- a/bookwyrm/templates/book.html +++ b/bookwyrm/templates/book.html @@ -103,7 +103,7 @@ {% for shelf in other_edition_shelves %}

A different edition of this book is on your {{ shelf.shelf.name }} shelf. - {% include 'snippets/switch_edition_button.html' with desired_edition=book %} + {% include 'snippets/switch_edition_button.html' with edition=book %}

{% endfor %} diff --git a/bookwyrm/templates/snippets/switch_edition_button.html b/bookwyrm/templates/snippets/switch_edition_button.html index 3fa09f4f..9771535a 100644 --- a/bookwyrm/templates/snippets/switch_edition_button.html +++ b/bookwyrm/templates/snippets/switch_edition_button.html @@ -1,3 +1,5 @@ -
- -
+
+ {% csrf_token %} + + +
diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index a9792038..01d6c56f 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -97,15 +97,16 @@ urlpatterns = [ re_path(r'^edit-profile/?$', actions.edit_profile), - re_path(r'^import-data/?', actions.import_data), - re_path(r'^retry-import/?', actions.retry_import), - re_path(r'^resolve-book/?', actions.resolve_book), - re_path(r'^edit-book/(?P\d+)/?', actions.edit_book), - re_path(r'^upload-cover/(?P\d+)/?', actions.upload_cover), - re_path(r'^add-description/(?P\d+)/?', actions.add_description), + re_path(r'^import-data/?$', actions.import_data), + re_path(r'^retry-import/?$', actions.retry_import), + re_path(r'^resolve-book/?$', actions.resolve_book), + re_path(r'^edit-book/(?P\d+)/?$', actions.edit_book), + re_path(r'^upload-cover/(?P\d+)/?$', actions.upload_cover), + re_path(r'^add-description/(?P\d+)/?$', actions.add_description), - re_path(r'^edit-readthrough/?', actions.edit_readthrough), - re_path(r'^delete-readthrough/?', actions.delete_readthrough), + re_path(r'^switch-edition/?$', actions.switch_edition), + re_path(r'^edit-readthrough/?$', actions.edit_readthrough), + re_path(r'^delete-readthrough/?$', actions.delete_readthrough), re_path(r'^rate/?$', actions.rate), re_path(r'^review/?$', actions.review), diff --git a/bookwyrm/view_actions.py b/bookwyrm/view_actions.py index bb862007..bd8baf73 100644 --- a/bookwyrm/view_actions.py +++ b/bookwyrm/view_actions.py @@ -10,6 +10,7 @@ from django.contrib.auth import authenticate, login, logout from django.contrib.auth.decorators import login_required, permission_required from django.core.exceptions import PermissionDenied from django.core.files.base import ContentFile +from django.db import transaction from django.http import HttpResponseBadRequest, HttpResponseNotFound from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse @@ -244,6 +245,32 @@ def edit_book(request, book_id): return redirect('/book/%s' % book.id) +@login_required +@require_POST +@transaction.atomic +def switch_edition(request): + ''' switch your copy of a book to a different edition ''' + edition_id = request.POST.get('edition') + new_edition = get_object_or_404(models.Edition, id=edition_id) + shelfbooks = models.ShelfBook.objects.filter( + book__parent_work=new_edition.parent_work, + added_by=request.user + ) + for shelfbook in shelfbooks.all(): + shelfbook.book = new_edition + shelfbook.save() + + readthroughs = models.ReadThrough.objects.filter( + book__parent_work=new_edition.parent_work, + user=request.user + ) + for readthrough in readthroughs.all(): + readthrough.book = new_edition + readthrough.save() + + return redirect('/book/%d' % new_edition.id) + + @login_required @require_POST def upload_cover(request, book_id): From 733e0e19acc24e0fc2b61b1f92cfd99afb3076c5 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 11 Dec 2020 17:39:17 -0800 Subject: [PATCH 082/356] Don't show boost and original status in timeline Fixes #381 --- bookwyrm/views.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bookwyrm/views.py b/bookwyrm/views.py index ab42557c..0fdfa1f9 100644 --- a/bookwyrm/views.py +++ b/bookwyrm/views.py @@ -131,7 +131,7 @@ def get_activity_feed(user, filter_level, model=models.Status): activities = model.objects activities = activities.filter( - deleted=False + deleted=False, ).order_by( '-published_date' ) @@ -160,6 +160,11 @@ def get_activity_feed(user, filter_level, model=models.Status): Q(user__in=following, privacy='followers') | Q(privacy='public') ) + try: + activities = activities.filter(~Q(boosters__in=activities)) + except ValueError: + pass + return activities From a176c6cd35c897d69ef418fb9438e0be7b832a7e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 09:38:05 -0800 Subject: [PATCH 083/356] Creates merge migration --- bookwyrm/migrations/0021_merge_20201212_1737.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 bookwyrm/migrations/0021_merge_20201212_1737.py diff --git a/bookwyrm/migrations/0021_merge_20201212_1737.py b/bookwyrm/migrations/0021_merge_20201212_1737.py new file mode 100644 index 00000000..4ccf8c8c --- /dev/null +++ b/bookwyrm/migrations/0021_merge_20201212_1737.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.7 on 2020-12-12 17:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0020_auto_20201208_0213'), + ('bookwyrm', '0016_auto_20201211_2026'), + ] + + operations = [ + ] From 5cf9e24ae5a2f23c434a723e262a83f2b62c90d2 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 09:43:07 -0800 Subject: [PATCH 084/356] Fixes name import in openlibrary --- bookwyrm/connectors/openlibrary.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 90bd7f28..28eb1ea0 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -65,6 +65,7 @@ class Connector(AbstractConnector): ] self.author_mappings = [ + Mapping('name'), Mapping('born', remote_field='birth_date', formatter=get_date), Mapping('died', remote_field='death_date', formatter=get_date), Mapping('bio', formatter=get_description), @@ -185,11 +186,6 @@ class Connector(AbstractConnector): author = models.Author(openlibrary_key=olkey) author = update_from_mappings(author, data, self.author_mappings) - name = data.get('name') - # TODO this is making some BOLD assumption - if name: - author.last_name = name.split(' ')[-1] - author.first_name = ' '.join(name.split(' ')[:-1]) author.save() return author From 31a407d74a3c5aec4aae8fa41c5132c6124be906 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 09:47:27 -0800 Subject: [PATCH 085/356] Use name field only for author name It feels janky to remove a more granular name designation, but all these first/last name fields were algorithmically populated by a dubious process of splitting the name by a space character. If it makes sense to have first/last name fields, it should be re-added with some consideration. --- bookwyrm/connectors/abstract_connector.py | 3 +- .../migrations/0022_auto_20201212_1744.py | 30 +++++++++++++++++++ bookwyrm/models/author.py | 12 -------- bookwyrm/templates/author.html | 4 +-- bookwyrm/templates/snippets/authors.html | 2 +- bookwyrm/templates/snippets/shelf.html | 2 +- bookwyrm/tests/activitypub/test_author.py | 2 -- 7 files changed, 35 insertions(+), 20 deletions(-) create mode 100644 bookwyrm/migrations/0022_auto_20201212_1744.py diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index a696232d..c9f1ad2e 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -178,8 +178,7 @@ class AbstractConnector(AbstractMinimalConnector): author_text = [] for author in self.get_authors_from_data(data): book.authors.add(author) - if author.display_name: - author_text.append(author.display_name) + author_text.append(author.name) book.author_text = ', '.join(author_text) book.save() diff --git a/bookwyrm/migrations/0022_auto_20201212_1744.py b/bookwyrm/migrations/0022_auto_20201212_1744.py new file mode 100644 index 00000000..0a98597f --- /dev/null +++ b/bookwyrm/migrations/0022_auto_20201212_1744.py @@ -0,0 +1,30 @@ +# Generated by Django 3.0.7 on 2020-12-12 17:44 + +from django.db import migrations + + +def set_author_name(app_registry, schema_editor): + db_alias = schema_editor.connection.alias + authors = app_registry.get_model('bookwyrm', 'Author') + for author in authors.objects.using(db_alias): + if not author.name: + author.name = '%s %s' % (author.first_name, author.last_name) + author.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0021_merge_20201212_1737'), + ] + + operations = [ + migrations.RunPython(set_author_name), + migrations.RemoveField( + model_name='author', + name='first_name', + ), + migrations.RemoveField( + model_name='author', + name='last_name', + ), + ] diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 5b70b57f..6098447f 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -21,8 +21,6 @@ class Author(ActivitypubMixin, BookWyrmModel): born = fields.DateTimeField(blank=True, null=True) died = fields.DateTimeField(blank=True, null=True) name = fields.CharField(max_length=255) - last_name = models.CharField(max_length=255, blank=True, null=True) - first_name = models.CharField(max_length=255, blank=True, null=True) aliases = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) @@ -42,14 +40,4 @@ class Author(ActivitypubMixin, BookWyrmModel): ''' editions and works both use "book" instead of model_name ''' return 'https://%s/author/%s' % (DOMAIN, self.id) - @property - def display_name(self): - ''' Helper to return a displayable name''' - if self.name: - return self.name - # don't want to return a spurious space if all of these are None - if self.first_name and self.last_name: - return self.first_name + ' ' + self.last_name - return self.last_name or self.first_name - activity_serializer = activitypub.Author diff --git a/bookwyrm/templates/author.html b/bookwyrm/templates/author.html index fb4970e4..3e3e0018 100644 --- a/bookwyrm/templates/author.html +++ b/bookwyrm/templates/author.html @@ -2,7 +2,7 @@ {% load fr_display %} {% block content %}
-

{{ author.display_name }}

+

{{ author.name }}

{% if author.bio %}

@@ -12,7 +12,7 @@

-

Books by {{ author.display_name }}

+

Books by {{ author.name }}

{% include 'snippets/book_tiles.html' with books=books %}
{% endblock %} diff --git a/bookwyrm/templates/snippets/authors.html b/bookwyrm/templates/snippets/authors.html index 165c4cda..e8106f5d 100644 --- a/bookwyrm/templates/snippets/authors.html +++ b/bookwyrm/templates/snippets/authors.html @@ -1 +1 @@ -{{ book.authors.first.display_name }} +{{ book.authors.first.name }} diff --git a/bookwyrm/templates/snippets/shelf.html b/bookwyrm/templates/snippets/shelf.html index 2df8b024..4e41cd30 100644 --- a/bookwyrm/templates/snippets/shelf.html +++ b/bookwyrm/templates/snippets/shelf.html @@ -43,7 +43,7 @@ {{ book.title }} - {{ book.authors.first.display_name }} + {{ book.authors.first.name }} {% if book.first_published_date %}{{ book.first_published_date }}{% endif %} diff --git a/bookwyrm/tests/activitypub/test_author.py b/bookwyrm/tests/activitypub/test_author.py index dd2e93af..fd31f105 100644 --- a/bookwyrm/tests/activitypub/test_author.py +++ b/bookwyrm/tests/activitypub/test_author.py @@ -12,8 +12,6 @@ class Author(TestCase): ) self.author = models.Author.objects.create( name='Author fullname', - first_name='Auth', - last_name='Or', aliases=['One', 'Two'], bio='bio bio bio', ) From 7c43fa1f7c316b72f788dc7758aa6b69493e827a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 13:39:55 -0800 Subject: [PATCH 086/356] Adds deduplication fields --- bookwyrm/activitypub/base_activity.py | 46 +++++++++--------- bookwyrm/context_processors.py | 2 +- bookwyrm/models/author.py | 6 +-- bookwyrm/models/base_model.py | 47 +++++++++++++++++++ bookwyrm/models/book.py | 24 ++++++---- bookwyrm/models/fields.py | 6 ++- bookwyrm/models/user.py | 4 +- .../tests/activitypub/test_base_activity.py | 30 +----------- bookwyrm/tests/models/test_base_model.py | 40 +++++++++++++++- bookwyrm/tests/models/test_fields.py | 42 +++++++++++++++-- 10 files changed, 174 insertions(+), 73 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index c784c3c1..585f2449 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -78,11 +78,11 @@ class ActivityObject: # check for an existing instance, if we're not updating a known obj if not instance: - instance = find_existing_by_remote_id(model, self.id) or model() - # TODO: deduplicate books by identifiers + instance = model.find_existing(self.serialize()) or model() many_to_many_fields = {} for field in model._meta.get_fields(): + # check if it's an activitypub field if not hasattr(field, 'field_to_activity'): continue # call the formatter associated with the model field class @@ -167,34 +167,25 @@ def set_related_field( if isinstance(data, str): item = resolve_remote_id(model, data, save=False) else: - item = model.activity_serializer(**data) - item = item.to_model(model, save=False) - instance = find_existing_by_remote_id(origin_model, related_remote_id) + # look for a match based on all the available data + item = model.find_existing(data) + if not item: + # create a new model instance + item = model.activity_serializer(**data) + item = item.to_model(model, save=False) + # this must exist because it's the object that triggered this function + instance = origin_model.find_existing_by_remote_id(related_remote_id) + if not instance: + raise ValueError('Invalid related remote id: %s' % related_remote_id) + + # edition.parent_work = instance, for example setattr(item, related_field_name, instance) item.save() -def find_existing_by_remote_id(model, remote_id): - ''' check for an existing instance of this id in the db ''' - objects = model.objects - if hasattr(model.objects, 'select_subclasses'): - objects = objects.select_subclasses() - - # first, check for an existing copy in the database - result = objects.filter( - remote_id=remote_id - ).first() - - if not result and hasattr(model, 'origin_id'): - result = objects.filter( - origin_id=remote_id - ).first() - return result - - def resolve_remote_id(model, remote_id, refresh=False, save=True): - ''' look up the remote_id in the database or load it remotely ''' - result = find_existing_by_remote_id(model, remote_id) + ''' take a remote_id and return an instance, creating if necessary ''' + result = model.find_existing_by_remote_id(remote_id) if result and not refresh: return result @@ -206,6 +197,11 @@ def resolve_remote_id(model, remote_id, refresh=False, save=True): 'Could not connect to host for remote_id in %s model: %s' % \ (model.__name__, remote_id)) + # check for existing items with shared unique identifiers + item = model.find_existing(data) + if item: + return item + item = model.activity_serializer(**data) # if we're refreshing, "result" will be set and we'll update it return item.to_model(model, instance=result, save=save) diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py index 72839dce..8422f3c3 100644 --- a/bookwyrm/context_processors.py +++ b/bookwyrm/context_processors.py @@ -4,5 +4,5 @@ from bookwyrm import models def site_settings(request): ''' include the custom info about the site ''' return { - 'site': models.SiteSettings.objects.get() + 'site': models.SiteSettings.objects.first() } diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 6098447f..79973a37 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -12,11 +12,11 @@ from . import fields class Author(ActivitypubMixin, BookWyrmModel): ''' basic biographic info ''' origin_id = models.CharField(max_length=255, null=True) - ''' copy of an author from OL ''' - openlibrary_key = fields.CharField(max_length=255, blank=True, null=True) + openlibrary_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) sync = models.BooleanField(default=True) last_sync_date = models.DateTimeField(default=timezone.now) - wikipedia_link = fields.CharField(max_length=255, blank=True, null=True) + wikipedia_link = fields.CharField(max_length=255, blank=True, null=True, deduplication_field=True) # idk probably other keys would be useful here? born = fields.DateTimeField(blank=True, null=True) died = fields.DateTimeField(blank=True, null=True) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 1e437152..f44797ab 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -1,5 +1,7 @@ ''' base model with default fields ''' from base64 import b64encode +from functools import reduce +import operator from uuid import uuid4 from Crypto.PublicKey import RSA @@ -7,6 +9,7 @@ from Crypto.Signature import pkcs1_15 from Crypto.Hash import SHA256 from django.core.paginator import Paginator from django.db import models +from django.db.models import Q from django.dispatch import receiver from bookwyrm import activitypub @@ -64,6 +67,50 @@ class ActivitypubMixin: activity_serializer = lambda: {} reverse_unfurl = False + @classmethod + def find_existing_by_remote_id(cls, remote_id): + ''' look up a remote id in the db ''' + return cls.find_existing({'id': remote_id}) + + @classmethod + def find_existing(cls, data): + ''' compare data to fields that can be used for deduplation. + This always includes remote_id, but can also be unique identifiers + like an isbn for an edition ''' + filters = [] + for field in cls._meta.get_fields(): + if not hasattr(field, 'deduplication_field') or \ + not field.deduplication_field: + continue + + value = data.get(field.activitypub_field) + if not value: + continue + filters.append({field.name: value}) + + if hasattr(cls, 'origin_id') and 'id' in data: + # kinda janky, but this handles special case for books + filters.append({'origin_id': data['id']}) + + if not filters: + # if there are no deduplication fields, it will match the first + # item no matter what. this shouldn't happen but just in case. + return None + + objects = cls.objects + if hasattr(objects, 'select_subclasses'): + objects = objects.select_subclasses() + + # an OR operation on all the match fields + match = objects.filter( + reduce( + operator.or_, (Q(**f) for f in filters) + ) + ) + # there OUGHT to be only one match + return match.first() + + def to_activity(self): ''' convert from a model to an activity ''' activity = {} diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 47b8b99e..bcd4bc04 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -16,9 +16,12 @@ class Book(ActivitypubMixin, BookWyrmModel): ''' a generic book, which can mean either an edition or a work ''' origin_id = models.CharField(max_length=255, null=True, blank=True) # these identifiers apply to both works and editions - openlibrary_key = fields.CharField(max_length=255, blank=True, null=True) - librarything_key = fields.CharField(max_length=255, blank=True, null=True) - goodreads_key = fields.CharField(max_length=255, blank=True, null=True) + openlibrary_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + librarything_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + goodreads_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) # info about where the data comes from and where/if to sync sync = models.BooleanField(default=True) @@ -83,7 +86,8 @@ class Book(ActivitypubMixin, BookWyrmModel): class Work(OrderedCollectionPageMixin, Book): ''' a work (an abstract concept of a book that manifests in an edition) ''' # library of congress catalog control number - lccn = fields.CharField(max_length=255, blank=True, null=True) + lccn = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) # this has to be nullable but should never be null default_edition = fields.ForeignKey( 'Edition', @@ -103,10 +107,14 @@ class Work(OrderedCollectionPageMixin, Book): class Edition(Book): ''' an edition of a book ''' # these identifiers only apply to editions, not works - isbn_10 = fields.CharField(max_length=255, blank=True, null=True) - isbn_13 = fields.CharField(max_length=255, blank=True, null=True) - oclc_number = fields.CharField(max_length=255, blank=True, null=True) - asin = fields.CharField(max_length=255, blank=True, null=True) + isbn_10 = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + isbn_13 = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + oclc_number = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + asin = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) pages = fields.IntegerField(blank=True, null=True) physical_format = fields.CharField(max_length=255, blank=True, null=True) publishers = fields.ArrayField( diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index fb94e1a6..e6878fb9 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -28,7 +28,9 @@ def validate_remote_id(value): class ActivitypubFieldMixin: ''' make a database field serializable ''' def __init__(self, *args, \ - activitypub_field=None, activitypub_wrapper=None, **kwargs): + activitypub_field=None, activitypub_wrapper=None, + deduplication_field=False, **kwargs): + self.deduplication_field = deduplication_field if activitypub_wrapper: self.activitypub_wrapper = activitypub_field self.activitypub_field = activitypub_wrapper @@ -86,6 +88,8 @@ class RemoteIdField(ActivitypubFieldMixin, models.CharField): *args, max_length=max_length, validators=validators, **kwargs ) + # for this field, the default is true. false everywhere else. + self.deduplication_field = kwargs.get('deduplication_field', True) class UsernameField(ActivitypubFieldMixin, models.CharField): diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 531b0da2..0ef9a2e6 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -32,7 +32,9 @@ class User(OrderedCollectionPageMixin, AbstractUser): inbox = fields.RemoteIdField(unique=True) shared_inbox = fields.RemoteIdField( activitypub_field='sharedInbox', - activitypub_wrapper='endpoints', null=True) + activitypub_wrapper='endpoints', + deduplication_field=False, + null=True) federated_server = models.ForeignKey( 'FederatedServer', on_delete=models.PROTECT, diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py index 26d63b61..100ff6cf 100644 --- a/bookwyrm/tests/activitypub/test_base_activity.py +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -11,7 +11,7 @@ import responses from bookwyrm import activitypub from bookwyrm.activitypub.base_activity import ActivityObject, \ - find_existing_by_remote_id, resolve_remote_id, set_related_field + resolve_remote_id, set_related_field from bookwyrm.activitypub import ActivitySerializerError from bookwyrm import models @@ -77,34 +77,6 @@ class BaseActivity(TestCase): self.assertEqual(serialized['id'], 'a') self.assertEqual(serialized['type'], 'b') - def test_find_existing_by_remote_id(self): - ''' attempt to match a remote id to an object in the db ''' - # uses a different remote id scheme - # this isn't really part of this test directly but it's helpful to state - self.assertEqual(self.book.origin_id, 'http://book.com/book') - self.assertNotEqual(self.book.remote_id, 'http://book.com/book') - - # uses subclasses - models.Comment.objects.create( - user=self.user, content='test status', book=self.book, \ - remote_id='https://comment.net') - - result = find_existing_by_remote_id(models.User, 'hi') - self.assertIsNone(result) - - result = find_existing_by_remote_id( - models.User, 'http://example.com/a/b') - self.assertEqual(result, self.user) - - # test using origin id - result = find_existing_by_remote_id( - models.Edition, 'http://book.com/book') - self.assertEqual(result, self.book) - - # test subclass match - result = find_existing_by_remote_id( - models.Status, 'https://comment.net') - @responses.activate def test_resolve_remote_id(self): ''' look up or load remote data ''' diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py index a807c47f..966fe3e2 100644 --- a/bookwyrm/tests/models/test_base_model.py +++ b/bookwyrm/tests/models/test_base_model.py @@ -11,13 +11,16 @@ from bookwyrm.models.base_model import ActivitypubMixin from bookwyrm.settings import DOMAIN class BaseModel(TestCase): + ''' functionality shared across models ''' def test_remote_id(self): + ''' these should be generated ''' instance = base_model.BookWyrmModel() instance.id = 1 expected = instance.get_remote_id() self.assertEqual(expected, 'https://%s/bookwyrmmodel/1' % DOMAIN) def test_remote_id_with_user(self): + ''' format of remote id when there's a user object ''' user = models.User.objects.create_user( 'mouse', 'mouse@mouse.com', 'mouseword', local=True) instance = base_model.BookWyrmModel() @@ -46,6 +49,7 @@ class BaseModel(TestCase): self.assertIsNone(instance.remote_id) def test_to_create_activity(self): + ''' wrapper for ActivityPub "create" action ''' user = models.User.objects.create_user( 'mouse', 'mouse@mouse.com', 'mouseword', local=True) @@ -75,6 +79,7 @@ class BaseModel(TestCase): ) def test_to_delete_activity(self): + ''' wrapper for Delete activity ''' user = models.User.objects.create_user( 'mouse', 'mouse@mouse.com', 'mouseword', local=True) @@ -98,6 +103,7 @@ class BaseModel(TestCase): ['https://www.w3.org/ns/activitystreams#Public']) def test_to_update_activity(self): + ''' ditto above but for Update ''' user = models.User.objects.create_user( 'mouse', 'mouse@mouse.com', 'mouseword', local=True) @@ -121,6 +127,7 @@ class BaseModel(TestCase): self.assertEqual(activity['object'], {}) def test_to_undo_activity(self): + ''' and again, for Undo ''' user = models.User.objects.create_user( 'mouse', 'mouse@mouse.com', 'mouseword', local=True) @@ -140,12 +147,14 @@ class BaseModel(TestCase): def test_to_activity(self): + ''' model to ActivityPub json ''' @dataclass(init=False) class TestActivity(ActivityObject): + ''' real simple mock ''' type: str = 'Test' class TestModel(ActivitypubMixin, base_model.BookWyrmModel): - pass + ''' real simple mock model because BookWyrmModel is abstract ''' instance = TestModel() instance.remote_id = 'https://www.example.com/test' @@ -155,3 +164,32 @@ class BaseModel(TestCase): self.assertIsInstance(activity, dict) self.assertEqual(activity['id'], 'https://www.example.com/test') self.assertEqual(activity['type'], 'Test') + + + def test_find_existing_by_remote_id(self): + ''' attempt to match a remote id to an object in the db ''' + # uses a different remote id scheme + # this isn't really part of this test directly but it's helpful to state + self.assertEqual(self.book.origin_id, 'http://book.com/book') + self.assertNotEqual(self.book.remote_id, 'http://book.com/book') + + # uses subclasses + models.Comment.objects.create( + user=self.user, content='test status', book=self.book, \ + remote_id='https://comment.net') + + result = models.User.find_existing_by_remote_id('hi') + self.assertIsNone(result) + + result = models.User.find_existing_by_remote_id( + 'http://example.com/a/b') + self.assertEqual(result, self.user) + + # test using origin id + result = models.Edition.find_existing_by_remote_id( + 'http://book.com/book') + self.assertEqual(result, self.book) + + # test subclass match + result = models.Status.find_existing_by_remote_id( + 'https://comment.net') diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index 58d9e08c..a0a95ba9 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -48,6 +48,7 @@ class ActivitypubFields(TestCase): instance = fields.ActivitypubFieldMixin() self.assertEqual(instance.field_to_activity('fish'), 'fish') self.assertEqual(instance.field_from_activity('fish'), 'fish') + self.assertFalse(instance.deduplication_field) instance = fields.ActivitypubFieldMixin( activitypub_wrapper='endpoints', activitypub_field='outbox' @@ -70,6 +71,7 @@ class ActivitypubFields(TestCase): ''' just sets some defaults on charfield ''' instance = fields.RemoteIdField() self.assertEqual(instance.max_length, 255) + self.assertTrue(instance.deduplication_field) with self.assertRaises(ValidationError): instance.run_validators('http://www.example.com/dlfjg 23/x') @@ -97,7 +99,7 @@ class ActivitypubFields(TestCase): self.assertEqual(instance.field_to_activity(item), 'https://e.b/c') @responses.activate - def test_foreign_key_from_activity(self): + def test_foreign_key_from_activity_str(self): ''' this is the important stuff ''' instance = fields.ForeignKey(User, on_delete=models.CASCADE) @@ -117,15 +119,47 @@ class ActivitypubFields(TestCase): with patch('bookwyrm.models.user.set_remote_server.delay'): value = instance.field_from_activity( 'https://example.com/user/mouse') + self.assertIsInstance(value, User) + self.assertEqual(value.remote_id, 'https://example.com/user/mouse') + self.assertEqual(value.name, 'MOUSE?? MOUSE!!') - # test recieving activity json - value = instance.field_from_activity(userdata) + + def test_foreign_key_from_activity_dict(self): + ''' test recieving activity json ''' + instance = fields.ForeignKey(User, on_delete=models.CASCADE) + + datafile = pathlib.Path(__file__).parent.joinpath( + '../data/ap_user.json' + ) + userdata = json.loads(datafile.read_bytes()) + # don't try to load the user icon + del userdata['icon'] + with patch('bookwyrm.models.user.set_remote_server.delay'): + value = instance.field_from_activity(userdata) self.assertIsInstance(value, User) self.assertEqual(value.remote_id, 'https://example.com/user/mouse') self.assertEqual(value.name, 'MOUSE?? MOUSE!!') # et cetera but we're not testing serializing user json - # test receiving a remote id of an object in the db + + def test_foreign_key_from_activity_dict_existing(self): + ''' test receiving a dict of an existing object in the db ''' + instance = fields.ForeignKey(User, on_delete=models.CASCADE) + datafile = pathlib.Path(__file__).parent.joinpath( + '../data/ap_user.json' + ) + userdata = json.loads(datafile.read_bytes()) + user = User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) + user.remote_id = 'https://example.com/user/mouse' + user.save() + value = instance.field_from_activity(userdata) + self.assertEqual(value, user) + + + def test_foreign_key_from_activity_str_existing(self): + ''' test receiving a remote id of an existing object in the db ''' + instance = fields.ForeignKey(User, on_delete=models.CASCADE) user = User.objects.create_user( 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) value = instance.field_from_activity(user.remote_id) From 804066c523de59c535aa22e09c7d4ac6465e1ca9 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 13:47:51 -0800 Subject: [PATCH 087/356] a couple more assertions for testing fk field --- bookwyrm/tests/models/test_fields.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index a0a95ba9..a1e4ff71 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -100,16 +100,18 @@ class ActivitypubFields(TestCase): @responses.activate def test_foreign_key_from_activity_str(self): - ''' this is the important stuff ''' + ''' create a new object from a foreign key ''' instance = fields.ForeignKey(User, on_delete=models.CASCADE) - datafile = pathlib.Path(__file__).parent.joinpath( - '../data/ap_user.json' - ) + '../data/ap_user.json') userdata = json.loads(datafile.read_bytes()) # don't try to load the user icon del userdata['icon'] + # it shouldn't match with this unrelated user: + unrelated_user = User.objects.create_user( + 'rat', 'rat@rat.rat', 'ratword', local=True) + # test receiving an unknown remote id and loading data responses.add( responses.GET, @@ -120,6 +122,7 @@ class ActivitypubFields(TestCase): value = instance.field_from_activity( 'https://example.com/user/mouse') self.assertIsInstance(value, User) + self.assertNotEqual(value, unrelated_user) self.assertEqual(value.remote_id, 'https://example.com/user/mouse') self.assertEqual(value.name, 'MOUSE?? MOUSE!!') @@ -127,16 +130,19 @@ class ActivitypubFields(TestCase): def test_foreign_key_from_activity_dict(self): ''' test recieving activity json ''' instance = fields.ForeignKey(User, on_delete=models.CASCADE) - datafile = pathlib.Path(__file__).parent.joinpath( - '../data/ap_user.json' - ) + '../data/ap_user.json') userdata = json.loads(datafile.read_bytes()) # don't try to load the user icon del userdata['icon'] + + # it shouldn't match with this unrelated user: + unrelated_user = User.objects.create_user( + 'rat', 'rat@rat.rat', 'ratword', local=True) with patch('bookwyrm.models.user.set_remote_server.delay'): value = instance.field_from_activity(userdata) self.assertIsInstance(value, User) + self.assertNotEqual(value, unrelated_user) self.assertEqual(value.remote_id, 'https://example.com/user/mouse') self.assertEqual(value.name, 'MOUSE?? MOUSE!!') # et cetera but we're not testing serializing user json @@ -153,6 +159,9 @@ class ActivitypubFields(TestCase): 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) user.remote_id = 'https://example.com/user/mouse' user.save() + User.objects.create_user( + 'rat', 'rat@rat.rat', 'ratword', local=True) + value = instance.field_from_activity(userdata) self.assertEqual(value, user) @@ -162,6 +171,9 @@ class ActivitypubFields(TestCase): instance = fields.ForeignKey(User, on_delete=models.CASCADE) user = User.objects.create_user( 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) + User.objects.create_user( + 'rat', 'rat@rat.rat', 'ratword', local=True) + value = instance.field_from_activity(user.remote_id) self.assertEqual(value, user) From 4ed713662eca97d9a5ab50b9386c283fc4d188ac Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 13:57:53 -0800 Subject: [PATCH 088/356] Fixes skipping refresh on matched object --- bookwyrm/activitypub/base_activity.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 585f2449..b77e9e77 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -198,9 +198,10 @@ def resolve_remote_id(model, remote_id, refresh=False, save=True): (model.__name__, remote_id)) # check for existing items with shared unique identifiers - item = model.find_existing(data) - if item: - return item + if not result: + result = model.find_existing(data) + if result and not refresh: + return result item = model.activity_serializer(**data) # if we're refreshing, "result" will be set and we'll update it From e7f400533e46827ddfc53776a4e30ed26a60b232 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 14:15:10 -0800 Subject: [PATCH 089/356] Fixes missing book and user objects --- .../tests/activitypub/test_base_activity.py | 7 +++---- bookwyrm/tests/models/test_base_model.py | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py index 100ff6cf..88997c44 100644 --- a/bookwyrm/tests/activitypub/test_base_activity.py +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -31,9 +31,6 @@ class BaseActivity(TestCase): # don't try to load the user icon del self.userdata['icon'] - self.book = models.Edition.objects.create( - title='Test Edition', remote_id='http://book.com/book') - image_file = pathlib.Path(__file__).parent.joinpath( '../../static/images/default_avi.jpg') image = Image.open(image_file) @@ -146,6 +143,8 @@ class BaseActivity(TestCase): content='test status', user=self.user, ) + book = models.Edition.objects.create( + title='Test Edition', remote_id='http://book.com/book') update_data = activitypub.Note(**status.to_activity()) update_data.tag = [ { @@ -161,7 +160,7 @@ class BaseActivity(TestCase): ] update_data.to_model(models.Status, instance=status) self.assertEqual(status.mention_users.first(), self.user) - self.assertEqual(status.mention_books.first(), self.book) + self.assertEqual(status.mention_books.first(), book) @responses.activate diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py index 966fe3e2..65cf892e 100644 --- a/bookwyrm/tests/models/test_base_model.py +++ b/bookwyrm/tests/models/test_base_model.py @@ -170,12 +170,19 @@ class BaseModel(TestCase): ''' attempt to match a remote id to an object in the db ''' # uses a different remote id scheme # this isn't really part of this test directly but it's helpful to state - self.assertEqual(self.book.origin_id, 'http://book.com/book') - self.assertNotEqual(self.book.remote_id, 'http://book.com/book') + book = models.Edition.objects.create( + title='Test Edition', remote_id='http://book.com/book') + user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) + user.remote_id = 'http://example.com/a/b' + user.save() + + self.assertEqual(book.origin_id, 'http://book.com/book') + self.assertNotEqual(book.remote_id, 'http://book.com/book') # uses subclasses models.Comment.objects.create( - user=self.user, content='test status', book=self.book, \ + user=user, content='test status', book=book, \ remote_id='https://comment.net') result = models.User.find_existing_by_remote_id('hi') @@ -183,12 +190,12 @@ class BaseModel(TestCase): result = models.User.find_existing_by_remote_id( 'http://example.com/a/b') - self.assertEqual(result, self.user) + self.assertEqual(result, user) # test using origin id result = models.Edition.find_existing_by_remote_id( 'http://book.com/book') - self.assertEqual(result, self.book) + self.assertEqual(result, book) # test subclass match result = models.Status.find_existing_by_remote_id( From eb28708230d375f527c15f47375806c9b44fc13a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 14:33:51 -0800 Subject: [PATCH 090/356] Reverts site settings to correct state this was just changed to debug tests --- bookwyrm/context_processors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py index 8422f3c3..72839dce 100644 --- a/bookwyrm/context_processors.py +++ b/bookwyrm/context_processors.py @@ -4,5 +4,5 @@ from bookwyrm import models def site_settings(request): ''' include the custom info about the site ''' return { - 'site': models.SiteSettings.objects.first() + 'site': models.SiteSettings.objects.get() } From 49979fabef7390ad6344c3ce164496af5cecf303 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 15:00:20 -0800 Subject: [PATCH 091/356] More user serialization tests --- bookwyrm/models/user.py | 1 + bookwyrm/tests/activitypub/test_person.py | 13 ++++++++++++- bookwyrm/tests/models/test_user_model.py | 8 ++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 0ef9a2e6..63549d36 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -190,6 +190,7 @@ class KeyPair(ActivitypubMixin, BookWyrmModel): @receiver(models.signals.post_save, sender=User) +#pylint: disable=unused-argument def execute_after_save(sender, instance, created, *args, **kwargs): ''' create shelves for new users ''' if not created: diff --git a/bookwyrm/tests/activitypub/test_person.py b/bookwyrm/tests/activitypub/test_person.py index 3e0d74e0..c7a8221c 100644 --- a/bookwyrm/tests/activitypub/test_person.py +++ b/bookwyrm/tests/activitypub/test_person.py @@ -1,8 +1,10 @@ +# pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring import json import pathlib +from unittest.mock import patch from django.test import TestCase -from bookwyrm import activitypub +from bookwyrm import activitypub, models class Person(TestCase): @@ -18,3 +20,12 @@ class Person(TestCase): self.assertEqual(activity.id, 'https://example.com/user/mouse') self.assertEqual(activity.preferredUsername, 'mouse') self.assertEqual(activity.type, 'Person') + + + def test_user_to_model(self): + activity = activitypub.Person(**self.user_data) + with patch('bookwyrm.models.user.set_remote_server.delay'): + user = activity.to_model(models.User) + self.assertEqual(user.username, 'mouse@example.com') + self.assertEqual(user.remote_id, 'https://example.com/user/mouse') + self.assertFalse(user.local) diff --git a/bookwyrm/tests/models/test_user_model.py b/bookwyrm/tests/models/test_user_model.py index dabf760c..0454fb40 100644 --- a/bookwyrm/tests/models/test_user_model.py +++ b/bookwyrm/tests/models/test_user_model.py @@ -1,4 +1,5 @@ ''' testing models ''' +from unittest.mock import patch from django.test import TestCase from bookwyrm import models @@ -22,6 +23,13 @@ class User(TestCase): self.assertIsNotNone(self.user.key_pair.private_key) self.assertIsNotNone(self.user.key_pair.public_key) + def test_remote_user(self): + with patch('bookwyrm.models.user.set_remote_server.delay'): + user = models.User.objects.create_user( + 'rat', 'rat@rat.rat', 'ratword', local=False, + remote_id='https://example.com/dfjkg') + self.assertEqual(user.username, 'rat@example.com') + def test_user_shelves(self): shelves = models.Shelf.objects.filter(user=self.user).all() From cb28c19abc36636ff0ecadaeebff0cbaf5a3549c Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 15:19:57 -0800 Subject: [PATCH 092/356] Use get_data in resolving remote id --- bookwyrm/outgoing.py | 12 +--- .../tests/outgoing/test_remote_webfinger.py | 61 +++++++++++++++++++ bookwyrm/utils/regex.py | 2 +- 3 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 bookwyrm/tests/outgoing/test_remote_webfinger.py diff --git a/bookwyrm/outgoing.py b/bookwyrm/outgoing.py index 545ac491..2f94d2c0 100644 --- a/bookwyrm/outgoing.py +++ b/bookwyrm/outgoing.py @@ -4,10 +4,10 @@ import re from django.db import IntegrityError, transaction from django.http import HttpResponseNotFound, JsonResponse from django.views.decorators.csrf import csrf_exempt -import requests from bookwyrm import activitypub from bookwyrm import models +from bookwyrm.connectors import get_data from bookwyrm.broadcast import broadcast from bookwyrm.status import create_notification from bookwyrm.status import create_generated_note @@ -52,14 +52,8 @@ def handle_remote_webfinger(query): except models.User.DoesNotExist: url = 'https://%s/.well-known/webfinger?resource=acct:%s' % \ (domain, query) - try: - response = requests.get(url) - except requests.exceptions.ConnectionError: - return None - if not response.ok: - return None - data = response.json() - for link in data['links']: + data = get_data(url) + for link in data.get('links'): if link.get('rel') == 'self': try: user = activitypub.resolve_remote_id( diff --git a/bookwyrm/tests/outgoing/test_remote_webfinger.py b/bookwyrm/tests/outgoing/test_remote_webfinger.py new file mode 100644 index 00000000..1bf884a6 --- /dev/null +++ b/bookwyrm/tests/outgoing/test_remote_webfinger.py @@ -0,0 +1,61 @@ +''' testing user lookup ''' +import json +import pathlib +from unittest.mock import patch + +from django.test import TestCase +import responses + +from bookwyrm import models, outgoing +from bookwyrm.settings import DOMAIN + +class TestOutgoingRemoteWebfinger(TestCase): + ''' overwrites standard model feilds to work with activitypub ''' + def setUp(self): + ''' get user data ready ''' + datafile = pathlib.Path(__file__).parent.joinpath( + '../data/ap_user.json' + ) + self.userdata = json.loads(datafile.read_bytes()) + del self.userdata['icon'] + + def test_existing_user(self): + ''' simple database lookup by username ''' + user = models.User.objects.create_user( + 'mouse', 'mouse@mouse.mouse', 'mouseword', local=True) + + result = outgoing.handle_remote_webfinger('@mouse@%s' % DOMAIN) + self.assertEqual(result, user) + + result = outgoing.handle_remote_webfinger('mouse@%s' % DOMAIN) + self.assertEqual(result, user) + + + @responses.activate + def test_load_user(self): + username = 'mouse@example.com' + wellknown = { + "subject": "acct:mouse@example.com", + "links": [ + { + "rel": "self", + "type": "application/activity+json", + "href": "https://example.com/user/mouse" + } + ] + } + responses.add( + responses.GET, + 'https://example.com/.well-known/webfinger?resource=acct:%s' \ + % username, + json=wellknown, + status=200) + responses.add( + responses.GET, + 'https://example.com/user/mouse', + json=self.userdata, + status=200) + with patch('bookwyrm.models.user.set_remote_server.delay'): + result = outgoing.handle_remote_webfinger('@mouse@example.com') + self.assertIsInstance(result, models.User) + self.assertEqual(result.username, 'mouse@example.com') diff --git a/bookwyrm/utils/regex.py b/bookwyrm/utils/regex.py index 36e211d9..70a43b84 100644 --- a/bookwyrm/utils/regex.py +++ b/bookwyrm/utils/regex.py @@ -2,4 +2,4 @@ domain = r'[a-z-A-Z0-9_\-]+\.[a-z]+' username = r'@[a-zA-Z_\-\.0-9]+(@%s)?' % domain -full_username = r'@[a-zA-Z_\-\.0-9]+@%s' % domain +full_username = r'@?[a-zA-Z_\-\.0-9]+@%s' % domain From 37aaaa97b20eff2cf97696b612d8a0d80cc7568b Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 15:33:07 -0800 Subject: [PATCH 093/356] Catch http erros for remote_id --- bookwyrm/outgoing.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bookwyrm/outgoing.py b/bookwyrm/outgoing.py index 2f94d2c0..38b48282 100644 --- a/bookwyrm/outgoing.py +++ b/bookwyrm/outgoing.py @@ -4,10 +4,11 @@ import re from django.db import IntegrityError, transaction from django.http import HttpResponseNotFound, JsonResponse from django.views.decorators.csrf import csrf_exempt +from requests import HTTPError from bookwyrm import activitypub from bookwyrm import models -from bookwyrm.connectors import get_data +from bookwyrm.connectors import get_data, ConnectorException from bookwyrm.broadcast import broadcast from bookwyrm.status import create_notification from bookwyrm.status import create_generated_note @@ -52,7 +53,11 @@ def handle_remote_webfinger(query): except models.User.DoesNotExist: url = 'https://%s/.well-known/webfinger?resource=acct:%s' % \ (domain, query) - data = get_data(url) + try: + data = get_data(url) + except (ConnectorException, HTTPError): + return None + for link in data.get('links'): if link.get('rel') == 'self': try: From e58ef83f20c17a62859a81228531e1564d5f4134 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 15:44:17 -0800 Subject: [PATCH 094/356] Fixes image fields breaking user import --- bookwyrm/activitypub/base_activity.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index b77e9e77..ed19af99 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -81,6 +81,7 @@ class ActivityObject: instance = model.find_existing(self.serialize()) or model() many_to_many_fields = {} + image_fields = {} for field in model._meta.get_fields(): # check if it's an activitypub field if not hasattr(field, 'field_to_activity'): @@ -99,11 +100,15 @@ class ActivityObject: many_to_many_fields[field.name] = value elif isinstance(model_field, ImageFileDescriptor): # image fields need custom handling - getattr(instance, field.name).save(*value, save=save) + image_fields[field.name] = value else: # just a good old fashioned model.field = value setattr(instance, field.name, value) + # if this isn't here, it messes up saving users. who even knows. + for (model_key, value) in image_fields.items(): + getattr(instance, model_key).save(*value, save=save) + if not save: # we can't set many to many and reverse fields on an unsaved object return instance From 9b7f0366e786e79f70cdfd53165a21f80c53fa1c Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 16:01:43 -0800 Subject: [PATCH 095/356] Adds site settings to initdb --- bookwyrm/management/commands/initdb.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py index f29ed102..9fd11787 100644 --- a/bookwyrm/management/commands/initdb.py +++ b/bookwyrm/management/commands/initdb.py @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import Group, Permission from django.contrib.contenttypes.models import ContentType -from bookwyrm.models import Connector, User +from bookwyrm.models import Connector, SiteSettings, User from bookwyrm.settings import DOMAIN def init_groups(): @@ -73,7 +73,7 @@ def init_connectors(): identifier='bookwyrm.social', name='BookWyrm dot Social', connector_file='bookwyrm_connector', - base_url='https://bookwyrm.social' , + base_url='https://bookwyrm.social', books_url='https://bookwyrm.social/book', covers_url='https://bookwyrm.social/images/covers', search_url='https://bookwyrm.social/search?q=', @@ -91,10 +91,14 @@ def init_connectors(): priority=3, ) +def init_settings(): + SiteSettings.objects.create() + class Command(BaseCommand): help = 'Initializes the database with starter data' def handle(self, *args, **options): init_groups() init_permissions() - init_connectors() \ No newline at end of file + init_connectors() + init_settings() From 823da2e7de849678935d42f1c2e0d64586626976 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 16:14:41 -0800 Subject: [PATCH 096/356] Re-create migration --- ...{0053_auto_20201005_2233.py => 0023_auto_20201213_0014.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename bookwyrm/migrations/{0053_auto_20201005_2233.py => 0023_auto_20201213_0014.py} (88%) diff --git a/bookwyrm/migrations/0053_auto_20201005_2233.py b/bookwyrm/migrations/0023_auto_20201213_0014.py similarity index 88% rename from bookwyrm/migrations/0053_auto_20201005_2233.py rename to bookwyrm/migrations/0023_auto_20201213_0014.py index ce944af2..efff4388 100644 --- a/bookwyrm/migrations/0053_auto_20201005_2233.py +++ b/bookwyrm/migrations/0023_auto_20201213_0014.py @@ -1,4 +1,4 @@ -# Generated by Django 3.0.7 on 2020-10-05 22:33 +# Generated by Django 3.0.7 on 2020-12-13 00:14 from django.db import migrations, models @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('bookwyrm', '0052_auto_20201005_2145'), + ('bookwyrm', '0022_auto_20201212_1744'), ] operations = [ From aacf5b7ba41d7ac5196aaecf1d8cab3cd5f9c398 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 18:00:39 -0800 Subject: [PATCH 097/356] fields for content warnings --- bookwyrm/activitypub/note.py | 1 + bookwyrm/models/status.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py index df28bf8d..263ddb2a 100644 --- a/bookwyrm/activitypub/note.py +++ b/bookwyrm/activitypub/note.py @@ -23,6 +23,7 @@ class Note(ActivityObject): cc: List[str] = field(default_factory=lambda: []) replies: Dict = field(default_factory=lambda: {}) inReplyTo: str = '' + summary: str = '' tag: List[Link] = field(default_factory=lambda: []) attachment: List[Image] = field(default_factory=lambda: []) sensitive: bool = False diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 55036f2c..308a3c8c 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -23,6 +23,8 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): default='public', choices=PrivacyLevels.choices ) + content_warning = fields.CharField( + max_length=150, blank=True, null=True, activitypub_field='summary') sensitive = fields.BooleanField(default=False) # the created date can't be this, because of receiving federated posts published_date = fields.DateTimeField( @@ -68,7 +70,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): **kwargs ) - def to_activity(self, pure=False): + def to_activity(self, pure=False):# pylint: disable=arguments-differ ''' return tombstone if the status is deleted ''' if self.deleted: return activitypub.Tombstone( From 1e01e76ac25d9bb064317f6cfce794cda6a95b86 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 18:06:48 -0800 Subject: [PATCH 098/356] removes unneeded imports --- bookwyrm/models/author.py | 3 ++- bookwyrm/models/base_model.py | 1 + bookwyrm/models/import_job.py | 1 - bookwyrm/models/relationship.py | 2 +- bookwyrm/status.py | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 79973a37..331d2dd6 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -16,7 +16,8 @@ class Author(ActivitypubMixin, BookWyrmModel): max_length=255, blank=True, null=True, deduplication_field=True) sync = models.BooleanField(default=True) last_sync_date = models.DateTimeField(default=timezone.now) - wikipedia_link = fields.CharField(max_length=255, blank=True, null=True, deduplication_field=True) + wikipedia_link = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) # idk probably other keys would be useful here? born = fields.DateTimeField(blank=True, null=True) died = fields.DateTimeField(blank=True, null=True) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index f44797ab..dd3065c9 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -44,6 +44,7 @@ class BookWyrmModel(models.Model): @receiver(models.signals.post_save) +#pylint: disable=unused-argument def execute_after_save(sender, instance, created, *args, **kwargs): ''' set the remote_id after save (when the id is available) ''' if not created or not hasattr(instance, 'get_remote_id'): diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index fe39325f..8b09216f 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -6,7 +6,6 @@ from django.db import models from django.utils import timezone from bookwyrm import books_manager -from bookwyrm.connectors import ConnectorException from bookwyrm.models import ReadThrough, User, Book from bookwyrm.utils.fields import JSONField from .base_model import PrivacyLevels diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index 8913b9ab..debe2ace 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -37,7 +37,7 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel): activity_serializer = activitypub.Follow - def get_remote_id(self, status=None): + def get_remote_id(self, status=None):# pylint: disable=arguments-differ ''' use shelf identifier in remote_id ''' status = status or 'follows' base_path = self.user_subject.remote_id diff --git a/bookwyrm/status.py b/bookwyrm/status.py index 83a106e5..648f2e7d 100644 --- a/bookwyrm/status.py +++ b/bookwyrm/status.py @@ -1,7 +1,7 @@ ''' Handle user activity ''' from django.utils import timezone -from bookwyrm import activitypub, books_manager, models +from bookwyrm import models from bookwyrm.sanitize_html import InputHtmlParser From 2b3daa022711e6e7181cf2574cf95dc59b8cbb27 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 18:13:00 -0800 Subject: [PATCH 099/356] disable some warnings --- bookwyrm/context_processors.py | 2 +- bookwyrm/forms.py | 3 ++- bookwyrm/goodreads_import.py | 2 +- bookwyrm/incoming.py | 3 --- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py index 72839dce..a1471ac4 100644 --- a/bookwyrm/context_processors.py +++ b/bookwyrm/context_processors.py @@ -1,7 +1,7 @@ ''' customize the info available in context for rendering templates ''' from bookwyrm import models -def site_settings(request): +def site_settings(request):# pylint: disable=unused-argument ''' include the custom info about the site ''' return { 'site': models.SiteSettings.objects.get() diff --git a/bookwyrm/forms.py b/bookwyrm/forms.py index 784f1038..a2c3e24b 100644 --- a/bookwyrm/forms.py +++ b/bookwyrm/forms.py @@ -30,7 +30,7 @@ class CustomForm(ModelForm): visible.field.widget.attrs['rows'] = None visible.field.widget.attrs['class'] = css_classes[input_type] - +# pylint: disable=missing-class-docstring class LoginForm(CustomForm): class Meta: model = models.User @@ -131,6 +131,7 @@ class ImportForm(forms.Form): class ExpiryWidget(widgets.Select): def value_from_datadict(self, data, files, name): + ''' human-readable exiration time buckets ''' selected_string = super().value_from_datadict(data, files, name) if selected_string == 'day': diff --git a/bookwyrm/goodreads_import.py b/bookwyrm/goodreads_import.py index 3fd330ab..93fc1c48 100644 --- a/bookwyrm/goodreads_import.py +++ b/bookwyrm/goodreads_import.py @@ -53,7 +53,7 @@ def import_data(job_id): for item in job.items.all(): try: item.resolve() - except Exception as e: + except Exception as e:# pylint: disable=broad-except logger.exception(e) item.fail_reason = 'Error loading book' item.save() diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index bbbebf0f..4964d393 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -17,9 +17,6 @@ from bookwyrm.signatures import Signature @csrf_exempt def inbox(request, username): ''' incoming activitypub events ''' - # TODO: should do some kind of checking if the user accepts - # this action from the sender probably? idk - # but this will just throw a 404 if the user doesn't exist try: models.User.objects.get(localname=username) except models.User.DoesNotExist: From 1e08eeb4c295b79e8cce1aff6e411f32352ea442 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 12 Dec 2020 18:25:04 -0800 Subject: [PATCH 100/356] Renames custom template tags file --- bookwyrm/templates/author.html | 2 +- bookwyrm/templates/book.html | 2 +- bookwyrm/templates/editions.html | 2 +- bookwyrm/templates/feed.html | 2 +- bookwyrm/templates/followers.html | 2 +- bookwyrm/templates/following.html | 2 +- bookwyrm/templates/import_status.html | 2 +- bookwyrm/templates/layout.html | 2 +- bookwyrm/templates/notifications.html | 2 +- bookwyrm/templates/shelf.html | 2 +- bookwyrm/templates/snippets/avatar.html | 2 +- bookwyrm/templates/snippets/book_cover.html | 2 +- bookwyrm/templates/snippets/book_preview.html | 2 +- bookwyrm/templates/snippets/boost_button.html | 2 +- bookwyrm/templates/snippets/cover_alt.html | 2 +- bookwyrm/templates/snippets/create_status.html | 2 +- bookwyrm/templates/snippets/fav_button.html | 2 +- bookwyrm/templates/snippets/finish_reading_modal.html | 2 +- bookwyrm/templates/snippets/follow_request_buttons.html | 2 +- bookwyrm/templates/snippets/privacy_select.html | 2 +- bookwyrm/templates/snippets/rate_action.html | 2 +- bookwyrm/templates/snippets/reply_form.html | 2 +- bookwyrm/templates/snippets/shelf.html | 2 +- bookwyrm/templates/snippets/shelve_button.html | 2 +- bookwyrm/templates/snippets/status.html | 2 +- bookwyrm/templates/snippets/status_body.html | 2 +- bookwyrm/templates/snippets/status_content.html | 2 +- bookwyrm/templates/snippets/status_header.html | 2 +- bookwyrm/templates/snippets/thread.html | 2 +- bookwyrm/templates/snippets/trimmed_text.html | 2 +- bookwyrm/templates/snippets/user_header.html | 2 +- bookwyrm/templates/snippets/username.html | 2 +- bookwyrm/templates/tag.html | 2 +- bookwyrm/templatetags/{fr_display.py => bookwyrm_tags.py} | 2 +- 34 files changed, 34 insertions(+), 34 deletions(-) rename bookwyrm/templatetags/{fr_display.py => bookwyrm_tags.py} (98%) diff --git a/bookwyrm/templates/author.html b/bookwyrm/templates/author.html index 3e3e0018..9a7a20ab 100644 --- a/bookwyrm/templates/author.html +++ b/bookwyrm/templates/author.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

{{ author.name }}

diff --git a/bookwyrm/templates/book.html b/bookwyrm/templates/book.html index 8b21b88c..10c2a27b 100644 --- a/bookwyrm/templates/book.html +++ b/bookwyrm/templates/book.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% load humanize %} {% block content %} diff --git a/bookwyrm/templates/editions.html b/bookwyrm/templates/editions.html index 273b2cd6..619ceafb 100644 --- a/bookwyrm/templates/editions.html +++ b/bookwyrm/templates/editions.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

Editions of "{{ work.title }}"

diff --git a/bookwyrm/templates/feed.html b/bookwyrm/templates/feed.html index 6e49943a..07ad8d0f 100644 --- a/bookwyrm/templates/feed.html +++ b/bookwyrm/templates/feed.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}
diff --git a/bookwyrm/templates/followers.html b/bookwyrm/templates/followers.html index 645e46a1..00cb13ca 100644 --- a/bookwyrm/templates/followers.html +++ b/bookwyrm/templates/followers.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

diff --git a/bookwyrm/templates/following.html b/bookwyrm/templates/following.html index 2cca9127..478ca813 100644 --- a/bookwyrm/templates/following.html +++ b/bookwyrm/templates/following.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html index f91e2cce..6bb903b0 100644 --- a/bookwyrm/templates/import_status.html +++ b/bookwyrm/templates/import_status.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% load humanize %} {% block content %}
diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html index b37c9cda..bcbdca2a 100644 --- a/bookwyrm/templates/layout.html +++ b/bookwyrm/templates/layout.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %} diff --git a/bookwyrm/templates/notifications.html b/bookwyrm/templates/notifications.html index f9494c31..f31df76d 100644 --- a/bookwyrm/templates/notifications.html +++ b/bookwyrm/templates/notifications.html @@ -1,6 +1,6 @@ {% extends 'layout.html' %} {% load humanize %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

Notifications

diff --git a/bookwyrm/templates/shelf.html b/bookwyrm/templates/shelf.html index d6842d13..390b9fc6 100644 --- a/bookwyrm/templates/shelf.html +++ b/bookwyrm/templates/shelf.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}
diff --git a/bookwyrm/templates/snippets/avatar.html b/bookwyrm/templates/snippets/avatar.html index cb0a12ea..da4b5fd2 100644 --- a/bookwyrm/templates/snippets/avatar.html +++ b/bookwyrm/templates/snippets/avatar.html @@ -1,3 +1,3 @@ -{% load fr_display %} +{% load bookwyrm_tags %} avatar for {{ user|username }} diff --git a/bookwyrm/templates/snippets/book_cover.html b/bookwyrm/templates/snippets/book_cover.html index 3fd83616..ceeef426 100644 --- a/bookwyrm/templates/snippets/book_cover.html +++ b/bookwyrm/templates/snippets/book_cover.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %}
{% if book.cover %} {% include 'snippets/cover_alt.html' with book=book %} diff --git a/bookwyrm/templates/snippets/book_preview.html b/bookwyrm/templates/snippets/book_preview.html index c675c45f..e7eca455 100644 --- a/bookwyrm/templates/snippets/book_preview.html +++ b/bookwyrm/templates/snippets/book_preview.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %}
diff --git a/bookwyrm/templates/snippets/boost_button.html b/bookwyrm/templates/snippets/boost_button.html index 57765bed..bf06cef7 100644 --- a/bookwyrm/templates/snippets/boost_button.html +++ b/bookwyrm/templates/snippets/boost_button.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %} {% with status.id|uuid as uuid %}
{% csrf_token %} diff --git a/bookwyrm/templates/snippets/cover_alt.html b/bookwyrm/templates/snippets/cover_alt.html index 52bd52f2..0cccc2e1 100644 --- a/bookwyrm/templates/snippets/cover_alt.html +++ b/bookwyrm/templates/snippets/cover_alt.html @@ -1,2 +1,2 @@ -{% load fr_display %} +{% load bookwyrm_tags %} '{{ book.title }}' Cover ({{ book|edition_info }}) diff --git a/bookwyrm/templates/snippets/create_status.html b/bookwyrm/templates/snippets/create_status.html index e36b1b19..ac8c0b75 100644 --- a/bookwyrm/templates/snippets/create_status.html +++ b/bookwyrm/templates/snippets/create_status.html @@ -1,5 +1,5 @@ {% load humanize %} -{% load fr_display %} +{% load bookwyrm_tags %}
    diff --git a/bookwyrm/templates/snippets/fav_button.html b/bookwyrm/templates/snippets/fav_button.html index 58ece1e4..11e03cdb 100644 --- a/bookwyrm/templates/snippets/fav_button.html +++ b/bookwyrm/templates/snippets/fav_button.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %} {% with status.id|uuid as uuid %} {% csrf_token %} diff --git a/bookwyrm/templates/snippets/finish_reading_modal.html b/bookwyrm/templates/snippets/finish_reading_modal.html index 8f8aeeff..d04d508d 100644 --- a/bookwyrm/templates/snippets/finish_reading_modal.html +++ b/bookwyrm/templates/snippets/finish_reading_modal.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %}