From ebb82287c2cc2c8e5ebfc8437515b862b9f0b467 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 27 Nov 2020 22:10:38 -0800
Subject: [PATCH 001/104] First pass at recursively resolving foreign keys

---
 bookwyrm/activitypub/__init__.py      |  1 -
 bookwyrm/activitypub/base_activity.py | 59 +++++++++++++++------------
 bookwyrm/models/status.py             |  4 +-
 3 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py
index 85245929..c97e8c49 100644
--- a/bookwyrm/activitypub/__init__.py
+++ b/bookwyrm/activitypub/__init__.py
@@ -5,7 +5,6 @@ import sys
 from .base_activity import ActivityEncoder, PublicKey, Signature
 from .base_activity import Link, Mention
 from .base_activity import ActivitySerializerError
-from .base_activity import tag_formatter
 from .image import Image
 from .note import Note, GeneratedNote, Article, Comment, Review, Quotation
 from .note import Tombstone
diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index caa4aeb8..e4d95b0e 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -25,12 +25,13 @@ class ActivityEncoder(JSONEncoder):
 
 
 @dataclass
-class Link():
+class Link:
     ''' for tagging a book in a status '''
     href: str
     name: str
     type: str = 'Link'
 
+
 @dataclass
 class Mention(Link):
     ''' a subtype of Link for mentioning an actor '''
@@ -125,7 +126,7 @@ class ActivityObject:
 
         with transaction.atomic():
             if instance:
-                # updating an existing model isntance
+                # updating an existing model instance
                 for k, v in mapped_fields.items():
                     setattr(instance, k, v)
                 instance.save()
@@ -133,6 +134,9 @@ class ActivityObject:
                 # creating a new model instance
                 instance = model.objects.create(**mapped_fields)
 
+            # --- these are all fields that can't be saved until after the
+            # instance has an id (after it's been saved). ---------------#
+
             # add images
             for (model_key, value) in image_fields.items():
                 formatted_value = image_formatter(value)
@@ -140,9 +144,20 @@ class ActivityObject:
                     continue
                 getattr(instance, model_key).save(*formatted_value, save=True)
 
+            # add many to many fields
             for (model_key, values) in many_to_many_fields.items():
                 # mention books, mention users
-                getattr(instance, model_key).set(values)
+                if values == MISSING:
+                    continue
+                model_field = getattr(instance, model_key)
+                model = model_field.model
+                items = []
+                for link in values:
+                    items.append(
+                        resolve_foreign_key(model, link.get('href'))
+                    )
+                getattr(instance, model_key).set(items)
+
 
             # add one to many fields
             for (model_key, values) in one_to_many_fields.items():
@@ -177,36 +192,30 @@ def resolve_foreign_key(model, remote_id):
     if hasattr(model.objects, 'select_subclasses'):
         result = result.select_subclasses()
 
+    # first, check for an existing copy in the database
     result = result.filter(
         remote_id=remote_id
     ).first()
+    if result:
+        return result
 
-    if not result:
+    # failing that, load the data and create the object
+    try:
+        response = requests.get(
+            remote_id,
+            headers={'Accept': 'application/json; charset=utf-8'},
+        )
+    except ConnectionError:
+        raise ActivitySerializerError(
+            'Could not connect to host for remote_id in %s model: %s' % \
+                (model.__name__, remote_id))
+    if not response.ok:
         raise ActivitySerializerError(
             'Could not resolve remote_id in %s model: %s' % \
                 (model.__name__, remote_id))
-    return result
 
-
-def tag_formatter(tags, tag_type):
-    ''' helper function to extract foreign keys from tag activity json '''
-    if not isinstance(tags, list):
-        return []
-    items = []
-    types = {
-        'Book': models.Book,
-        'Mention': models.User,
-    }
-    for tag in [t for t in tags if t.get('type') == tag_type]:
-        if not tag_type in types:
-            continue
-        remote_id = tag.get('href')
-        try:
-            item = resolve_foreign_key(types[tag_type], remote_id)
-        except ActivitySerializerError:
-            continue
-        items.append(item)
-    return items
+    item = model.activity_serializer(**response.json())
+    return item.to_model(model)
 
 
 def image_formatter(image_slug):
diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index 9d45379c..b55d2da6 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -80,12 +80,12 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         ActivityMapping(
             'tag', 'mention_books',
             lambda x: tag_formatter(x, 'title', 'Book'),
-            lambda x: activitypub.tag_formatter(x, 'Book')
+            lambda x: [i for i in x if x.get('type') == 'Book']
         ),
         ActivityMapping(
             'tag', 'mention_users',
             lambda x: tag_formatter(x, 'username', 'Mention'),
-            lambda x: activitypub.tag_formatter(x, 'Mention')
+            lambda x: [i for i in x if x.get('type') == 'Mention']
         ),
         ActivityMapping(
             'attachment', 'attachments',

From a93b5cf5bc2505f382cd02d8f9821417e395e6b5 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 10:18:24 -0800
Subject: [PATCH 002/104] Use remote_id resolver to load books, user

---
 bookwyrm/activitypub/__init__.py              |  4 +-
 bookwyrm/activitypub/base_activity.py         | 50 ++++++++++++++-----
 bookwyrm/activitypub/book.py                  | 26 +++++-----
 bookwyrm/activitypub/ordered_collection.py    |  1 +
 bookwyrm/activitypub/verbs.py                 |  8 +++
 bookwyrm/books_manager.py                     | 17 -------
 bookwyrm/incoming.py                          | 14 +-----
 .../migrations/0016_auto_20201128_1804.py     | 24 +++++++++
 bookwyrm/models/base_model.py                 |  4 ++
 bookwyrm/models/book.py                       |  7 +--
 bookwyrm/models/shelf.py                      | 18 ++++++-
 bookwyrm/models/status.py                     |  2 -
 bookwyrm/models/user.py                       |  1 -
 bookwyrm/remote_user.py                       | 29 +----------
 bookwyrm/views.py                             |  3 +-
 15 files changed, 115 insertions(+), 93 deletions(-)
 create mode 100644 bookwyrm/migrations/0016_auto_20201128_1804.py

diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py
index c97e8c49..eee9345d 100644
--- a/bookwyrm/activitypub/__init__.py
+++ b/bookwyrm/activitypub/__init__.py
@@ -4,7 +4,7 @@ import sys
 
 from .base_activity import ActivityEncoder, PublicKey, Signature
 from .base_activity import Link, Mention
-from .base_activity import ActivitySerializerError
+from .base_activity import ActivitySerializerError, resolve_remote_id
 from .image import Image
 from .note import Note, GeneratedNote, Article, Comment, Review, Quotation
 from .note import Tombstone
@@ -14,7 +14,7 @@ from .person import Person
 from .book import Edition, Work, Author
 from .verbs import Create, Delete, Undo, Update
 from .verbs import Follow, Accept, Reject
-from .verbs import Add, Remove
+from .verbs import Add, AddBook, Remove
 
 # this creates a list of all the Activity types that we can serialize,
 # so when an Activity comes in from outside, we can check if it's known
diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index e4d95b0e..dd46753f 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -3,15 +3,20 @@ from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
 from uuid import uuid4
 
+import dateutil.parser
+from dateutil.parser import ParserError
 from django.core.files.base import ContentFile
 from django.db import transaction
 from django.db.models.fields.related_descriptors \
         import ForwardManyToOneDescriptor, ManyToManyDescriptor, \
         ReverseManyToOneDescriptor
+from django.db.models.fields import DateTimeField
 from django.db.models.fields.files import ImageFileDescriptor
+from django.db.models.query_utils import DeferredAttribute
+from django.utils import timezone
 import requests
 
-from bookwyrm import books_manager, models
+from bookwyrm import models
 
 
 class ActivitySerializerError(ValueError):
@@ -106,11 +111,27 @@ class ActivityObject:
             model_field = getattr(model, mapping.model_key)
 
             formatted_value = mapping.model_formatter(value)
-            if isinstance(model_field, ForwardManyToOneDescriptor) and \
+            if isinstance(model_field, DeferredAttribute) and \
+                    isinstance(model_field.field, DateTimeField):
+                print("DATE")
+                try:
+                    formatted_value = timezone.make_aware(
+                        dateutil.parser.parse(formatted_value)
+                    )
+                except ParserError:
+                    formatted_value = None
+            elif isinstance(model_field, ForwardManyToOneDescriptor) and \
                     formatted_value:
                 # foreign key remote id reolver (work on Edition, for example)
                 fk_model = model_field.field.related_model
-                reference = resolve_foreign_key(fk_model, formatted_value)
+                if isinstance(formatted_value, dict) and \
+                        formatted_value.get('id'):
+                    # if the AP field is a serialized object (as in Add)
+                    remote_id = formatted_value['id']
+                else:
+                    # if the AP field is just a remote_id (as in every other case)
+                    remote_id = formatted_value
+                reference = resolve_remote_id(fk_model, remote_id)
                 mapped_fields[mapping.model_key] = reference
             elif isinstance(model_field, ManyToManyDescriptor):
                 # status mentions book/users
@@ -122,6 +143,8 @@ class ActivityObject:
                 # image fields need custom handling
                 image_fields[mapping.model_key] = formatted_value
             else:
+                if formatted_value == MISSING:
+                    formatted_value = None
                 mapped_fields[mapping.model_key] = formatted_value
 
         with transaction.atomic():
@@ -153,12 +176,15 @@ class ActivityObject:
                 model = model_field.model
                 items = []
                 for link in values:
+                    # check that the Type matches the model (because Status
+                    # tags contain both user mentions and book tags)
+                    if not model.activity_serializer.type == link.get('type'):
+                        continue
                     items.append(
-                        resolve_foreign_key(model, link.get('href'))
+                        resolve_remote_id(model, link.get('href'))
                     )
                 getattr(instance, model_key).set(items)
 
-
             # add one to many fields
             for (model_key, values) in one_to_many_fields.items():
                 if values == MISSING:
@@ -183,11 +209,8 @@ class ActivityObject:
         return data
 
 
-def resolve_foreign_key(model, remote_id):
-    ''' look up the remote_id on an activity json field '''
-    if model in [models.Edition, models.Work, models.Book]:
-        return books_manager.get_or_create_book(remote_id)
-
+def resolve_remote_id(model, remote_id, refresh=False):
+    ''' look up the remote_id in the database or load it remotely '''
     result = model.objects
     if hasattr(model.objects, 'select_subclasses'):
         result = result.select_subclasses()
@@ -196,10 +219,10 @@ def resolve_foreign_key(model, remote_id):
     result = result.filter(
         remote_id=remote_id
     ).first()
-    if result:
+    if result and not refresh:
         return result
 
-    # failing that, load the data and create the object
+    # load the data and create the object
     try:
         response = requests.get(
             remote_id,
@@ -215,7 +238,8 @@ def resolve_foreign_key(model, remote_id):
                 (model.__name__, remote_id))
 
     item = model.activity_serializer(**response.json())
-    return item.to_model(model)
+    # if we're refreshing, "result" will be set and we'll update it
+    return item.to_model(model, instance=result)
 
 
 def image_formatter(image_slug):
diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py
index 02cab281..8a6b88d9 100644
--- a/bookwyrm/activitypub/book.py
+++ b/bookwyrm/activitypub/book.py
@@ -12,13 +12,13 @@ class Book(ActivityObject):
     sortTitle: str = ''
     subtitle: str = ''
     description: str = ''
-    languages: List[str]
+    languages: List[str] = field(default_factory=lambda: [])
     series: str = ''
     seriesNumber: str = ''
-    subjects: List[str]
-    subjectPlaces: List[str]
+    subjects: List[str] = field(default_factory=lambda: [])
+    subjectPlaces: List[str] = field(default_factory=lambda: [])
 
-    authors: List[str]
+    authors: List[str] = field(default_factory=lambda: [])
     firstPublishedDate: str = ''
     publishedDate: str = ''
 
@@ -33,22 +33,22 @@ class Book(ActivityObject):
 @dataclass(init=False)
 class Edition(Book):
     ''' Edition instance of a book object '''
-    isbn10: str
-    isbn13: str
-    oclcNumber: str
-    asin: str
-    pages: str
-    physicalFormat: str
-    publishers: List[str]
-
     work: str
+    isbn10: str = ''
+    isbn13: str = ''
+    oclcNumber: str = ''
+    asin: str = ''
+    pages: str = ''
+    physicalFormat: str = ''
+    publishers: List[str] = field(default_factory=lambda: [])
+
     type: str = 'Edition'
 
 
 @dataclass(init=False)
 class Work(Book):
     ''' work instance of a book object '''
-    lccn: str
+    lccn: str = ''
     editions: List[str]
     type: str = 'Work'
 
diff --git a/bookwyrm/activitypub/ordered_collection.py b/bookwyrm/activitypub/ordered_collection.py
index efd23d5a..9aeaf664 100644
--- a/bookwyrm/activitypub/ordered_collection.py
+++ b/bookwyrm/activitypub/ordered_collection.py
@@ -12,6 +12,7 @@ class OrderedCollection(ActivityObject):
     first: str
     last: str = ''
     name: str = ''
+    owner: str = ''
     type: str = 'OrderedCollection'
 
 
diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py
index eb166260..e890d81f 100644
--- a/bookwyrm/activitypub/verbs.py
+++ b/bookwyrm/activitypub/verbs.py
@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from typing import List
 
 from .base_activity import ActivityObject, Signature
+from .book import Book
 
 @dataclass(init=False)
 class Verb(ActivityObject):
@@ -69,6 +70,13 @@ class Add(Verb):
     type: str = 'Add'
 
 
+@dataclass(init=False)
+class AddBook(Verb):
+    '''Add activity that's aware of the book obj '''
+    target: Book
+    type: str = 'Add'
+
+
 @dataclass(init=False)
 class Remove(Verb):
     '''Remove activity '''
diff --git a/bookwyrm/books_manager.py b/bookwyrm/books_manager.py
index 461017a0..fb0a747e 100644
--- a/bookwyrm/books_manager.py
+++ b/bookwyrm/books_manager.py
@@ -16,23 +16,6 @@ def get_edition(book_id):
     return book
 
 
-def get_or_create_book(remote_id):
-    ''' pull up a book record by whatever means possible '''
-    book = models.Book.objects.select_subclasses().filter(
-        remote_id=remote_id
-    ).first()
-    if book:
-        return book
-
-    connector = get_or_create_connector(remote_id)
-
-    # raises ConnectorException
-    book = connector.get_or_create_book(remote_id)
-    if book:
-        load_more_data.delay(book.id)
-    return book
-
-
 def get_or_create_connector(remote_id):
     ''' get the connector related to the author's server '''
     url = urlparse(remote_id)
diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index 0e7c1856..9a348483 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -317,7 +317,7 @@ def handle_tag(activity):
     user = get_or_create_remote_user(activity['actor'])
     if not user.local:
         # ordered collection weirndess so we can't just to_model
-        book = books_manager.get_or_create_book(activity['object']['id'])
+        book = (activity['object']['id'])
         name = activity['object']['target'].split('/')[-1]
         name = unquote_plus(name)
         models.Tag.objects.get_or_create(
@@ -330,17 +330,7 @@ def handle_tag(activity):
 @app.task
 def handle_shelve(activity):
     ''' putting a book on a shelf '''
-    user = get_or_create_remote_user(activity['actor'])
-    book = books_manager.get_or_create_book(activity['object'])
-    try:
-        shelf = models.Shelf.objects.get(remote_id=activity['target'])
-    except models.Shelf.DoesNotExist:
-        return
-    if shelf.user != user:
-        # this doesn't add up.
-        return
-    shelf.books.add(book)
-    shelf.save()
+    activitypub.AddBook(**activity).to_model(models.ShelfBook)
 
 
 @app.task
diff --git a/bookwyrm/migrations/0016_auto_20201128_1804.py b/bookwyrm/migrations/0016_auto_20201128_1804.py
new file mode 100644
index 00000000..9becde82
--- /dev/null
+++ b/bookwyrm/migrations/0016_auto_20201128_1804.py
@@ -0,0 +1,24 @@
+# Generated by Django 3.0.7 on 2020-11-28 18:04
+
+import bookwyrm.utils.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0015_auto_20201128_0349'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='book',
+            name='subject_places',
+            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subjects',
+            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+    ]
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 4109a49b..29301fb5 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -227,12 +227,16 @@ class OrderedCollectionPageMixin(ActivitypubMixin):
         name = ''
         if hasattr(self, 'name'):
             name = self.name
+        owner = ''
+        if hasattr(self, 'user'):
+            owner = self.user.remote_id
 
         size = queryset.count()
         return activitypub.OrderedCollection(
             id=remote_id,
             totalItems=size,
             name=name,
+            owner=owner,
             first='%s%s' % (remote_id, self.page()),
             last='%s%s' % (remote_id, self.page(min_id=0))
         ).serialize()
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index 132b4c07..8892119a 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -41,10 +41,10 @@ class Book(ActivitypubMixin, BookWyrmModel):
     series = models.CharField(max_length=255, blank=True, null=True)
     series_number = models.CharField(max_length=255, blank=True, null=True)
     subjects = ArrayField(
-        models.CharField(max_length=255), blank=True, default=list
+        models.CharField(max_length=255), blank=True, null=True, default=list
     )
     subject_places = ArrayField(
-        models.CharField(max_length=255), blank=True, default=list
+        models.CharField(max_length=255), blank=True, null=True, default=list
     )
     # TODO: include an annotation about the type of authorship (ie, translator)
     authors = models.ManyToManyField('Author')
@@ -132,7 +132,8 @@ class Work(OrderedCollectionPageMixin, Book):
         ''' it'd be nice to serialize the edition instead but, recursion '''
         default = self.default_edition
         ed_list = [
-            e.remote_id for e in self.edition_set.filter(~Q(id=default.id)).all()
+            e.remote_id for e in \
+                    self.edition_set.filter(~Q(id=default.id)).all()
         ]
         return [default.remote_id] + ed_list
 
diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py
index e85294ba..3f443f6d 100644
--- a/bookwyrm/models/shelf.py
+++ b/bookwyrm/models/shelf.py
@@ -3,7 +3,8 @@ import re
 from django.db import models
 
 from bookwyrm import activitypub
-from .base_model import BookWyrmModel, OrderedCollectionMixin, PrivacyLevels
+from .base_model import ActivityMapping, BookWyrmModel
+from .base_model import OrderedCollectionMixin, PrivacyLevels
 
 
 class Shelf(OrderedCollectionMixin, BookWyrmModel):
@@ -47,6 +48,12 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
         ''' user/shelf unqiueness '''
         unique_together = ('user', 'identifier')
 
+    activity_mappings = [
+        ActivityMapping('id', 'remote_id'),
+        ActivityMapping('owner', 'user'),
+        ActivityMapping('name', 'name'),
+    ]
+
 
 class ShelfBook(BookWyrmModel):
     ''' many to many join table for books and shelves '''
@@ -59,6 +66,15 @@ class ShelfBook(BookWyrmModel):
         on_delete=models.PROTECT
     )
 
+    activity_mappings = [
+        ActivityMapping('id', 'remote_id'),
+        ActivityMapping('actor', 'added_by'),
+        ActivityMapping('object', 'book'),
+        ActivityMapping('target', 'shelf')
+    ]
+
+    activity_serializer = activitypub.AddBook
+
     def to_add_activity(self, user):
         ''' AP for shelving a book'''
         return activitypub.Add(
diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index b55d2da6..a520164a 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -80,12 +80,10 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         ActivityMapping(
             'tag', 'mention_books',
             lambda x: tag_formatter(x, 'title', 'Book'),
-            lambda x: [i for i in x if x.get('type') == 'Book']
         ),
         ActivityMapping(
             'tag', 'mention_users',
             lambda x: tag_formatter(x, 'username', 'Mention'),
-            lambda x: [i for i in x if x.get('type') == 'Mention']
         ),
         ActivityMapping(
             'attachment', 'attachments',
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index 4d511d56..a9812037 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -11,7 +11,6 @@ from bookwyrm.models.status import Status
 from bookwyrm.settings import DOMAIN
 from bookwyrm.signatures import create_key_pair
 from .base_model import ActivityMapping, OrderedCollectionPageMixin
-from .base_model import image_formatter
 
 
 class User(OrderedCollectionPageMixin, AbstractUser):
diff --git a/bookwyrm/remote_user.py b/bookwyrm/remote_user.py
index 23a805b3..205b0893 100644
--- a/bookwyrm/remote_user.py
+++ b/bookwyrm/remote_user.py
@@ -16,11 +16,9 @@ def get_or_create_remote_user(actor):
     except models.User.DoesNotExist:
         pass
 
-    data = fetch_user_data(actor)
-
     actor_parts = urlparse(actor)
     with transaction.atomic():
-        user = activitypub.Person(**data).to_model(models.User)
+        user = activitypub.resolve_remote_id(models.User, actor)
         user.federated_server = get_or_create_remote_server(actor_parts.netloc)
         user.save()
     if user.bookwyrm_user:
@@ -28,32 +26,9 @@ def get_or_create_remote_user(actor):
     return user
 
 
-def fetch_user_data(actor):
-    ''' load the user's info from the actor url '''
-    try:
-        response = requests.get(
-            actor,
-            headers={'Accept': 'application/activity+json'}
-        )
-    except ConnectionError:
-        return None
-
-    if not response.ok:
-        response.raise_for_status()
-    data = response.json()
-
-    # make sure our actor is who they say they are
-    if actor != data['id']:
-        raise ValueError("Remote actor id must match url.")
-    return data
-
-
 def refresh_remote_user(user):
     ''' get updated user data from its home instance '''
-    data = fetch_user_data(user.remote_id)
-
-    activity = activitypub.Person(**data)
-    activity.to_model(models.User, instance=user)
+    activitypub.resolve_remote_id(user.remote_id, refresh=True)
 
 
 @app.task
diff --git a/bookwyrm/views.py b/bookwyrm/views.py
index e0feaee7..e2887255 100644
--- a/bookwyrm/views.py
+++ b/bookwyrm/views.py
@@ -5,8 +5,7 @@ from django.contrib.auth.decorators import login_required, permission_required
 from django.contrib.postgres.search import TrigramSimilarity
 from django.core.paginator import Paginator
 from django.db.models import Avg, Q
-from django.http import HttpResponseBadRequest, HttpResponseNotFound,\
-        JsonResponse
+from django.http import HttpResponseNotFound, JsonResponse
 from django.core.exceptions import PermissionDenied
 from django.shortcuts import get_object_or_404, redirect
 from django.template.response import TemplateResponse

From fd7e476c9bec84c213d2e8f24a1911ecc126d7db Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 11:00:40 -0800
Subject: [PATCH 003/104] refactors tag model to fit ordered collection
 structure

---
 bookwyrm/activitypub/base_activity.py         |  7 ++-
 bookwyrm/incoming.py                          | 31 ++++---------
 .../migrations/0017_auto_20201128_1849.py     | 42 ++++++++++++++++++
 bookwyrm/models/__init__.py                   |  2 +-
 bookwyrm/models/shelf.py                      |  2 +-
 bookwyrm/models/tag.py                        | 43 ++++++++++++++-----
 bookwyrm/signatures.py                        |  2 +-
 bookwyrm/templates/snippets/tag.html          | 10 ++---
 bookwyrm/view_actions.py                      |  7 ++-
 bookwyrm/views.py                             | 16 +++----
 10 files changed, 110 insertions(+), 52 deletions(-)
 create mode 100644 bookwyrm/migrations/0017_auto_20201128_1849.py

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index dd46753f..a6204536 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -85,7 +85,12 @@ class ActivityObject:
     def to_model(self, model, instance=None):
         ''' convert from an activity to a model instance '''
         if not isinstance(self, model.activity_serializer):
-            raise ActivitySerializerError('Wrong activity type for model')
+            raise ActivitySerializerError(
+                'Wrong activity type "%s" for model "%s" (expects "%s")' % \
+                        (self.__class__,
+                         model.__name__,
+                         model.activity_serializer)
+            )
 
         # check for an existing instance, if we're not updating a known obj
         if not instance:
diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index 9a348483..ef05cc4e 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -1,6 +1,6 @@
 ''' handles all of the activity coming in to the server '''
 import json
-from urllib.parse import urldefrag, unquote_plus
+from urllib.parse import urldefrag
 
 import django.db.utils
 from django.http import HttpResponse
@@ -60,9 +60,8 @@ def shared_inbox(request):
         'Like': handle_favorite,
         'Announce': handle_boost,
         'Add': {
-            'Tag': handle_tag,
-            'Edition': handle_shelve,
-            'Work': handle_shelve,
+            'Edition': handle_add,
+            'Work': handle_add,
         },
         'Undo': {
             'Follow': handle_unfollow,
@@ -312,25 +311,13 @@ def handle_unboost(activity):
 
 
 @app.task
-def handle_tag(activity):
-    ''' someone is tagging a book '''
-    user = get_or_create_remote_user(activity['actor'])
-    if not user.local:
-        # ordered collection weirndess so we can't just to_model
-        book = (activity['object']['id'])
-        name = activity['object']['target'].split('/')[-1]
-        name = unquote_plus(name)
-        models.Tag.objects.get_or_create(
-            user=user,
-            book=book,
-            name=name
-        )
-
-
-@app.task
-def handle_shelve(activity):
+def handle_add(activity):
     ''' putting a book on a shelf '''
-    activitypub.AddBook(**activity).to_model(models.ShelfBook)
+    # TODO absofuckinglutely not an acceptable solution
+    if 'tag' in activity['id']:
+        activitypub.AddBook(**activity).to_model(models.Tag)
+    else:
+        activitypub.AddBook(**activity).to_model(models.ShelfBook)
 
 
 @app.task
diff --git a/bookwyrm/migrations/0017_auto_20201128_1849.py b/bookwyrm/migrations/0017_auto_20201128_1849.py
new file mode 100644
index 00000000..722458b2
--- /dev/null
+++ b/bookwyrm/migrations/0017_auto_20201128_1849.py
@@ -0,0 +1,42 @@
+# Generated by Django 3.0.7 on 2020-11-28 18:49
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0016_auto_20201128_1804'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='tag',
+            unique_together=set(),
+        ),
+        migrations.RemoveField(
+            model_name='tag',
+            name='book',
+        ),
+        migrations.RemoveField(
+            model_name='tag',
+            name='user',
+        ),
+        migrations.CreateModel(
+            name='UserTag',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_date', models.DateTimeField(auto_now_add=True)),
+                ('updated_date', models.DateTimeField(auto_now=True)),
+                ('remote_id', models.CharField(max_length=255, null=True)),
+                ('book', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition')),
+                ('tag', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'unique_together': {('user', 'book', 'tag')},
+            },
+        ),
+    ]
diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py
index 3d854478..9a4d6014 100644
--- a/bookwyrm/models/__init__.py
+++ b/bookwyrm/models/__init__.py
@@ -12,7 +12,7 @@ from .status import Status, GeneratedNote, Review, Comment, Quotation
 from .status import Favorite, Boost, Notification, ReadThrough
 from .attachment import Image
 
-from .tag import Tag
+from .tag import Tag, UserTag
 
 from .user import User
 from .relationship import UserFollows, UserFollowRequest, UserBlocks
diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py
index 3f443f6d..b8d5ea17 100644
--- a/bookwyrm/models/shelf.py
+++ b/bookwyrm/models/shelf.py
@@ -70,7 +70,7 @@ class ShelfBook(BookWyrmModel):
         ActivityMapping('id', 'remote_id'),
         ActivityMapping('actor', 'added_by'),
         ActivityMapping('object', 'book'),
-        ActivityMapping('target', 'shelf')
+        ActivityMapping('target', 'shelf'),
     ]
 
     activity_serializer = activitypub.AddBook
diff --git a/bookwyrm/models/tag.py b/bookwyrm/models/tag.py
index cd98e2b1..8b7efceb 100644
--- a/bookwyrm/models/tag.py
+++ b/bookwyrm/models/tag.py
@@ -5,16 +5,19 @@ from django.db import models
 
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
-from .base_model import OrderedCollectionMixin, BookWyrmModel
+from .base_model import OrderedCollectionMixin, BookWyrmModel, ActivityMapping
 
 
 class Tag(OrderedCollectionMixin, BookWyrmModel):
     ''' freeform tags for books '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
-    name = models.CharField(max_length=100)
+    name = models.CharField(max_length=100, unique=True)
     identifier = models.CharField(max_length=100)
 
+    activity_mappings = [
+        ActivityMapping('id', 'remote_id'),
+        ActivityMapping('name', 'name'),
+    ]
+
     @classmethod
     def book_queryset(cls, identifier):
         ''' county of books associated with this tag '''
@@ -30,6 +33,30 @@ class Tag(OrderedCollectionMixin, BookWyrmModel):
         base_path = 'https://%s' % DOMAIN
         return '%s/tag/%s' % (base_path, self.identifier)
 
+
+    def save(self, *args, **kwargs):
+        ''' create a url-safe lookup key for the tag '''
+        if not self.id:
+            # add identifiers to new tags
+            self.identifier = urllib.parse.quote_plus(self.name)
+        super().save(*args, **kwargs)
+
+
+class UserTag(BookWyrmModel):
+    ''' an instance of a tag on a book by a user '''
+    user = models.ForeignKey('User', on_delete=models.PROTECT)
+    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
+    tag = models.ForeignKey('Tag', on_delete=models.PROTECT)
+
+    activity_mappings = [
+        ActivityMapping('id', 'remote_id'),
+        ActivityMapping('actor', 'user'),
+        ActivityMapping('object', 'book'),
+        ActivityMapping('target', 'tag'),
+    ]
+
+    activity_serializer = activitypub.AddBook
+
     def to_add_activity(self, user):
         ''' AP for shelving a book'''
         return activitypub.Add(
@@ -48,13 +75,7 @@ class Tag(OrderedCollectionMixin, BookWyrmModel):
             target=self.to_activity(),
         ).serialize()
 
-    def save(self, *args, **kwargs):
-        ''' create a url-safe lookup key for the tag '''
-        if not self.id:
-            # add identifiers to new tags
-            self.identifier = urllib.parse.quote_plus(self.name)
-        super().save(*args, **kwargs)
 
     class Meta:
         ''' unqiueness constraint '''
-        unique_together = ('user', 'book', 'name')
+        unique_together = ('user', 'book', 'tag')
diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py
index 57c181df..dbb88d8a 100644
--- a/bookwyrm/signatures.py
+++ b/bookwyrm/signatures.py
@@ -89,7 +89,7 @@ class Signature:
 
     def verify(self, public_key, request):
         ''' verify rsa signature '''
-        if http_date_age(request.headers['date']) > MAX_SIGNATURE_AGE:
+        if False:#http_date_age(request.headers['date']) > MAX_SIGNATURE_AGE:
             raise ValueError(
                 "Request too old: %s" % (request.headers['date'],))
         public_key = RSA.import_key(public_key)
diff --git a/bookwyrm/templates/snippets/tag.html b/bookwyrm/templates/snippets/tag.html
index e62167f9..482cffc3 100644
--- a/bookwyrm/templates/snippets/tag.html
+++ b/bookwyrm/templates/snippets/tag.html
@@ -1,14 +1,14 @@
 <div class="control">
-    <form name="tag" action="/{% if tag.identifier in user_tags %}untag{% else %}tag{% endif %}/" method="post">
+    <form name="tag" action="/{% if tag.tag.identifier in user_tags %}untag{% else %}tag{% endif %}/" method="post">
         {% csrf_token %}
         <input type="hidden" name="book" value="{{ book.id }}">
-        <input type="hidden" name="name" value="{{ tag.name }}">
+        <input type="hidden" name="name" value="{{ tag.tag.name }}">
 
         <div class="tags has-addons">
-            <a class="tag" href="/tag/{{ tag.identifier|urlencode }}">
-                {{ tag.name }}
+            <a class="tag" href="/tag/{{ tag.tag.identifier|urlencode }}">
+                {{ tag.tag.name }}
             </a>
-            {% if tag.identifier in user_tags %}
+            {% if tag.tag.identifier in user_tags %}
             <button class="tag is-delete" type="submit">
                 <span class="is-sr-only">remove tag</span>
             </button>
diff --git a/bookwyrm/view_actions.py b/bookwyrm/view_actions.py
index ca306bcb..0f51c66f 100644
--- a/bookwyrm/view_actions.py
+++ b/bookwyrm/view_actions.py
@@ -531,12 +531,15 @@ def tag(request):
     book = get_object_or_404(models.Edition, id=book_id)
     tag_obj, created = models.Tag.objects.get_or_create(
         name=name,
+    )
+    user_tag = models.UserTag.objects.get_or_create(
+        user=request.user,
         book=book,
-        user=request.user
+        tag=tag_obj,
     )
 
     if created:
-        outgoing.handle_tag(request.user, tag_obj)
+        outgoing.handle_tag(request.user, user_tag)
     return redirect('/book/%s' % book_id)
 
 
diff --git a/bookwyrm/views.py b/bookwyrm/views.py
index e2887255..c6992e6b 100644
--- a/bookwyrm/views.py
+++ b/bookwyrm/views.py
@@ -560,9 +560,9 @@ def book_page(request, book_id):
     user_tags = []
     readthroughs = []
     if request.user.is_authenticated:
-        user_tags = models.Tag.objects.filter(
+        user_tags = models.UserTag.objects.filter(
             book=book, user=request.user
-        ).values_list('identifier', flat=True)
+        ).values_list('tag__identifier', flat=True)
 
         readthroughs = models.ReadThrough.objects.filter(
             user=request.user,
@@ -570,11 +570,9 @@ def book_page(request, book_id):
         ).order_by('start_date')
 
     rating = reviews.aggregate(Avg('rating'))
-    tags = models.Tag.objects.filter(
-        book=book
-    ).values(
-        'book', 'name', 'identifier'
-    ).distinct().all()
+    tags = models.UserTag.objects.filter(
+        book=book,
+    )
 
     data = {
         'title': book.title,
@@ -664,7 +662,9 @@ def tag_page(request, tag_id):
         return JsonResponse(
             tag_obj.to_activity(**request.GET), encoder=ActivityEncoder)
 
-    books = models.Edition.objects.filter(tag__identifier=tag_id).distinct()
+    books = models.Edition.objects.filter(
+        usertag__tag__identifier=tag_id
+    ).distinct()
     data = {
         'title': tag_obj.name,
         'books': books,

From b0202eb8e8248907c650355a4b7b4b0f282d161c Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 11:48:17 -0800
Subject: [PATCH 004/104] Remove special remote user handling code

also fixes date parsing
---
 bookwyrm/activitypub/base_activity.py |  9 +--
 bookwyrm/incoming.py                  | 49 +++++++--------
 bookwyrm/models/user.py               | 69 ++++++++++++++++++++-
 bookwyrm/outgoing.py                  |  7 ++-
 bookwyrm/remote_user.py               | 86 ---------------------------
 bookwyrm/status.py                    | 31 ----------
 celerywyrm/celery.py                  |  2 +-
 7 files changed, 102 insertions(+), 151 deletions(-)
 delete mode 100644 bookwyrm/remote_user.py

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index a6204536..0672fbb8 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -118,11 +118,12 @@ class ActivityObject:
             formatted_value = mapping.model_formatter(value)
             if isinstance(model_field, DeferredAttribute) and \
                     isinstance(model_field.field, DateTimeField):
-                print("DATE")
                 try:
-                    formatted_value = timezone.make_aware(
-                        dateutil.parser.parse(formatted_value)
-                    )
+                    date_value = dateutil.parser.parse(formatted_value)
+                    try:
+                        formatted_value = timezone.make_aware(date_value)
+                    except ValueError:
+                        formatted_value = date_value
                 except ParserError:
                     formatted_value = None
             elif isinstance(model_field, ForwardManyToOneDescriptor) and \
diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index ef05cc4e..6ce2ea0b 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -10,7 +10,6 @@ import requests
 
 from bookwyrm import activitypub, books_manager, models, outgoing
 from bookwyrm import status as status_builder
-from bookwyrm.remote_user import get_or_create_remote_user, refresh_remote_user
 from bookwyrm.tasks import app
 from bookwyrm.signatures import Signature
 
@@ -96,13 +95,15 @@ def has_valid_signature(request, activity):
         if key_actor != activity.get('actor'):
             raise ValueError("Wrong actor created signature.")
 
-        remote_user = get_or_create_remote_user(key_actor)
+        remote_user = activitypub.resolve_remote_id(models.User, key_actor)
 
         try:
             signature.verify(remote_user.public_key, request)
         except ValueError:
             old_key = remote_user.public_key
-            refresh_remote_user(remote_user)
+            activitypub.resolve_remote_id(
+                models.User, remote_user, refresh=True
+            )
             if remote_user.public_key == old_key:
                 raise # Key unchanged.
             signature.verify(remote_user.public_key, request)
@@ -127,7 +128,7 @@ def handle_follow(activity):
         return
 
     # figure out who the actor is
-    actor = get_or_create_remote_user(activity['actor'])
+    actor = activitypub.resolve_remote_id(models.User, activity['actor'])
     try:
         relationship = models.UserFollowRequest.objects.create(
             user_subject=actor,
@@ -162,7 +163,7 @@ def handle_follow(activity):
 def handle_unfollow(activity):
     ''' unfollow a local user '''
     obj = activity['object']
-    requester = get_or_create_remote_user(obj['actor'])
+    requester = activitypub.resolve_remote_id(models.user, obj['actor'])
     to_unfollow = models.User.objects.get(remote_id=obj['object'])
     # raises models.User.DoesNotExist
 
@@ -175,7 +176,7 @@ def handle_follow_accept(activity):
     # figure out who they want to follow
     requester = models.User.objects.get(remote_id=activity['object']['actor'])
     # figure out who they are
-    accepter = get_or_create_remote_user(activity['actor'])
+    accepter = activitypub.resolve_remote_id(models.User, activity['actor'])
 
     try:
         request = models.UserFollowRequest.objects.get(
@@ -192,7 +193,7 @@ def handle_follow_accept(activity):
 def handle_follow_reject(activity):
     ''' someone is rejecting a follow request '''
     requester = models.User.objects.get(remote_id=activity['object']['actor'])
-    rejecter = get_or_create_remote_user(activity['actor'])
+    rejecter = activitypub.resolve_remote_id(models.User, activity['actor'])
 
     request = models.UserFollowRequest.objects.get(
         user_subject=requester,
@@ -205,25 +206,27 @@ def handle_follow_reject(activity):
 @app.task
 def handle_create(activity):
     ''' someone did something, good on them '''
-    if activity['object'].get('type') not in \
-            ['Note', 'Comment', 'Quotation', 'Review', 'GeneratedNote']:
-        # if it's an article or unknown type, ignore it
-        return
-
-    user = get_or_create_remote_user(activity['actor'])
-    if user.local:
-        # we really oughtn't even be sending in this case
-        return
-
     # deduplicate incoming activities
     status_id = activity['object']['id']
     if models.Status.objects.filter(remote_id=status_id).count():
         return
 
-    status = status_builder.create_status(activity['object'])
-    if not status:
+    serializer = activitypub.activity_objects[activity['type']]
+    status = serializer(**activity)
+    try:
+        model = models.activity_models[activity.type]
+    except KeyError:
+        # not a type of status we are prepared to deserialize
         return
 
+    if activity.type == 'Note':
+        reply = models.Status.objects.filter(
+            remote_id=activity.inReplyTo
+        ).first()
+        if not reply:
+            return
+
+    activity.to_model(model)
     # create a notification if this is a reply
     if status.reply_parent and status.reply_parent.user.local:
         status_builder.create_notification(
@@ -257,16 +260,14 @@ def handle_favorite(activity):
     ''' approval of your good good post '''
     fav = activitypub.Like(**activity)
 
-    liker = get_or_create_remote_user(activity['actor'])
-    if liker.local:
-        return
-
     fav = fav.to_model(models.Favorite)
+    if fav.user.local:
+        return
 
     status_builder.create_notification(
         fav.status.user,
         'FAVORITE',
-        related_user=liker,
+        related_user=fav.user,
         related_status=fav.status,
     )
 
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index a9812037..a32a35b7 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -1,5 +1,6 @@
 ''' database schema for user data '''
 from urllib.parse import urlparse
+import requests
 
 from django.contrib.auth.models import AbstractUser
 from django.db import models
@@ -7,10 +8,12 @@ from django.dispatch import receiver
 
 from bookwyrm import activitypub
 from bookwyrm.models.shelf import Shelf
-from bookwyrm.models.status import Status
+from bookwyrm.models.status import Status, Review
 from bookwyrm.settings import DOMAIN
 from bookwyrm.signatures import create_key_pair
+from bookwyrm.tasks import app
 from .base_model import ActivityMapping, OrderedCollectionPageMixin
+from .federated_server import FederatedServer
 
 
 class User(OrderedCollectionPageMixin, AbstractUser):
@@ -188,7 +191,16 @@ class User(OrderedCollectionPageMixin, AbstractUser):
 @receiver(models.signals.post_save, sender=User)
 def execute_after_save(sender, instance, created, *args, **kwargs):
     ''' create shelves for new users '''
-    if not instance.local or not created:
+    if not created:
+        return
+
+    if not instance.local:
+        actor_parts = urlparse(instance.remote_id)
+        instance.federated_server = \
+            get_or_create_remote_server(actor_parts.netloc)
+        instance.save()
+        if instance.bookwyrm_user:
+            get_remote_reviews.delay(instance.outbox)
         return
 
     shelves = [{
@@ -209,3 +221,56 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
             user=instance,
             editable=False
         ).save()
+
+
+def get_or_create_remote_server(domain):
+    ''' get info on a remote server '''
+    try:
+        return FederatedServer.objects.get(
+            server_name=domain
+        )
+    except FederatedServer.DoesNotExist:
+        pass
+
+    response = requests.get(
+        'https://%s/.well-known/nodeinfo' % domain,
+        headers={'Accept': 'application/activity+json'}
+    )
+
+    if response.status_code != 200:
+        return None
+
+    data = response.json()
+    try:
+        nodeinfo_url = data.get('links')[0].get('href')
+    except (TypeError, KeyError):
+        return None
+
+    response = requests.get(
+        nodeinfo_url,
+        headers={'Accept': 'application/activity+json'}
+    )
+    data = response.json()
+
+    server = FederatedServer.objects.create(
+        server_name=domain,
+        application_type=data['software']['name'],
+        application_version=data['software']['version'],
+    )
+    return server
+
+
+@app.task
+def get_remote_reviews(outbox):
+    ''' ingest reviews by a new remote bookwyrm user '''
+    outbox_page = outbox + '?page=true'
+    response = requests.get(
+        outbox_page,
+        headers={'Accept': 'application/activity+json'}
+    )
+    data = response.json()
+    # TODO: pagination?
+    for activity in data['orderedItems']:
+        if not activity['type'] == 'Review':
+            continue
+        activitypub.Review(**activity).to_model(Review)
diff --git a/bookwyrm/outgoing.py b/bookwyrm/outgoing.py
index a196fcec..545ac491 100644
--- a/bookwyrm/outgoing.py
+++ b/bookwyrm/outgoing.py
@@ -12,7 +12,6 @@ from bookwyrm.broadcast import broadcast
 from bookwyrm.status import create_notification
 from bookwyrm.status import create_generated_note
 from bookwyrm.status import delete_status
-from bookwyrm.remote_user import get_or_create_remote_user
 from bookwyrm.settings import DOMAIN
 from bookwyrm.utils import regex
 
@@ -61,9 +60,11 @@ def handle_remote_webfinger(query):
             return None
         data = response.json()
         for link in data['links']:
-            if link['rel'] == 'self':
+            if link.get('rel') == 'self':
                 try:
-                    user = get_or_create_remote_user(link['href'])
+                    user = activitypub.resolve_remote_id(
+                        models.User, link['href']
+                    )
                 except KeyError:
                     return None
     return user
diff --git a/bookwyrm/remote_user.py b/bookwyrm/remote_user.py
deleted file mode 100644
index 205b0893..00000000
--- a/bookwyrm/remote_user.py
+++ /dev/null
@@ -1,86 +0,0 @@
-''' manage remote users '''
-from urllib.parse import urlparse
-import requests
-
-from django.db import transaction
-
-from bookwyrm import activitypub, models
-from bookwyrm import status as status_builder
-from bookwyrm.tasks import app
-
-
-def get_or_create_remote_user(actor):
-    ''' look up a remote user or add them '''
-    try:
-        return models.User.objects.get(remote_id=actor)
-    except models.User.DoesNotExist:
-        pass
-
-    actor_parts = urlparse(actor)
-    with transaction.atomic():
-        user = activitypub.resolve_remote_id(models.User, actor)
-        user.federated_server = get_or_create_remote_server(actor_parts.netloc)
-        user.save()
-    if user.bookwyrm_user:
-        get_remote_reviews.delay(user.id)
-    return user
-
-
-def refresh_remote_user(user):
-    ''' get updated user data from its home instance '''
-    activitypub.resolve_remote_id(user.remote_id, refresh=True)
-
-
-@app.task
-def get_remote_reviews(user_id):
-    ''' ingest reviews by a new remote bookwyrm user '''
-    try:
-        user = models.User.objects.get(id=user_id)
-    except models.User.DoesNotExist:
-        return
-    outbox_page = user.outbox + '?page=true'
-    response = requests.get(
-        outbox_page,
-        headers={'Accept': 'application/activity+json'}
-    )
-    data = response.json()
-    # TODO: pagination?
-    for activity in data['orderedItems']:
-        status_builder.create_status(activity)
-
-
-def get_or_create_remote_server(domain):
-    ''' get info on a remote server '''
-    try:
-        return models.FederatedServer.objects.get(
-            server_name=domain
-        )
-    except models.FederatedServer.DoesNotExist:
-        pass
-
-    response = requests.get(
-        'https://%s/.well-known/nodeinfo' % domain,
-        headers={'Accept': 'application/activity+json'}
-    )
-
-    if response.status_code != 200:
-        return None
-
-    data = response.json()
-    try:
-        nodeinfo_url = data.get('links')[0].get('href')
-    except (TypeError, KeyError):
-        return None
-
-    response = requests.get(
-        nodeinfo_url,
-        headers={'Accept': 'application/activity+json'}
-    )
-    data = response.json()
-
-    server = models.FederatedServer.objects.create(
-        server_name=domain,
-        application_type=data['software']['name'],
-        application_version=data['software']['version'],
-    )
-    return server
diff --git a/bookwyrm/status.py b/bookwyrm/status.py
index 6a86209f..83a106e5 100644
--- a/bookwyrm/status.py
+++ b/bookwyrm/status.py
@@ -12,37 +12,6 @@ def delete_status(status):
     status.save()
 
 
-def create_status(activity):
-    ''' unfortunately, it's not QUITE as simple as deserializing it '''
-    # render the json into an activity object
-    serializer = activitypub.activity_objects[activity['type']]
-    activity = serializer(**activity)
-    try:
-        model = models.activity_models[activity.type]
-    except KeyError:
-        # not a type of status we are prepared to deserialize
-        return None
-
-    # ignore notes that aren't replies to known statuses
-    if activity.type == 'Note':
-        reply = models.Status.objects.filter(
-            remote_id=activity.inReplyTo
-        ).first()
-        if not reply:
-            return None
-
-    # look up books
-    book_urls = []
-    if hasattr(activity, 'inReplyToBook'):
-        book_urls.append(activity.inReplyToBook)
-    if hasattr(activity, 'tag'):
-        book_urls += [t['href'] for t in activity.tag if t['type'] == 'Book']
-    for remote_id in book_urls:
-        books_manager.get_or_create_book(remote_id)
-
-    return activity.to_model(model)
-
-
 def create_generated_note(user, content, mention_books=None, privacy='public'):
     ''' a note created by the app about user activity '''
     # sanitize input html
diff --git a/celerywyrm/celery.py b/celerywyrm/celery.py
index 361b76ec..a47aad32 100644
--- a/celerywyrm/celery.py
+++ b/celerywyrm/celery.py
@@ -24,4 +24,4 @@ app.autodiscover_tasks(['bookwyrm'], related_name='books_manager')
 app.autodiscover_tasks(['bookwyrm'], related_name='emailing')
 app.autodiscover_tasks(['bookwyrm'], related_name='goodreads_import')
 app.autodiscover_tasks(['bookwyrm'], related_name='incoming')
-app.autodiscover_tasks(['bookwyrm'], related_name='remote_user')
+app.autodiscover_tasks(['bookwyrm'], related_name='models.user')

From 76ce20a5e0a1b3b69fb367a31b9b6c258e4c9081 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 11:52:00 -0800
Subject: [PATCH 005/104] Fixes tests

---
 bookwyrm/tests/test_remote_user.py | 27 ---------------------------
 bookwyrm/tests/test_signing.py     | 10 +++++-----
 2 files changed, 5 insertions(+), 32 deletions(-)
 delete mode 100644 bookwyrm/tests/test_remote_user.py

diff --git a/bookwyrm/tests/test_remote_user.py b/bookwyrm/tests/test_remote_user.py
deleted file mode 100644
index febf9f67..00000000
--- a/bookwyrm/tests/test_remote_user.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import json
-import pathlib
-from django.test import TestCase
-
-from bookwyrm import models, remote_user
-
-
-class RemoteUser(TestCase):
-    ''' not too much going on in the books model but here we are '''
-    def setUp(self):
-        self.remote_user = models.User.objects.create_user(
-            'rat', 'rat@rat.com', 'ratword',
-            local=False,
-            remote_id='https://example.com/users/rat',
-            inbox='https://example.com/users/rat/inbox',
-            outbox='https://example.com/users/rat/outbox',
-        )
-        datafile = pathlib.Path(__file__).parent.joinpath(
-            'data/ap_user.json'
-        )
-        self.user_data = json.loads(datafile.read_bytes())
-
-
-    def test_get_remote_user(self):
-        actor = 'https://example.com/users/rat'
-        user = remote_user.get_or_create_remote_user(actor)
-        self.assertEqual(user, self.remote_user)
diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py
index a5039306..129a4333 100644
--- a/bookwyrm/tests/test_signing.py
+++ b/bookwyrm/tests/test_signing.py
@@ -107,7 +107,7 @@ class Signature(TestCase):
             status=200
         )
 
-        with patch('bookwyrm.remote_user.get_remote_reviews.delay') as _:
+        with patch('bookwyrm.models.user.get_remote_reviews.delay') as _:
             response = self.send_test_request(sender=self.fake_remote)
             self.assertEqual(response.status_code, 200)
 
@@ -145,7 +145,7 @@ class Signature(TestCase):
             json=data,
             status=200)
 
-        with patch('bookwyrm.remote_user.get_remote_reviews.delay') as _:
+        with patch('bookwyrm.models.user.get_remote_reviews.delay') as _:
             # Key correct:
             response = self.send_test_request(sender=self.fake_remote)
             self.assertEqual(response.status_code, 200)
@@ -177,7 +177,7 @@ class Signature(TestCase):
     @pytest.mark.integration
     def test_changed_data(self):
         '''Message data must match the digest header.'''
-        with patch('bookwyrm.remote_user.fetch_user_data') as _:
+        with patch('bookwyrm.activitypub.resolve_remote_id') as _:
             response = self.send_test_request(
                 self.mouse,
                 send_data=get_follow_data(self.mouse, self.cat))
@@ -185,7 +185,7 @@ class Signature(TestCase):
 
     @pytest.mark.integration
     def test_invalid_digest(self):
-        with patch('bookwyrm.remote_user.fetch_user_data') as _:
+        with patch('bookwyrm.activitypub.resolve_remote_id') as _:
             response = self.send_test_request(
                 self.mouse,
                 digest='SHA-256=AAAAAAAAAAAAAAAAAA')
@@ -194,7 +194,7 @@ class Signature(TestCase):
     @pytest.mark.integration
     def test_old_message(self):
         '''Old messages should be rejected to prevent replay attacks.'''
-        with patch('bookwyrm.remote_user.fetch_user_data') as _:
+        with patch('bookwyrm.activitypub.resolve_remote_id') as _:
             response = self.send_test_request(
                 self.mouse,
                 date=http_date(time.time() - 301)

From e99394e6f7c2cd732938f059c59ad1c030c0a5dc Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 12:25:53 -0800
Subject: [PATCH 006/104] User serializer to create follow request

---
 bookwyrm/incoming.py | 43 +++++++++++--------------------------------
 1 file changed, 11 insertions(+), 32 deletions(-)

diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index 6ce2ea0b..e9b0d019 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -115,26 +115,10 @@ def has_valid_signature(request, activity):
 @app.task
 def handle_follow(activity):
     ''' someone wants to follow a local user '''
-    # figure out who they want to follow -- not using get_or_create because
-    # we only care if you want to follow local users
     try:
-        to_follow = models.User.objects.get(remote_id=activity['object'])
-    except models.User.DoesNotExist:
-        # some rando, who cares
-        return
-    if not to_follow.local:
-        # just ignore follow alerts about other servers. maybe they should be
-        # handled. maybe they shouldn't be sent at all.
-        return
-
-    # figure out who the actor is
-    actor = activitypub.resolve_remote_id(models.User, activity['actor'])
-    try:
-        relationship = models.UserFollowRequest.objects.create(
-            user_subject=actor,
-            user_object=to_follow,
-            remote_id=activity['id']
-        )
+        relationship = activitypub.Follow(
+            **activity
+        ).to_model(models.UserFollowRequest)
     except django.db.utils.IntegrityError as err:
         if err.__cause__.diag.constraint_name != 'userfollowrequest_unique':
             raise
@@ -143,20 +127,15 @@ def handle_follow(activity):
         )
         # send the accept normally for a duplicate request
 
-    if not to_follow.manually_approves_followers:
-        status_builder.create_notification(
-            to_follow,
-            'FOLLOW',
-            related_user=actor
-        )
+    manually_approves = relationship.user_object.manually_approves_followers
+
+    status_builder.create_notification(
+        relationship.user_object,
+        'FOLLOW_REQUEST' if manually_approves else 'FOLLOW',
+        related_user=relationship.user_subject
+    )
+    if not manually_approves:
         outgoing.handle_accept(relationship)
-    else:
-        # Accept will be triggered manually
-        status_builder.create_notification(
-            to_follow,
-            'FOLLOW_REQUEST',
-            related_user=actor
-        )
 
 
 @app.task

From 421a13fda06de6dd4466130ecae52823f930eccd Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 13:00:04 -0800
Subject: [PATCH 007/104] automatically load authors and editions

---
 bookwyrm/activitypub/base_activity.py | 17 +++++++++++------
 bookwyrm/models/base_model.py         |  9 +++++++--
 bookwyrm/models/book.py               | 19 ++-----------------
 3 files changed, 20 insertions(+), 25 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 0672fbb8..620579a0 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -124,7 +124,7 @@ class ActivityObject:
                         formatted_value = timezone.make_aware(date_value)
                     except ValueError:
                         formatted_value = date_value
-                except ParserError:
+                except (ParserError, TypeError):
                     formatted_value = None
             elif isinstance(model_field, ForwardManyToOneDescriptor) and \
                     formatted_value:
@@ -182,12 +182,17 @@ class ActivityObject:
                 model = model_field.model
                 items = []
                 for link in values:
-                    # check that the Type matches the model (because Status
-                    # tags contain both user mentions and book tags)
-                    if not model.activity_serializer.type == link.get('type'):
-                        continue
+                    if isinstance(link, dict):
+                        # check that the Type matches the model (Status
+                        # tags contain both user mentions and book tags)
+                        if not model.activity_serializer.type == \
+                                link.get('type'):
+                            continue
+                        remote_id = link.get('href')
+                    else:
+                        remote_id = link
                     items.append(
-                        resolve_remote_id(model, link.get('href'))
+                        resolve_remote_id(model, remote_id)
                     )
                 getattr(instance, model_key).set(items)
 
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 29301fb5..877f094e 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -10,7 +10,9 @@ from Crypto.PublicKey import RSA
 from Crypto.Signature import pkcs1_15
 from Crypto.Hash import SHA256
 from django.db import models
-from django.db.models.fields.files import ImageFieldFile
+from django.db.models.fields.files import ImageFileDescriptor
+from django.db.models.fields.related_descriptors \
+        import ManyToManyDescriptor
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
@@ -72,13 +74,16 @@ class ActivitypubMixin:
                 # this field on the model isn't serialized
                 continue
             value = getattr(self, mapping.model_key)
+            model_field_type = getattr(self.__class__, mapping.model_key)
             if hasattr(value, 'remote_id'):
                 # this is probably a foreign key field, which we want to
                 # serialize as just the remote_id url reference
                 value = value.remote_id
+            elif isinstance(model_field_type, ManyToManyDescriptor):
+                value = [i.remote_id for i in value.all()]
             elif isinstance(value, datetime):
                 value = value.isoformat()
-            elif isinstance(value, ImageFieldFile):
+            elif isinstance(model_field_type, ImageFileDescriptor):
                 value = image_formatter(value)
 
             # run the custom formatter function set in the model
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index 8892119a..70eb8652 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -55,15 +55,10 @@ class Book(ActivitypubMixin, BookWyrmModel):
     published_date = models.DateTimeField(blank=True, null=True)
     objects = InheritanceManager()
 
-    @property
-    def ap_authors(self):
-        ''' the activitypub serialization should be a list of author ids '''
-        return [a.remote_id for a in self.authors.all()]
-
     activity_mappings = [
         ActivityMapping('id', 'remote_id'),
 
-        ActivityMapping('authors', 'ap_authors'),
+        ActivityMapping('authors', 'authors'),
         ActivityMapping('firstPublishedDate', 'firstpublished_date'),
         ActivityMapping('publishedDate', 'published_date'),
 
@@ -91,7 +86,7 @@ class Book(ActivitypubMixin, BookWyrmModel):
         ActivityMapping('publishers', 'publishers'),
 
         ActivityMapping('lccn', 'lccn'),
-        ActivityMapping('editions', 'editions_path'),
+        ActivityMapping('editions', 'editions'),
         ActivityMapping('cover', 'cover'),
     ]
 
@@ -127,16 +122,6 @@ class Work(OrderedCollectionPageMixin, Book):
         null=True
     )
 
-    @property
-    def editions_path(self):
-        ''' it'd be nice to serialize the edition instead but, recursion '''
-        default = self.default_edition
-        ed_list = [
-            e.remote_id for e in \
-                    self.edition_set.filter(~Q(id=default.id)).all()
-        ]
-        return [default.remote_id] + ed_list
-
 
     def to_edition_list(self, **kwargs):
         ''' activitypub serialization for this work's editions '''

From 0a8ef98854bed6df5bf5e7e3ef7c5b67e5cbb9b0 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 13:14:18 -0800
Subject: [PATCH 008/104] use localized remote_ids for books

---
 bookwyrm/models/book.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index 70eb8652..d5e811bb 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -2,7 +2,6 @@
 import re
 
 from django.db import models
-from django.db.models import Q
 from django.utils import timezone
 from model_utils.managers import InheritanceManager
 
@@ -94,10 +93,17 @@ class Book(ActivitypubMixin, BookWyrmModel):
         ''' can't be abstract for query reasons, but you shouldn't USE it '''
         if not isinstance(self, Edition) and not isinstance(self, Work):
             raise ValueError('Books should be added as Editions or Works')
+
         if self.id and not self.remote_id:
             self.remote_id = self.get_remote_id()
 
-        super().save(*args, **kwargs)
+        if not self.id:
+            # force set the remote id to a local version
+            self.origin_id = self.remote_id
+            saved = super().save(*args, **kwargs)
+            saved.remote_id = self.get_remote_id()
+            return saved.save()
+        return super().save(*args, **kwargs)
 
     def get_remote_id(self):
         ''' editions and works both use "book" instead of model_name '''

From 7ed2e310c02d3b175d2a3566374f09c313cba350 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 13:40:09 -0800
Subject: [PATCH 009/104] User origin ids for books and authors

---
 bookwyrm/activitypub/base_activity.py | 10 +++++++---
 bookwyrm/models/author.py             | 23 +++++++++++++++++++++++
 bookwyrm/models/book.py               |  9 +++++----
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 620579a0..b9b6b9b4 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -222,14 +222,18 @@ class ActivityObject:
 
 def resolve_remote_id(model, remote_id, refresh=False):
     ''' look up the remote_id in the database or load it remotely '''
-    result = model.objects
+    objects = model.objects
     if hasattr(model.objects, 'select_subclasses'):
-        result = result.select_subclasses()
+        objects = objects.select_subclasses()
 
     # first, check for an existing copy in the database
-    result = result.filter(
+    result = objects.filter(
         remote_id=remote_id
     ).first()
+    if not result and hasattr(model, 'origin_id'):
+        result = objects.filter(
+            origin_id=remote_id
+        ).first()
     if result and not refresh:
         return result
 
diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py
index 1d701797..0a934e45 100644
--- a/bookwyrm/models/author.py
+++ b/bookwyrm/models/author.py
@@ -1,8 +1,12 @@
 ''' database schema for info about authors '''
+from uuid import uuid4
+import re
+
 from django.db import models
 from django.utils import timezone
 
 from bookwyrm import activitypub
+from bookwyrm.settings import DOMAIN
 from bookwyrm.utils.fields import ArrayField
 
 from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel
@@ -27,6 +31,25 @@ class Author(ActivitypubMixin, BookWyrmModel):
     )
     bio = models.TextField(null=True, blank=True)
 
+    def save(self, *args, **kwargs):
+        ''' can't be abstract for query reasons, but you shouldn't USE it '''
+        if self.id and not self.remote_id:
+            self.remote_id = self.get_remote_id()
+
+        if not self.id:
+            # force set the remote id to a local version
+            self.origin_id = self.remote_id
+            self.remote_id = self.get_remote_id()
+        return super().save(*args, **kwargs)
+
+    def get_remote_id(self):
+        ''' editions and works both use "book" instead of model_name '''
+        uuid = str(uuid4())[:8]
+        # in Book, the title is used to make the url more readable, but
+        # since an author's name can change, I didn't want to lock in a
+        # potential deadname (or maiden name) in the urk.
+        return 'https://%s/author/%s' % (DOMAIN, uuid)
+
     @property
     def display_name(self):
         ''' Helper to return a displayable name'''
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index d5e811bb..d3460aac 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -1,4 +1,5 @@
 ''' database schema for books and shelves '''
+from uuid import uuid4
 import re
 
 from django.db import models
@@ -100,14 +101,14 @@ class Book(ActivitypubMixin, BookWyrmModel):
         if not self.id:
             # force set the remote id to a local version
             self.origin_id = self.remote_id
-            saved = super().save(*args, **kwargs)
-            saved.remote_id = self.get_remote_id()
-            return saved.save()
+            self.remote_id = self.get_remote_id()
         return super().save(*args, **kwargs)
 
     def get_remote_id(self):
         ''' editions and works both use "book" instead of model_name '''
-        return 'https://%s/book/%d' % (DOMAIN, self.id)
+        uuid = str(uuid4())[:8]
+        clean_title = re.sub(r'[\W-]', '', self.title.replace(' ', '-')).lower()
+        return 'https://%s/author/%s-%s' % (DOMAIN, clean_title, uuid)
 
     def __repr__(self):
         return "<{} key={!r} title={!r}>".format(

From 72c7829bab4a997fec24be2b9d1925a21bd6e3bb Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 17:29:03 -0800
Subject: [PATCH 010/104] Preserve remote_id syntax for authors and books

---
 bookwyrm/activitypub/base_activity.py | 114 +++++++++++++-------------
 bookwyrm/models/author.py             |  12 +--
 bookwyrm/models/base_model.py         |  10 ++-
 bookwyrm/models/book.py               |  23 ++----
 bookwyrm/views.py                     |   2 +-
 5 files changed, 72 insertions(+), 89 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index b9b6b9b4..4c73ab97 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -6,9 +6,8 @@ from uuid import uuid4
 import dateutil.parser
 from dateutil.parser import ParserError
 from django.core.files.base import ContentFile
-from django.db import transaction
 from django.db.models.fields.related_descriptors \
-        import ForwardManyToOneDescriptor, ManyToManyDescriptor, \
+    import ForwardManyToOneDescriptor, ManyToManyDescriptor, \
         ReverseManyToOneDescriptor
 from django.db.models.fields import DateTimeField
 from django.db.models.fields.files import ImageFileDescriptor
@@ -16,8 +15,6 @@ from django.db.models.query_utils import DeferredAttribute
 from django.utils import timezone
 import requests
 
-from bookwyrm import models
-
 
 class ActivitySerializerError(ValueError):
     ''' routine problems serializing activitypub json '''
@@ -135,7 +132,7 @@ class ActivityObject:
                     # if the AP field is a serialized object (as in Add)
                     remote_id = formatted_value['id']
                 else:
-                    # if the AP field is just a remote_id (as in every other case)
+                    # if the field is just a remote_id (as in every other case)
                     remote_id = formatted_value
                 reference = resolve_remote_id(fk_model, remote_id)
                 mapped_fields[mapping.model_key] = reference
@@ -153,62 +150,67 @@ class ActivityObject:
                     formatted_value = None
                 mapped_fields[mapping.model_key] = formatted_value
 
-        with transaction.atomic():
-            if instance:
-                # updating an existing model instance
-                for k, v in mapped_fields.items():
-                    setattr(instance, k, v)
-                instance.save()
-            else:
-                # creating a new model instance
-                instance = model.objects.create(**mapped_fields)
+        if instance:
+            # updating an existing model instance
+            for k, v in mapped_fields.items():
+                setattr(instance, k, v)
+            instance.save()
+        else:
+            # creating a new model instance
+            instance = model.objects.create(**mapped_fields)
+            print('CREATING')
+            print(instance)
+            print(instance.id)
 
-            # --- these are all fields that can't be saved until after the
-            # instance has an id (after it's been saved). ---------------#
+        # --- these are all fields that can't be saved until after the
+        # instance has an id (after it's been saved). ---------------#
 
-            # add images
-            for (model_key, value) in image_fields.items():
-                formatted_value = image_formatter(value)
-                if not formatted_value:
-                    continue
-                getattr(instance, model_key).save(*formatted_value, save=True)
+        # add images
+        for (model_key, value) in image_fields.items():
+            formatted_value = image_formatter(value)
+            if not formatted_value:
+                continue
+            getattr(instance, model_key).save(*formatted_value, save=True)
 
-            # add many to many fields
-            for (model_key, values) in many_to_many_fields.items():
-                # mention books, mention users
-                if values == MISSING:
-                    continue
-                model_field = getattr(instance, model_key)
-                model = model_field.model
-                items = []
-                for link in values:
-                    if isinstance(link, dict):
-                        # check that the Type matches the model (Status
-                        # tags contain both user mentions and book tags)
-                        if not model.activity_serializer.type == \
-                                link.get('type'):
-                            continue
-                        remote_id = link.get('href')
-                    else:
-                        remote_id = link
-                    items.append(
-                        resolve_remote_id(model, remote_id)
-                    )
-                getattr(instance, model_key).set(items)
+        # add many to many fields
+        for (model_key, values) in many_to_many_fields.items():
+            # mention books, mention users
+            if values == MISSING:
+                continue
+            model_field = getattr(instance, model_key)
+            model = model_field.model
+            items = []
+            for link in values:
+                if isinstance(link, dict):
+                    # check that the Type matches the model (Status
+                    # tags contain both user mentions and book tags)
+                    if not model.activity_serializer.type == \
+                            link.get('type'):
+                        continue
+                    remote_id = link.get('href')
+                else:
+                    remote_id = link
+                items.append(
+                    resolve_remote_id(model, remote_id)
+                )
+            getattr(instance, model_key).set(items)
 
-            # add one to many fields
-            for (model_key, values) in one_to_many_fields.items():
-                if values == MISSING:
-                    continue
-                model_field = getattr(instance, model_key)
-                model = model_field.model
-                for item in values:
+        # add one to many fields
+        for (model_key, values) in one_to_many_fields.items():
+            if values == MISSING:
+                continue
+            model_field = getattr(instance, model_key)
+            model = model_field.model
+            for item in values:
+                if isinstance(item, str):
+                    print(model)
+                    item = resolve_remote_id(model, item)
+                else:
                     item = model.activity_serializer(**item)
-                    field_name = instance.__class__.__name__.lower()
-                    with transaction.atomic():
-                        item = item.to_model(model)
-                        setattr(item, field_name, instance)
-                        item.save()
+                    item = item.to_model(model)
+                field_name = instance.__class__.__name__.lower()
+                setattr(item, field_name, instance)
+                item.save()
 
         return instance
 
diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py
index 0a934e45..fdaf73d9 100644
--- a/bookwyrm/models/author.py
+++ b/bookwyrm/models/author.py
@@ -1,7 +1,4 @@
 ''' database schema for info about authors '''
-from uuid import uuid4
-import re
-
 from django.db import models
 from django.utils import timezone
 
@@ -37,18 +34,13 @@ class Author(ActivitypubMixin, BookWyrmModel):
             self.remote_id = self.get_remote_id()
 
         if not self.id:
-            # force set the remote id to a local version
             self.origin_id = self.remote_id
-            self.remote_id = self.get_remote_id()
+            self.remote_id = None
         return super().save(*args, **kwargs)
 
     def get_remote_id(self):
         ''' editions and works both use "book" instead of model_name '''
-        uuid = str(uuid4())[:8]
-        # in Book, the title is used to make the url more readable, but
-        # since an author's name can change, I didn't want to lock in a
-        # potential deadname (or maiden name) in the urk.
-        return 'https://%s/author/%s' % (DOMAIN, uuid)
+        return 'https://%s/author/%s' % (DOMAIN, self.id)
 
     @property
     def display_name(self):
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 877f094e..71f3c7b7 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -12,7 +12,7 @@ from Crypto.Hash import SHA256
 from django.db import models
 from django.db.models.fields.files import ImageFileDescriptor
 from django.db.models.fields.related_descriptors \
-        import ManyToManyDescriptor
+        import ManyToManyDescriptor, ReverseManyToOneDescriptor
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
@@ -74,16 +74,18 @@ class ActivitypubMixin:
                 # this field on the model isn't serialized
                 continue
             value = getattr(self, mapping.model_key)
-            model_field_type = getattr(self.__class__, mapping.model_key)
+            model_field = getattr(self.__class__, mapping.model_key)
+            print(mapping.model_key, type(model_field))
             if hasattr(value, 'remote_id'):
                 # this is probably a foreign key field, which we want to
                 # serialize as just the remote_id url reference
                 value = value.remote_id
-            elif isinstance(model_field_type, ManyToManyDescriptor):
+            elif isinstance(model_field, ManyToManyDescriptor) or \
+                     isinstance(model_field, ReverseManyToOneDescriptor):
                 value = [i.remote_id for i in value.all()]
             elif isinstance(value, datetime):
                 value = value.isoformat()
-            elif isinstance(model_field_type, ImageFileDescriptor):
+            elif isinstance(model_field, ImageFileDescriptor):
                 value = image_formatter(value)
 
             # run the custom formatter function set in the model
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index d3460aac..cc377d21 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -1,5 +1,4 @@
 ''' database schema for books and shelves '''
-from uuid import uuid4
 import re
 
 from django.db import models
@@ -99,16 +98,13 @@ class Book(ActivitypubMixin, BookWyrmModel):
             self.remote_id = self.get_remote_id()
 
         if not self.id:
-            # force set the remote id to a local version
             self.origin_id = self.remote_id
-            self.remote_id = self.get_remote_id()
+            self.remote_id = None
         return super().save(*args, **kwargs)
 
     def get_remote_id(self):
         ''' editions and works both use "book" instead of model_name '''
-        uuid = str(uuid4())[:8]
-        clean_title = re.sub(r'[\W-]', '', self.title.replace(' ', '-')).lower()
-        return 'https://%s/author/%s-%s' % (DOMAIN, clean_title, uuid)
+        return 'https://%s/book/%d' % (DOMAIN, self.id)
 
     def __repr__(self):
         return "<{} key={!r} title={!r}>".format(
@@ -129,17 +125,6 @@ class Work(OrderedCollectionPageMixin, Book):
         null=True
     )
 
-
-    def to_edition_list(self, **kwargs):
-        ''' activitypub serialization for this work's editions '''
-        remote_id = self.remote_id + '/editions'
-        return self.to_ordered_collection(
-            self.edition_set,
-            remote_id=remote_id,
-            **kwargs
-        )
-
-
     activity_serializer = activitypub.Work
 
 
@@ -161,7 +146,8 @@ class Edition(Book):
         through='ShelfBook',
         through_fields=('book', 'shelf')
     )
-    parent_work = models.ForeignKey('Work', on_delete=models.PROTECT, null=True)
+    parent_work = models.ForeignKey(
+        'Work', on_delete=models.PROTECT, null=True, related_name='editions')
 
     activity_serializer = activitypub.Edition
 
@@ -175,6 +161,7 @@ class Edition(Book):
         return super().save(*args, **kwargs)
 
 
+
 def isbn_10_to_13(isbn_10):
     ''' convert an isbn 10 into an isbn 13 '''
     isbn_10 = re.sub(r'[^0-9X]', '', isbn_10)
diff --git a/bookwyrm/views.py b/bookwyrm/views.py
index c6992e6b..bd62902c 100644
--- a/bookwyrm/views.py
+++ b/bookwyrm/views.py
@@ -540,7 +540,7 @@ def book_page(request, book_id):
         return HttpResponseNotFound()
 
     reviews = models.Review.objects.filter(
-        book__in=work.edition_set.all(),
+        book__in=work.editions.all(),
     )
     # all reviews for the book
     reviews = get_activity_feed(request.user, 'federated', model=reviews)

From dfd730757d01091500f1d819807945de4aac927c Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 17:41:57 -0800
Subject: [PATCH 011/104] handle unset default editions

---
 bookwyrm/activitypub/book.py                  |  1 +
 .../migrations/0018_auto_20201128_2142.py     | 24 +++++++++++++++++++
 bookwyrm/models/book.py                       |  5 ++++
 bookwyrm/views.py                             |  4 ++--
 4 files changed, 32 insertions(+), 2 deletions(-)
 create mode 100644 bookwyrm/migrations/0018_auto_20201128_2142.py

diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py
index 8a6b88d9..ae9c334d 100644
--- a/bookwyrm/activitypub/book.py
+++ b/bookwyrm/activitypub/book.py
@@ -49,6 +49,7 @@ class Edition(Book):
 class Work(Book):
     ''' work instance of a book object '''
     lccn: str = ''
+    defaultEdition: str = ''
     editions: List[str]
     type: str = 'Work'
 
diff --git a/bookwyrm/migrations/0018_auto_20201128_2142.py b/bookwyrm/migrations/0018_auto_20201128_2142.py
new file mode 100644
index 00000000..86d02870
--- /dev/null
+++ b/bookwyrm/migrations/0018_auto_20201128_2142.py
@@ -0,0 +1,24 @@
+# Generated by Django 3.0.7 on 2020-11-28 21:42
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0017_auto_20201128_1849'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='edition',
+            name='parent_work',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=models.CharField(max_length=100, unique=True),
+        ),
+    ]
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index cc377d21..ffc9dd8a 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -86,6 +86,7 @@ class Book(ActivitypubMixin, BookWyrmModel):
 
         ActivityMapping('lccn', 'lccn'),
         ActivityMapping('editions', 'editions'),
+        ActivityMapping('defaultEdition', 'default_edition'),
         ActivityMapping('cover', 'cover'),
     ]
 
@@ -125,6 +126,10 @@ class Work(OrderedCollectionPageMixin, Book):
         null=True
     )
 
+    def get_default_edition(self):
+        ''' in case the default edition is not set '''
+        return self.default_edition or self.editions.first()
+
     activity_serializer = activitypub.Work
 
 
diff --git a/bookwyrm/views.py b/bookwyrm/views.py
index bd62902c..4f3e3f1c 100644
--- a/bookwyrm/views.py
+++ b/bookwyrm/views.py
@@ -531,7 +531,7 @@ def book_page(request, book_id):
         return JsonResponse(book.to_activity(), encoder=ActivityEncoder)
 
     if isinstance(book, models.Work):
-        book = book.default_edition
+        book = book.get_default_edition()
     if not book:
         return HttpResponseNotFound()
 
@@ -646,7 +646,7 @@ def author_page(request, author_id):
     data = {
         'title': author.name,
         'author': author,
-        'books': [b.default_edition for b in books],
+        'books': [b.get_default_edition() for b in books],
     }
     return TemplateResponse(request, 'author.html', data)
 

From b4fe9f160f47d1936a87fde0715187076ad12f4c Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 18:11:52 -0800
Subject: [PATCH 012/104] Correctly look up books by remote/origin id

---
 bookwyrm/activitypub/base_activity.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 4c73ab97..5b4d6335 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -91,10 +91,7 @@ class ActivityObject:
 
         # check for an existing instance, if we're not updating a known obj
         if not instance:
-            try:
-                return model.objects.get(remote_id=self.id)
-            except model.DoesNotExist:
-                pass
+            instance = find_existing_by_remote_id(model, self.id)
 
         model_fields = [m.name for m in model._meta.get_fields()]
         mapped_fields = {}
@@ -123,8 +120,9 @@ class ActivityObject:
                         formatted_value = date_value
                 except (ParserError, TypeError):
                     formatted_value = None
-            elif isinstance(model_field, ForwardManyToOneDescriptor) and \
-                    formatted_value:
+            elif isinstance(model_field, ForwardManyToOneDescriptor):
+                if not formatted_value:
+                    continue
                 # foreign key remote id reolver (work on Edition, for example)
                 fk_model = model_field.field.related_model
                 if isinstance(formatted_value, dict) and \
@@ -158,9 +156,6 @@ class ActivityObject:
         else:
             # creating a new model instance
             instance = model.objects.create(**mapped_fields)
-            print('CREATING')
-            print(instance)
-            print(instance.id)
 
         # --- these are all fields that can't be saved until after the
         # instance has an id (after it's been saved). ---------------#
@@ -203,7 +198,6 @@ class ActivityObject:
             model = model_field.model
             for item in values:
                 if isinstance(item, str):
-                    print(model)
                     item = resolve_remote_id(model, item)
                 else:
                     item = model.activity_serializer(**item)
@@ -222,8 +216,8 @@ class ActivityObject:
         return data
 
 
-def resolve_remote_id(model, remote_id, refresh=False):
-    ''' look up the remote_id in the database or load it remotely '''
+def find_existing_by_remote_id(model, remote_id):
+    ''' check for an existing instance of this id in the db '''
     objects = model.objects
     if hasattr(model.objects, 'select_subclasses'):
         objects = objects.select_subclasses()
@@ -232,10 +226,17 @@ def resolve_remote_id(model, remote_id, refresh=False):
     result = objects.filter(
         remote_id=remote_id
     ).first()
+
     if not result and hasattr(model, 'origin_id'):
         result = objects.filter(
             origin_id=remote_id
         ).first()
+    return result
+
+
+def resolve_remote_id(model, remote_id, refresh=False):
+    ''' look up the remote_id in the database or load it remotely '''
+    result = find_existing_by_remote_id(model, remote_id)
     if result and not refresh:
         return result
 

From 1789b091d6d08e5c9c56e04238d2a94ed5f1af72 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 18:23:34 -0800
Subject: [PATCH 013/104] not all that better way to distinguish add book to
 shelf vs tag

---
 bookwyrm/incoming.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index e9b0d019..1978e0fe 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -293,11 +293,11 @@ def handle_unboost(activity):
 @app.task
 def handle_add(activity):
     ''' putting a book on a shelf '''
-    # TODO absofuckinglutely not an acceptable solution
-    if 'tag' in activity['id']:
-        activitypub.AddBook(**activity).to_model(models.Tag)
-    else:
+    #this is janky as heck but I haven't thought of a better solution
+    try:
         activitypub.AddBook(**activity).to_model(models.ShelfBook)
+    except activitypub.ActivitySerializerError:
+        activitypub.AddBook(**activity).to_model(models.Tag)
 
 
 @app.task

From 34e8fb3e5caf0bc4545d1d4bb72da38ee6a6d7aa Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 18:28:09 -0800
Subject: [PATCH 014/104] style fix and removing stray print statement

---
 bookwyrm/models/base_model.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 71f3c7b7..f5f244c5 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -75,13 +75,12 @@ class ActivitypubMixin:
                 continue
             value = getattr(self, mapping.model_key)
             model_field = getattr(self.__class__, mapping.model_key)
-            print(mapping.model_key, type(model_field))
             if hasattr(value, 'remote_id'):
                 # this is probably a foreign key field, which we want to
                 # serialize as just the remote_id url reference
                 value = value.remote_id
-            elif isinstance(model_field, ManyToManyDescriptor) or \
-                     isinstance(model_field, ReverseManyToOneDescriptor):
+            elif isinstance(model_field, \
+                    (ManyToManyDescriptor, ReverseManyToOneDescriptor)):
                 value = [i.remote_id for i in value.all()]
             elif isinstance(value, datetime):
                 value = value.isoformat()

From 9d84346d3c1f825a9c9b74ecc6aef7cb8be1827e Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 18:46:50 -0800
Subject: [PATCH 015/104] remove need for get_or_create_book

---
 bookwyrm/connectors/bookwyrm_connector.py | 86 ++++++-----------------
 bookwyrm/view_actions.py                  |  4 +-
 2 files changed, 23 insertions(+), 67 deletions(-)

diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py
index 1bc81450..74fce0ac 100644
--- a/bookwyrm/connectors/bookwyrm_connector.py
+++ b/bookwyrm/connectors/bookwyrm_connector.py
@@ -1,83 +1,37 @@
 ''' using another bookwyrm instance as a source of book data '''
-from django.db import transaction
-
 from bookwyrm import activitypub, models
 from .abstract_connector import AbstractConnector, SearchResult
-from .abstract_connector import get_data
 
 
 class Connector(AbstractConnector):
-    ''' interact with other instances '''
-
-    def update_from_mappings(self, obj, data, mappings):
-        ''' serialize book data into a model '''
-        if self.is_work_data(data):
-            work_data = activitypub.Work(**data)
-            return work_data.to_model(models.Work, instance=obj)
-        edition_data = activitypub.Edition(**data)
-        return edition_data.to_model(models.Edition, instance=obj)
-
-
-    def get_remote_id_from_data(self, data):
-        return data.get('id')
-
-
-    def is_work_data(self, data):
-        return data.get('type') == 'Work'
-
-
-    def get_edition_from_work_data(self, data):
-        ''' we're served a list of edition urls '''
-        path = data['editions'][0]
-        return get_data(path)
-
-
-    def get_work_from_edition_date(self, data):
-        return get_data(data['work'])
-
-
-    def get_authors_from_data(self, data):
-        ''' load author data '''
-        for author_id in data.get('authors', []):
-            try:
-                yield models.Author.objects.get(origin_id=author_id)
-            except models.Author.DoesNotExist:
-                pass
-            data = get_data(author_id)
-            author_data = activitypub.Author(**data)
-            author = author_data.to_model(models.Author)
-            yield author
-
-
-    def get_cover_from_data(self, data):
-        pass
+    ''' this is basically just for search '''
 
+    def get_or_create_book(self, remote_id):
+        return activitypub.resolve_remote_id(models.Edition, remote_id)
 
     def parse_search_data(self, data):
         return data
 
-
     def format_search_result(self, search_result):
         return SearchResult(**search_result)
 
+    def get_remote_id_from_data(self, data):
+        pass
+
+    def is_work_data(self, data):
+        pass
+
+    def get_edition_from_work_data(self, data):
+        pass
+
+    def get_work_from_edition_date(self, data):
+        pass
+
+    def get_cover_from_data(self, data):
+        pass
 
     def expand_book_data(self, book):
-        work = book
-        # go from the edition to the work, if necessary
-        if isinstance(book, models.Edition):
-            work = book.parent_work
+        pass
 
-        # it may be that we actually want to request this url
-        editions_url = '%s/editions?page=true' % work.remote_id
-        edition_options = get_data(editions_url)
-        for edition_data in edition_options['orderedItems']:
-            with transaction.atomic():
-                edition = self.create_book(
-                    edition_data['id'],
-                    edition_data,
-                    models.Edition
-                )
-                edition.parent_work = work
-                edition.save()
-            if not edition.authors.exists() and work.authors.exists():
-                edition.authors.set(work.authors.all())
+    def get_authors_from_data(self, data):
+        pass
diff --git a/bookwyrm/view_actions.py b/bookwyrm/view_actions.py
index 28dd8b87..29dc6406 100644
--- a/bookwyrm/view_actions.py
+++ b/bookwyrm/view_actions.py
@@ -219,7 +219,9 @@ def edit_profile(request):
 def resolve_book(request):
     ''' figure out the local path to a book from a remote_id '''
     remote_id = request.POST.get('remote_id')
-    book = books_manager.get_or_create_book(remote_id)
+    connector = books_manager.get_or_create_connector(remote_id)
+    book = connector.get_or_create_book(remote_id)
+
     return redirect('/book/%d' % book.id)
 
 

From c9433a3c7e937f159113b900202544ef3ec1f6ea Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 18:56:28 -0800
Subject: [PATCH 016/104] Simplify bookwyrm connector abstract

---
 bookwyrm/connectors/abstract_connector.py | 96 ++++++++++++-----------
 bookwyrm/connectors/bookwyrm_connector.py | 25 +-----
 bookwyrm/connectors/openlibrary.py        |  4 +-
 3 files changed, 54 insertions(+), 71 deletions(-)

diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index d709b075..0cf4a011 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -16,20 +16,13 @@ class ConnectorException(HTTPError):
     ''' when the connector can't do what was asked '''
 
 
-class AbstractConnector(ABC):
-    ''' generic book data connector '''
-
+class AbstractMinimalConnector(ABC):
+    ''' just the bare bones, for other bookwyrm instances '''
     def __init__(self, identifier):
         # load connector settings
         info = models.Connector.objects.get(identifier=identifier)
         self.connector = info
 
-        self.key_mappings = []
-
-        # fields we want to look for in book data to copy over
-        # title we handle separately.
-        self.book_mappings = []
-
         # the things in the connector model to copy over
         self_fields = [
             'base_url',
@@ -44,15 +37,6 @@ class AbstractConnector(ABC):
         for field in self_fields:
             setattr(self, field, getattr(info, field))
 
-
-    def is_available(self):
-        ''' check if you're allowed to use this connector '''
-        if self.max_query_count is not None:
-            if self.connector.query_count >= self.max_query_count:
-                return False
-        return True
-
-
     def search(self, query, min_confidence=None):
         ''' free text search '''
         resp = requests.get(
@@ -70,9 +54,40 @@ class AbstractConnector(ABC):
             results.append(self.format_search_result(doc))
         return results
 
-
+    @abstractmethod
     def get_or_create_book(self, remote_id):
         ''' pull up a book record by whatever means possible '''
+
+    @abstractmethod
+    def parse_search_data(self, data):
+        ''' turn the result json from a search into a list '''
+
+    @abstractmethod
+    def format_search_result(self, search_result):
+        ''' create a SearchResult obj from json '''
+
+
+class AbstractConnector(AbstractMinimalConnector):
+    ''' generic book data connector '''
+    def __init__(self, identifier):
+        super().__init__(identifier)
+
+        self.key_mappings = []
+
+        # fields we want to look for in book data to copy over
+        # title we handle separately.
+        self.book_mappings = []
+
+
+    def is_available(self):
+        ''' check if you're allowed to use this connector '''
+        if self.max_query_count is not None:
+            if self.connector.query_count >= self.max_query_count:
+                return False
+        return True
+
+
+    def get_or_create_book(self, remote_id):
         # try to load the book
         book = models.Book.objects.select_subclasses().filter(
             origin_id=remote_id
@@ -157,7 +172,7 @@ class AbstractConnector(ABC):
 
     def update_book_from_data(self, book, data, update_cover=True):
         ''' for creating a new book or syncing with data '''
-        book = self.update_from_mappings(book, data, self.book_mappings)
+        book = update_from_mappings(book, data, self.book_mappings)
 
         author_text = []
         for author in self.get_authors_from_data(data):
@@ -246,39 +261,28 @@ class AbstractConnector(ABC):
     def get_cover_from_data(self, data):
         ''' load cover '''
 
-
-    @abstractmethod
-    def parse_search_data(self, data):
-        ''' turn the result json from a search into a list '''
-
-
-    @abstractmethod
-    def format_search_result(self, search_result):
-        ''' create a SearchResult obj from json '''
-
-
     @abstractmethod
     def expand_book_data(self, book):
         ''' get more info on a book '''
 
 
-    def update_from_mappings(self, obj, data, mappings):
-        ''' assign data to model with mappings '''
-        for mapping in mappings:
-            # check if this field is present in the data
-            value = data.get(mapping.remote_field)
-            if not value:
-                continue
+def update_from_mappings(obj, data, mappings):
+    ''' assign data to model with mappings '''
+    for mapping in mappings:
+        # check if this field is present in the data
+        value = data.get(mapping.remote_field)
+        if not value:
+            continue
 
-            # extract the value in the right format
-            try:
-                value = mapping.formatter(value)
-            except:
-                continue
+        # extract the value in the right format
+        try:
+            value = mapping.formatter(value)
+        except:
+            continue
 
-            # assign the formatted value to the model
-            obj.__setattr__(mapping.local_field, value)
-        return obj
+        # assign the formatted value to the model
+        obj.__setattr__(mapping.local_field, value)
+    return obj
 
 
 def get_date(date_string):
diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py
index 74fce0ac..e4d32fd3 100644
--- a/bookwyrm/connectors/bookwyrm_connector.py
+++ b/bookwyrm/connectors/bookwyrm_connector.py
@@ -1,9 +1,9 @@
 ''' using another bookwyrm instance as a source of book data '''
 from bookwyrm import activitypub, models
-from .abstract_connector import AbstractConnector, SearchResult
+from .abstract_connector import AbstractMinimalConnector, SearchResult
 
 
-class Connector(AbstractConnector):
+class Connector(AbstractMinimalConnector):
     ''' this is basically just for search '''
 
     def get_or_create_book(self, remote_id):
@@ -14,24 +14,3 @@ class Connector(AbstractConnector):
 
     def format_search_result(self, search_result):
         return SearchResult(**search_result)
-
-    def get_remote_id_from_data(self, data):
-        pass
-
-    def is_work_data(self, data):
-        pass
-
-    def get_edition_from_work_data(self, data):
-        pass
-
-    def get_work_from_edition_date(self, data):
-        pass
-
-    def get_cover_from_data(self, data):
-        pass
-
-    def expand_book_data(self, book):
-        pass
-
-    def get_authors_from_data(self, data):
-        pass
diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py
index 5e18616d..90bd7f28 100644
--- a/bookwyrm/connectors/openlibrary.py
+++ b/bookwyrm/connectors/openlibrary.py
@@ -7,7 +7,7 @@ from django.core.files.base import ContentFile
 from bookwyrm import models
 from .abstract_connector import AbstractConnector, SearchResult, Mapping
 from .abstract_connector import ConnectorException
-from .abstract_connector import get_date, get_data
+from .abstract_connector import get_date, get_data, update_from_mappings
 from .openlibrary_languages import languages
 
 
@@ -184,7 +184,7 @@ class Connector(AbstractConnector):
         data = get_data(url)
 
         author = models.Author(openlibrary_key=olkey)
-        author = self.update_from_mappings(author, data, self.author_mappings)
+        author = update_from_mappings(author, data, self.author_mappings)
         name = data.get('name')
         # TODO this is making some BOLD assumption
         if name:

From d8fdc664507b585f002330f9719a72656f909a81 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 19:03:37 -0800
Subject: [PATCH 017/104] removes outdated update book code

---
 bookwyrm/books_manager.py      |  6 ------
 bookwyrm/incoming.py           | 13 ++-----------
 bookwyrm/routine_book_tasks.py | 16 ----------------
 3 files changed, 2 insertions(+), 33 deletions(-)
 delete mode 100644 bookwyrm/routine_book_tasks.py

diff --git a/bookwyrm/books_manager.py b/bookwyrm/books_manager.py
index fb0a747e..3b865768 100644
--- a/bookwyrm/books_manager.py
+++ b/bookwyrm/books_manager.py
@@ -85,12 +85,6 @@ def first_search_result(query, min_confidence=0.1):
     return None
 
 
-def update_book(book, data=None):
-    ''' re-sync with the original data source '''
-    connector = load_connector(book.connector)
-    connector.update_book(book, data=data)
-
-
 def get_connectors():
     ''' load all connectors '''
     for info in models.Connector.objects.order_by('priority').all():
diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index 1978e0fe..017be19d 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -8,7 +8,7 @@ from django.http import HttpResponseBadRequest, HttpResponseNotFound
 from django.views.decorators.csrf import csrf_exempt
 import requests
 
-from bookwyrm import activitypub, books_manager, models, outgoing
+from bookwyrm import activitypub, models, outgoing
 from bookwyrm import status as status_builder
 from bookwyrm.tasks import app
 from bookwyrm.signatures import Signature
@@ -317,13 +317,4 @@ def handle_update_user(activity):
 @app.task
 def handle_update_book(activity):
     ''' a remote instance changed a book (Document) '''
-    document = activity['object']
-    # check if we have their copy and care about their updates
-    book = models.Book.objects.select_subclasses().filter(
-        remote_id=document['id'],
-        sync=True,
-    ).first()
-    if not book:
-        return
-
-    books_manager.update_book(book, data=document)
+    activitypub.Edition(**activity['object']).to_model(models.Edition)
diff --git a/bookwyrm/routine_book_tasks.py b/bookwyrm/routine_book_tasks.py
deleted file mode 100644
index eaa28d90..00000000
--- a/bookwyrm/routine_book_tasks.py
+++ /dev/null
@@ -1,16 +0,0 @@
-''' Routine tasks for keeping your library tidy '''
-from datetime import timedelta
-from django.utils import timezone
-from bookwyrm import books_manager
-from bookwyrm import models
-
-def sync_book_data():
-    ''' update books with any changes to their canonical source '''
-    expiry = timezone.now() - timedelta(days=1)
-    books = models.Edition.objects.filter(
-        sync=True,
-        last_sync_date__lte=expiry
-    ).all()
-    for book in books:
-        # TODO: create background tasks
-        books_manager.update_book(book)

From e9be31e9c159e6805860b035663d051e0c35f326 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 28 Nov 2020 19:05:12 -0800
Subject: [PATCH 018/104] merge migrations

---
 .../migrations/0016_auto_20201128_1804.py     | 24 ------------------
 ...128_1849.py => 0016_auto_20201129_0304.py} | 25 +++++++++++++++++--
 .../migrations/0018_auto_20201128_2142.py     | 24 ------------------
 3 files changed, 23 insertions(+), 50 deletions(-)
 delete mode 100644 bookwyrm/migrations/0016_auto_20201128_1804.py
 rename bookwyrm/migrations/{0017_auto_20201128_1849.py => 0016_auto_20201129_0304.py} (57%)
 delete mode 100644 bookwyrm/migrations/0018_auto_20201128_2142.py

diff --git a/bookwyrm/migrations/0016_auto_20201128_1804.py b/bookwyrm/migrations/0016_auto_20201128_1804.py
deleted file mode 100644
index 9becde82..00000000
--- a/bookwyrm/migrations/0016_auto_20201128_1804.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 3.0.7 on 2020-11-28 18:04
-
-import bookwyrm.utils.fields
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('bookwyrm', '0015_auto_20201128_0349'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='book',
-            name='subject_places',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
-        ),
-        migrations.AlterField(
-            model_name='book',
-            name='subjects',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
-        ),
-    ]
diff --git a/bookwyrm/migrations/0017_auto_20201128_1849.py b/bookwyrm/migrations/0016_auto_20201129_0304.py
similarity index 57%
rename from bookwyrm/migrations/0017_auto_20201128_1849.py
rename to bookwyrm/migrations/0016_auto_20201129_0304.py
index 722458b2..2bf820e1 100644
--- a/bookwyrm/migrations/0017_auto_20201128_1849.py
+++ b/bookwyrm/migrations/0016_auto_20201129_0304.py
@@ -1,5 +1,6 @@
-# Generated by Django 3.0.7 on 2020-11-28 18:49
+# Generated by Django 3.0.7 on 2020-11-29 03:04
 
+import bookwyrm.utils.fields
 from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
@@ -8,10 +9,30 @@ import django.db.models.deletion
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('bookwyrm', '0016_auto_20201128_1804'),
+        ('bookwyrm', '0015_auto_20201128_0349'),
     ]
 
     operations = [
+        migrations.AlterField(
+            model_name='book',
+            name='subject_places',
+            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subjects',
+            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='parent_work',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=models.CharField(max_length=100, unique=True),
+        ),
         migrations.AlterUniqueTogether(
             name='tag',
             unique_together=set(),
diff --git a/bookwyrm/migrations/0018_auto_20201128_2142.py b/bookwyrm/migrations/0018_auto_20201128_2142.py
deleted file mode 100644
index 86d02870..00000000
--- a/bookwyrm/migrations/0018_auto_20201128_2142.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 3.0.7 on 2020-11-28 21:42
-
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('bookwyrm', '0017_auto_20201128_1849'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='edition',
-            name='parent_work',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'),
-        ),
-        migrations.AlterField(
-            model_name='tag',
-            name='name',
-            field=models.CharField(max_length=100, unique=True),
-        ),
-    ]

From 9ddd60ce16d2eac044127af5c1587a162c0f96bb Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 29 Nov 2020 09:40:15 -0800
Subject: [PATCH 019/104] Fixes broadcast tests

---
 bookwyrm/activitypub/base_activity.py     | 22 +++++++---------------
 bookwyrm/connectors/__init__.py           |  1 +
 bookwyrm/connectors/abstract_connector.py | 11 +++++++++++
 bookwyrm/tests/test_broadcast.py          | 16 +++++++++-------
 4 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 5b4d6335..9944e368 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -15,6 +15,7 @@ from django.db.models.query_utils import DeferredAttribute
 from django.utils import timezone
 import requests
 
+from bookwyrm.connectors import ConnectorException, get_data, get_image
 
 class ActivitySerializerError(ValueError):
     ''' routine problems serializing activitypub json '''
@@ -242,20 +243,13 @@ def resolve_remote_id(model, remote_id, refresh=False):
 
     # load the data and create the object
     try:
-        response = requests.get(
-            remote_id,
-            headers={'Accept': 'application/json; charset=utf-8'},
-        )
-    except ConnectionError:
+        data = get_data(remote_id)
+    except (ConnectorException, ConnectionError):
         raise ActivitySerializerError(
             'Could not connect to host for remote_id in %s model: %s' % \
                 (model.__name__, remote_id))
-    if not response.ok:
-        raise ActivitySerializerError(
-            'Could not resolve remote_id in %s model: %s' % \
-                (model.__name__, remote_id))
 
-    item = model.activity_serializer(**response.json())
+    item = model.activity_serializer(**data)
     # if we're refreshing, "result" will be set and we'll update it
     return item.to_model(model, instance=result)
 
@@ -272,11 +266,9 @@ def image_formatter(image_slug):
         return None
     if not url:
         return None
-    try:
-        response = requests.get(url)
-    except ConnectionError:
-        return None
-    if not response.ok:
+
+    response = get_image(url)
+    if not response:
         return None
 
     image_name = str(uuid4()) + '.' + url.split('.')[-1]
diff --git a/bookwyrm/connectors/__init__.py b/bookwyrm/connectors/__init__.py
index b5d93b47..4eb91de4 100644
--- a/bookwyrm/connectors/__init__.py
+++ b/bookwyrm/connectors/__init__.py
@@ -1,3 +1,4 @@
 ''' bring connectors into the namespace '''
 from .settings import CONNECTORS
 from .abstract_connector import ConnectorException
+from .abstract_connector import get_data, get_image
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index 0cf4a011..4e756d8c 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -318,6 +318,17 @@ def get_data(url):
     return data
 
 
+def get_image(url):
+    ''' wrapper for requesting an image '''
+    try:
+        resp = requests.get(url)
+    except RequestError:
+        return None
+    if not resp.ok:
+        return None
+    return resp
+
+
 @dataclass
 class SearchResult:
     ''' standardized search result object '''
diff --git a/bookwyrm/tests/test_broadcast.py b/bookwyrm/tests/test_broadcast.py
index 1112b3fa..96faf892 100644
--- a/bookwyrm/tests/test_broadcast.py
+++ b/bookwyrm/tests/test_broadcast.py
@@ -1,3 +1,4 @@
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models, broadcast
@@ -37,13 +38,14 @@ class Book(TestCase):
             'joe', 'joe@mouse.mouse', 'jeoword')
         self.user.followers.add(local_follower)
 
-        models.User.objects.create_user(
-            'nutria', 'nutria@mouse.mouse', 'nuword',
-            remote_id='http://example.com/u/4',
-            outbox='http://example.com/u/4/o',
-            shared_inbox='http://example.com/inbox',
-            inbox='http://example.com/u/4/inbox',
-            local=False)
+        with patch('bookwyrm.models.user.get_remote_reviews.delay'):
+            models.User.objects.create_user(
+                'nutria', 'nutria@mouse.mouse', 'nuword',
+                remote_id='http://example.com/u/4',
+                outbox='http://example.com/u/4/o',
+                shared_inbox='http://example.com/inbox',
+                inbox='http://example.com/u/4/inbox',
+                local=False)
 
 
     def test_get_public_recipients(self):

From 205fa0d465ba8a1db31ad34b6cb1ba66ee98ae05 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 29 Nov 2020 10:08:19 -0800
Subject: [PATCH 020/104] set user's remote server in a celery task

---
 bookwyrm/models/user.py                      | 43 ++++++++------------
 bookwyrm/tests/activitypub/test_quotation.py | 16 ++++----
 bookwyrm/tests/test_broadcast.py             |  2 +-
 3 files changed, 28 insertions(+), 33 deletions(-)

diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index a32a35b7..5b5af77e 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -7,6 +7,7 @@ from django.db import models
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
+from bookwyrm.connectors import get_data
 from bookwyrm.models.shelf import Shelf
 from bookwyrm.models.status import Status, Review
 from bookwyrm.settings import DOMAIN
@@ -195,13 +196,7 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
         return
 
     if not instance.local:
-        actor_parts = urlparse(instance.remote_id)
-        instance.federated_server = \
-            get_or_create_remote_server(actor_parts.netloc)
-        instance.save()
-        if instance.bookwyrm_user:
-            get_remote_reviews.delay(instance.outbox)
-        return
+        set_remote_server.delay(instance.id)
 
     shelves = [{
         'name': 'To Read',
@@ -222,6 +217,18 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
             editable=False
         ).save()
 
+@app.task
+def set_remote_server(user_id):
+    ''' figure out the user's remote server in the background '''
+    user = User.objects.get(id=user_id)
+    actor_parts = urlparse(user.remote_id)
+    user.federated_server = \
+        get_or_create_remote_server(actor_parts.netloc)
+    user.save()
+    if user.bookwyrm_user:
+        get_remote_reviews.delay(user.outbox)
+    return
+
 
 def get_or_create_remote_server(domain):
     ''' get info on a remote server '''
@@ -232,25 +239,14 @@ def get_or_create_remote_server(domain):
     except FederatedServer.DoesNotExist:
         pass
 
-    response = requests.get(
-        'https://%s/.well-known/nodeinfo' % domain,
-        headers={'Accept': 'application/activity+json'}
-    )
+    data = get_data('https://%s/.well-known/nodeinfo' % domain)
 
-    if response.status_code != 200:
-        return None
-
-    data = response.json()
     try:
         nodeinfo_url = data.get('links')[0].get('href')
     except (TypeError, KeyError):
         return None
 
-    response = requests.get(
-        nodeinfo_url,
-        headers={'Accept': 'application/activity+json'}
-    )
-    data = response.json()
+    data = get_data(nodeinfo_url)
 
     server = FederatedServer.objects.create(
         server_name=domain,
@@ -264,11 +260,8 @@ def get_or_create_remote_server(domain):
 def get_remote_reviews(outbox):
     ''' ingest reviews by a new remote bookwyrm user '''
     outbox_page = outbox + '?page=true'
-    response = requests.get(
-        outbox_page,
-        headers={'Accept': 'application/activity+json'}
-    )
-    data = response.json()
+    data = get_data(outbox_page)
+
     # TODO: pagination?
     for activity in data['orderedItems']:
         if not activity['type'] == 'Review':
diff --git a/bookwyrm/tests/activitypub/test_quotation.py b/bookwyrm/tests/activitypub/test_quotation.py
index 50d0ac86..b0699571 100644
--- a/bookwyrm/tests/activitypub/test_quotation.py
+++ b/bookwyrm/tests/activitypub/test_quotation.py
@@ -1,5 +1,6 @@
 import json
 import pathlib
+from unittest.mock import patch
 
 from django.test import TestCase
 from bookwyrm import activitypub, models
@@ -8,13 +9,14 @@ from bookwyrm import activitypub, models
 class Quotation(TestCase):
     ''' we have hecka ways to create statuses '''
     def setUp(self):
-        self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword',
-            local=False,
-            inbox='https://example.com/user/mouse/inbox',
-            outbox='https://example.com/user/mouse/outbox',
-            remote_id='https://example.com/user/mouse',
-        )
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            self.user = models.User.objects.create_user(
+                'mouse', 'mouse@mouse.mouse', 'mouseword',
+                local=False,
+                inbox='https://example.com/user/mouse/inbox',
+                outbox='https://example.com/user/mouse/outbox',
+                remote_id='https://example.com/user/mouse',
+            )
         self.book = models.Edition.objects.create(
             title='Example Edition',
             remote_id='https://example.com/book/1',
diff --git a/bookwyrm/tests/test_broadcast.py b/bookwyrm/tests/test_broadcast.py
index 96faf892..3ee4eeba 100644
--- a/bookwyrm/tests/test_broadcast.py
+++ b/bookwyrm/tests/test_broadcast.py
@@ -38,7 +38,7 @@ class Book(TestCase):
             'joe', 'joe@mouse.mouse', 'jeoword')
         self.user.followers.add(local_follower)
 
-        with patch('bookwyrm.models.user.get_remote_reviews.delay'):
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
             models.User.objects.create_user(
                 'nutria', 'nutria@mouse.mouse', 'nuword',
                 remote_id='http://example.com/u/4',

From 96563598bf5fb01952188daca6c6542935147666 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 29 Nov 2020 10:13:30 -0800
Subject: [PATCH 021/104] mock celery tasks for broadcast tests

---
 bookwyrm/tests/test_broadcast.py | 61 ++++++++++++++++----------------
 1 file changed, 31 insertions(+), 30 deletions(-)

diff --git a/bookwyrm/tests/test_broadcast.py b/bookwyrm/tests/test_broadcast.py
index 3ee4eeba..ae4e6145 100644
--- a/bookwyrm/tests/test_broadcast.py
+++ b/bookwyrm/tests/test_broadcast.py
@@ -6,39 +6,40 @@ from bookwyrm import models, broadcast
 
 class Book(TestCase):
     def setUp(self):
-        self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
+        with patch('bookwyrm.models.user.get_remote_reviews.delay'):
+            self.user = models.User.objects.create_user(
+                'mouse', 'mouse@mouse.mouse', 'mouseword')
 
-        follower = models.User.objects.create_user(
-            'rat', 'rat@mouse.mouse', 'ratword', local=False,
-            remote_id='http://example.com/u/1',
-            outbox='http://example.com/u/1/o',
-            shared_inbox='http://example.com/inbox',
-            inbox='http://example.com/u/1/inbox')
-        self.user.followers.add(follower)
-
-        no_inbox_follower = models.User.objects.create_user(
-            'hamster', 'hamster@mouse.mouse', 'hamword',
-            shared_inbox=None, local=False,
-            remote_id='http://example.com/u/2',
-            outbox='http://example.com/u/2/o',
-            inbox='http://example.com/u/2/inbox')
-        self.user.followers.add(no_inbox_follower)
-
-        non_fr_follower = models.User.objects.create_user(
-            'gerbil', 'gerb@mouse.mouse', 'gerbword',
-            remote_id='http://example.com/u/3',
-            outbox='http://example2.com/u/3/o',
-            inbox='http://example2.com/u/3/inbox',
-            shared_inbox='http://example2.com/inbox',
-            bookwyrm_user=False, local=False)
-        self.user.followers.add(non_fr_follower)
-
-        local_follower = models.User.objects.create_user(
-            'joe', 'joe@mouse.mouse', 'jeoword')
-        self.user.followers.add(local_follower)
+            local_follower = models.User.objects.create_user(
+                'joe', 'joe@mouse.mouse', 'jeoword')
+            self.user.followers.add(local_follower)
 
         with patch('bookwyrm.models.user.set_remote_server.delay'):
+            follower = models.User.objects.create_user(
+                'rat', 'rat@mouse.mouse', 'ratword', local=False,
+                remote_id='http://example.com/u/1',
+                outbox='http://example.com/u/1/o',
+                shared_inbox='http://example.com/inbox',
+                inbox='http://example.com/u/1/inbox')
+            self.user.followers.add(follower)
+
+            no_inbox_follower = models.User.objects.create_user(
+                'hamster', 'hamster@mouse.mouse', 'hamword',
+                shared_inbox=None, local=False,
+                remote_id='http://example.com/u/2',
+                outbox='http://example.com/u/2/o',
+                inbox='http://example.com/u/2/inbox')
+            self.user.followers.add(no_inbox_follower)
+
+            non_fr_follower = models.User.objects.create_user(
+                'gerbil', 'gerb@mouse.mouse', 'gerbword',
+                remote_id='http://example.com/u/3',
+                outbox='http://example2.com/u/3/o',
+                inbox='http://example2.com/u/3/inbox',
+                shared_inbox='http://example2.com/inbox',
+                bookwyrm_user=False, local=False)
+            self.user.followers.add(non_fr_follower)
+
             models.User.objects.create_user(
                 'nutria', 'nutria@mouse.mouse', 'nuword',
                 remote_id='http://example.com/u/4',

From 74a58e52672f2380ce7434c265fa05d02cc15e24 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 30 Nov 2020 10:32:13 -0800
Subject: [PATCH 022/104] Use custom model fields in user model

---
 bookwyrm/activitypub/__init__.py              |   4 +-
 bookwyrm/activitypub/base_activity.py         |   9 -
 bookwyrm/activitypub/person.py                |  11 +-
 .../migrations/0017_auto_20201130_1819.py     | 188 ++++++++++++++++++
 bookwyrm/models/base_model.py                 |  56 ++----
 bookwyrm/models/fields.py                     | 165 +++++++++++++++
 bookwyrm/models/user.py                       | 117 +++++------
 bookwyrm/view_actions.py                      |   3 +-
 8 files changed, 429 insertions(+), 124 deletions(-)
 create mode 100644 bookwyrm/migrations/0017_auto_20201130_1819.py
 create mode 100644 bookwyrm/models/fields.py

diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py
index eee9345d..b5b124ec 100644
--- a/bookwyrm/activitypub/__init__.py
+++ b/bookwyrm/activitypub/__init__.py
@@ -2,7 +2,7 @@
 import inspect
 import sys
 
-from .base_activity import ActivityEncoder, PublicKey, Signature
+from .base_activity import ActivityEncoder, Signature
 from .base_activity import Link, Mention
 from .base_activity import ActivitySerializerError, resolve_remote_id
 from .image import Image
@@ -10,7 +10,7 @@ from .note import Note, GeneratedNote, Article, Comment, Review, Quotation
 from .note import Tombstone
 from .interaction import Boost, Like
 from .ordered_collection import OrderedCollection, OrderedCollectionPage
-from .person import Person
+from .person import Person, PublicKey
 from .book import Edition, Work, Author
 from .verbs import Create, Delete, Undo, Update
 from .verbs import Follow, Accept, Reject
diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 9944e368..9e1b5b82 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -13,7 +13,6 @@ from django.db.models.fields import DateTimeField
 from django.db.models.fields.files import ImageFileDescriptor
 from django.db.models.query_utils import DeferredAttribute
 from django.utils import timezone
-import requests
 
 from bookwyrm.connectors import ConnectorException, get_data, get_image
 
@@ -41,14 +40,6 @@ class Mention(Link):
     type: str = 'Mention'
 
 
-@dataclass
-class PublicKey:
-    ''' public key block '''
-    id: str
-    owner: str
-    publicKeyPem: str
-
-
 @dataclass
 class Signature:
     ''' public key block '''
diff --git a/bookwyrm/activitypub/person.py b/bookwyrm/activitypub/person.py
index e7d720ec..88349c02 100644
--- a/bookwyrm/activitypub/person.py
+++ b/bookwyrm/activitypub/person.py
@@ -2,9 +2,18 @@
 from dataclasses import dataclass, field
 from typing import Dict
 
-from .base_activity import ActivityObject, PublicKey
+from .base_activity import ActivityObject
 from .image import Image
 
+
+@dataclass(init=False)
+class PublicKey(ActivityObject):
+    ''' public key block '''
+    owner: str
+    publicKeyPem: str
+    type: str = 'PublicKey'
+
+
 @dataclass(init=False)
 class Person(ActivityObject):
     ''' actor activitypub json '''
diff --git a/bookwyrm/migrations/0017_auto_20201130_1819.py b/bookwyrm/migrations/0017_auto_20201130_1819.py
new file mode 100644
index 00000000..bab454cb
--- /dev/null
+++ b/bookwyrm/migrations/0017_auto_20201130_1819.py
@@ -0,0 +1,188 @@
+# Generated by Django 3.0.7 on 2020-11-30 18:19
+
+import bookwyrm.models.base_model
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+def copy_rsa_keys(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    users = app_registry.get_model('bookwyrm', 'User')
+    keypair = app_registry.get_model('bookwyrm', 'KeyPair')
+    for user in users.objects.using(db_alias):
+        if user.public_key or user.private_key:
+            user.key_pair = keypair.objects.create(
+                private_key=user.private_key,
+                public_key=user.public_key
+            )
+            user.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0016_auto_20201129_0304'),
+    ]
+    operations = [
+        migrations.CreateModel(
+            name='KeyPair',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_date', models.DateTimeField(auto_now_add=True)),
+                ('updated_date', models.DateTimeField(auto_now=True)),
+                ('remote_id', bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id])),
+                ('private_key', models.TextField(blank=True, null=True)),
+                ('public_key', bookwyrm.models.fields.TextField(blank=True, null=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(bookwyrm.models.base_model.ActivitypubMixin, models.Model),
+        ),
+        migrations.AddField(
+            model_name='user',
+            name='followers',
+            field=bookwyrm.models.fields.ManyToManyField(related_name='following', through='bookwyrm.UserFollows', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='connector',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='favorite',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='federatedserver',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='image',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='notification',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='readthrough',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='avatar',
+            field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='avatars/'),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='bookwyrm_user',
+            field=bookwyrm.models.fields.BooleanField(default=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='inbox',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='local',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='manually_approves_followers',
+            field=bookwyrm.models.fields.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='name',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='outbox',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='shared_inbox',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='summary',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='username',
+            field=bookwyrm.models.fields.UsernameField(),
+        ),
+        migrations.AlterField(
+            model_name='userblocks',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='userfollowrequest',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='userfollows',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AddField(
+            model_name='user',
+            name='key_pair',
+            field=bookwyrm.models.fields.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='owner', to='bookwyrm.KeyPair'),
+        ),
+        migrations.RunPython(copy_rsa_keys),
+    ]
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index f5f244c5..90b889e0 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -17,6 +17,7 @@ from django.dispatch import receiver
 
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
+from .fields import RemoteIdField
 
 
 PrivacyLevels = models.TextChoices('Privacy', [
@@ -30,7 +31,7 @@ class BookWyrmModel(models.Model):
     ''' shared fields '''
     created_date = models.DateTimeField(auto_now_add=True)
     updated_date = models.DateTimeField(auto_now=True)
-    remote_id = models.CharField(max_length=255, null=True)
+    remote_id = RemoteIdField(null=True, activitypub_field='id')
 
     def get_remote_id(self):
         ''' generate a url that resolves to the local object '''
@@ -61,49 +62,18 @@ class ActivitypubMixin:
 
     def to_activity(self, pure=False):
         ''' convert from a model to an activity '''
-        if pure:
-            # works around bookwyrm-specific fields for vanilla AP services
-            mappings = self.pure_activity_mappings
-        else:
-            # may include custom fields that bookwyrm instances will understand
-            mappings = self.activity_mappings
+        activity = {}
+        for field in self.__class__._meta.fields:
+            key, value = field.to_activity(getattr(self, field.name))
+            activity[key] = value
+        for related_object in self.__class__.meta.related_objects:
+            # TODO: check if it's serializable
+            related_model = related_object.related_model
+            key = related_object.name
+            related_values = getattr(self, key)
+            activity[key] = [i.remote_id for i in related_values]
 
-        fields = {}
-        for mapping in mappings:
-            if not hasattr(self, mapping.model_key) or not mapping.activity_key:
-                # this field on the model isn't serialized
-                continue
-            value = getattr(self, mapping.model_key)
-            model_field = getattr(self.__class__, mapping.model_key)
-            if hasattr(value, 'remote_id'):
-                # this is probably a foreign key field, which we want to
-                # serialize as just the remote_id url reference
-                value = value.remote_id
-            elif isinstance(model_field, \
-                    (ManyToManyDescriptor, ReverseManyToOneDescriptor)):
-                value = [i.remote_id for i in value.all()]
-            elif isinstance(value, datetime):
-                value = value.isoformat()
-            elif isinstance(model_field, ImageFileDescriptor):
-                value = image_formatter(value)
-
-            # run the custom formatter function set in the model
-            formatted_value = mapping.activity_formatter(value)
-            if mapping.activity_key in fields and \
-                    isinstance(fields[mapping.activity_key], list):
-                # there can be two database fields that map to the same AP list
-                # this happens in status tags, which combines user and book tags
-                fields[mapping.activity_key] += formatted_value
-            else:
-                fields[mapping.activity_key] = formatted_value
-
-        if pure:
-            return self.pure_activity_serializer(
-                **fields
-            ).serialize()
-        return self.activity_serializer(
-            **fields
-        ).serialize()
+        return self.activity_serializer(**activity_json).serialize()
 
 
     def to_create_activity(self, user, pure=False):
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
new file mode 100644
index 00000000..c231039c
--- /dev/null
+++ b/bookwyrm/models/fields.py
@@ -0,0 +1,165 @@
+''' activitypub-aware django model fields '''
+import re
+from uuid import uuid4
+
+from django.contrib.auth.models import AbstractUser
+from django.core.exceptions import ValidationError
+from django.core.files.base import ContentFile
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+from bookwyrm import activitypub
+from bookwyrm.settings import DOMAIN
+from bookwyrm.connectors import get_image
+
+
+def validate_remote_id(value):
+    ''' make sure the remote_id looks like a url '''
+    if not re.match(r'^http.?:\/\/[^\s]+$', value):
+        raise ValidationError(
+            _('%(value)s is not a valid remote_id'),
+            params={'value': value},
+        )
+
+
+def to_camel_case(snake_string):
+    ''' model_field_name to activitypubFieldName '''
+    components = snake_string.split('_')
+    return components[0] + ''.join(x.title() for x in components[1:])
+
+
+class ActivitypubFieldMixin:
+    ''' make a database field serializable '''
+    def __init__(self, *args, \
+            activitypub_field=None, activitypub_wrapper=None, **kwargs):
+        self.activitypub_wrapper = activitypub_wrapper
+        self.activitypub_field = activitypub_field
+        super().__init__(*args, **kwargs)
+
+    def to_activity(self, value):
+        ''' formatter to convert a model value into activitypub '''
+        if self.activitypub_wrapper:
+            value = {self.activitypub_wrapper: value}
+        return (self.activitypub_field, value)
+
+    def from_activity(self, activity_data):
+        ''' formatter to convert activitypub into a model value '''
+        value = activity_data.get(self.activitypub_field)
+        if self.activitypub_wrapper:
+            value = value.get(self.activitypub_wrapper)
+        return value
+
+
+class RemoteIdField(ActivitypubFieldMixin, models.CharField):
+    ''' a url that serves as a unique identifier '''
+    def __init__(self, *args, max_length=255, validators=None, **kwargs):
+        validators = validators or [validate_remote_id]
+        super().__init__(
+            *args, max_length=max_length, validators=validators,
+            **kwargs
+        )
+
+
+class UsernameField(ActivitypubFieldMixin, models.CharField):
+    ''' activitypub-aware username field '''
+    def __init__(self, activitypub_field='preferredUsername'):
+        self.activitypub_field = activitypub_field
+        super(ActivitypubFieldMixin, self).__init__(
+            _('username'),
+            max_length=150,
+            unique=True,
+            validators=[AbstractUser.username_validator],
+            error_messages={
+                'unique': _('A user with that username already exists.'),
+            },
+        )
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        del kwargs['verbose_name']
+        del kwargs['max_length']
+        del kwargs['unique']
+        del kwargs['validators']
+        del kwargs['error_messages']
+        return name, path, args, kwargs
+
+    def to_activity(self, value):
+        return value.split('@')[0]
+
+
+class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
+    ''' activitypub-aware foreign key field '''
+    def to_activity(self, value):
+        return value.remote_id
+    def from_activity(self, activity_data):
+        pass# TODO
+
+
+class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
+    ''' activitypub-aware foreign key field '''
+    def __init__(self, *args, **kwargs):
+        super(ActivitypubFieldMixin, self).__init__(*args, **kwargs)
+
+    def to_activity(self, value):
+        return value.remote_id
+    def from_activity(self, activity_data):
+        pass# TODO
+
+
+class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
+    ''' activitypub-aware many to many field '''
+    def __init__(self, *args, link_only=False, **kwargs):
+        self.link_only = link_only
+        super().__init__(*args, **kwargs)
+
+    def to_activity(self, value):
+        if self.link_only:
+            return '%s/followers' % self.instance.remote_id
+        return [i.remote_id for i in value]
+
+    def from_activity(self, activity_data):
+        if self.link_only:
+            return
+        values = super().from_activity(self, activity_data)
+        return values# TODO
+
+
+class ImageField(ActivitypubFieldMixin, models.ImageField):
+    ''' activitypub-aware image field '''
+    def to_activity(self, value):
+        if value and hasattr(value, 'url'):
+            url = value.url
+        else:
+            return None
+        url = 'https://%s%s' % (DOMAIN, url)
+        return activitypub.Image(url=url)
+
+    def from_activity(self, activity_data):
+        image_slug = super().from_activity(activity_data)
+        # when it's an inline image (User avatar/icon, Book cover), it's a json
+        # blob, but when it's an attached image, it's just a url
+        if isinstance(image_slug, dict):
+            url = image_slug.get('url')
+        elif isinstance(image_slug, str):
+            url = image_slug
+        else:
+            return None
+        if not url:
+            return None
+
+        response = get_image(url)
+        if not response:
+            return None
+
+        image_name = str(uuid4()) + '.' + url.split('.')[-1]
+        image_content = ContentFile(response.content)
+        return [image_name, image_content]
+
+
+class CharField(ActivitypubFieldMixin, models.CharField):
+    ''' activitypub-aware char field '''
+
+class TextField(ActivitypubFieldMixin, models.TextField):
+    ''' activitypub-aware text field '''
+
+class BooleanField(ActivitypubFieldMixin, models.BooleanField):
+    ''' activitypub-aware boolean field '''
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index 5b5af77e..19746eb1 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -1,6 +1,5 @@
 ''' database schema for user data '''
 from urllib.parse import urlparse
-import requests
 
 from django.contrib.auth.models import AbstractUser
 from django.db import models
@@ -13,40 +12,51 @@ from bookwyrm.models.status import Status, Review
 from bookwyrm.settings import DOMAIN
 from bookwyrm.signatures import create_key_pair
 from bookwyrm.tasks import app
-from .base_model import ActivityMapping, OrderedCollectionPageMixin
+from .base_model import OrderedCollectionPageMixin
+from .base_model import ActivitypubMixin, BookWyrmModel
 from .federated_server import FederatedServer
+from . import fields
 
 
 class User(OrderedCollectionPageMixin, AbstractUser):
     ''' a user who wants to read books '''
-    private_key = models.TextField(blank=True, null=True)
-    public_key = models.TextField(blank=True, null=True)
-    inbox = models.CharField(max_length=255, unique=True)
-    shared_inbox = models.CharField(max_length=255, blank=True, null=True)
+    username = fields.UsernameField()
+
+    key_pair = fields.OneToOneField(
+        'KeyPair',
+        on_delete=models.CASCADE,
+        blank=True, null=True,
+        related_name='owner'
+    )
+    inbox = fields.RemoteIdField(unique=True)
+    shared_inbox = fields.RemoteIdField(
+        activitypub_wrapper='endpoints', null=True)
     federated_server = models.ForeignKey(
         'FederatedServer',
         on_delete=models.PROTECT,
         null=True,
         blank=True,
     )
-    outbox = models.CharField(max_length=255, unique=True)
-    summary = models.TextField(blank=True, null=True)
-    local = models.BooleanField(default=True)
-    bookwyrm_user = models.BooleanField(default=True)
+    outbox = fields.RemoteIdField(unique=True)
+    summary = fields.TextField(blank=True, null=True)
+    local = models.BooleanField(default=False)
+    bookwyrm_user = fields.BooleanField(default=True)
     localname = models.CharField(
         max_length=255,
         null=True,
         unique=True
     )
     # name is your display name, which you can change at will
-    name = models.CharField(max_length=100, blank=True, null=True)
-    avatar = models.ImageField(upload_to='avatars/', blank=True, null=True)
-    following = models.ManyToManyField(
+    name = fields.CharField(max_length=100, blank=True, null=True)
+    avatar = fields.ImageField(
+        upload_to='avatars/', blank=True, null=True, activitypub_field='icon')
+    followers = fields.ManyToManyField(
         'self',
+        link_only=True,
         symmetrical=False,
         through='UserFollows',
-        through_fields=('user_subject', 'user_object'),
-        related_name='followers'
+        through_fields=('user_object', 'user_subject'),
+        related_name='following'
     )
     follow_requests = models.ManyToManyField(
         'self',
@@ -69,60 +79,13 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         through_fields=('user', 'status'),
         related_name='favorite_statuses'
     )
-    remote_id = models.CharField(max_length=255, null=True, unique=True)
+    remote_id = fields.RemoteIdField(
+        null=True, unique=True, activitypub_field='id')
     created_date = models.DateTimeField(auto_now_add=True)
     updated_date = models.DateTimeField(auto_now=True)
     last_active_date = models.DateTimeField(auto_now=True)
-    manually_approves_followers = models.BooleanField(default=False)
+    manually_approves_followers = fields.BooleanField(default=False)
 
-    # ---- activitypub serialization settings for this model ----- #
-    @property
-    def ap_followers(self):
-        ''' generates url for activitypub followers page '''
-        return '%s/followers' % self.remote_id
-
-    @property
-    def ap_public_key(self):
-        ''' format the public key block for activitypub '''
-        return activitypub.PublicKey(**{
-            'id': '%s/#main-key' % self.remote_id,
-            'owner': self.remote_id,
-            'publicKeyPem': self.public_key,
-        })
-
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping(
-            'preferredUsername',
-            'username',
-            activity_formatter=lambda x: x.split('@')[0]
-        ),
-        ActivityMapping('name', 'name'),
-        ActivityMapping('bookwyrmUser', 'bookwyrm_user'),
-        ActivityMapping('inbox', 'inbox'),
-        ActivityMapping('outbox', 'outbox'),
-        ActivityMapping('followers', 'ap_followers'),
-        ActivityMapping('summary', 'summary'),
-        ActivityMapping(
-            'publicKey',
-            'public_key',
-            model_formatter=lambda x: x.get('publicKeyPem')
-        ),
-        ActivityMapping('publicKey', 'ap_public_key'),
-        ActivityMapping(
-            'endpoints',
-            'shared_inbox',
-            activity_formatter=lambda x: {'sharedInbox': x},
-            model_formatter=lambda x: x.get('sharedInbox')
-        ),
-        ActivityMapping('icon', 'avatar'),
-        ActivityMapping(
-            'manuallyApprovesFollowers',
-            'manually_approves_followers'
-        ),
-        # this field isn't in the activity but should always be false
-        ActivityMapping(None, 'local', model_formatter=lambda x: False),
-    ]
     activity_serializer = activitypub.Person
 
     def to_outbox(self, **kwargs):
@@ -183,12 +146,30 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         self.inbox = '%s/inbox' % self.remote_id
         self.shared_inbox = 'https://%s/inbox' % DOMAIN
         self.outbox = '%s/outbox' % self.remote_id
-        if not self.private_key:
-            self.private_key, self.public_key = create_key_pair()
+        if not self.key_pair:
+            self.key_pair = KeyPair.objects.create()
 
         return super().save(*args, **kwargs)
 
 
+class KeyPair(ActivitypubMixin, BookWyrmModel):
+    ''' public and private keys for a user '''
+    private_key = models.TextField(blank=True, null=True)
+    public_key = fields.TextField(
+        blank=True, null=True, activitypub_field='publicKeyPem')
+
+    activity_serializer = activitypub.PublicKey
+
+    def get_remote_id(self):
+        # self.owner is set by the OneToOneField on User
+        return '%s/#main-key' % self.owner.remote_id
+
+    def save(self, *args, **kwargs):
+        ''' create a key pair '''
+        self.private_key, self.public_key = create_key_pair()
+        return super().save(*args, **kwargs)
+
+
 @receiver(models.signals.post_save, sender=User)
 def execute_after_save(sender, instance, created, *args, **kwargs):
     ''' create shelves for new users '''
@@ -217,6 +198,7 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
             editable=False
         ).save()
 
+
 @app.task
 def set_remote_server(user_id):
     ''' figure out the user's remote server in the background '''
@@ -227,7 +209,6 @@ def set_remote_server(user_id):
     user.save()
     if user.bookwyrm_user:
         get_remote_reviews.delay(user.outbox)
-    return
 
 
 def get_or_create_remote_server(domain):
diff --git a/bookwyrm/view_actions.py b/bookwyrm/view_actions.py
index 29dc6406..cf0e7644 100644
--- a/bookwyrm/view_actions.py
+++ b/bookwyrm/view_actions.py
@@ -84,7 +84,8 @@ def register(request):
         }
         return TemplateResponse(request, 'login.html', data)
 
-    user = models.User.objects.create_user(username, email, password)
+    user = models.User.objects.create_user(
+        username, email, password, local=True)
     if invite:
         invite.times_used += 1
         invite.save()

From 8bc0a57bd41ac48bf2ee3cf540e3f1a5d4637b0e Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 30 Nov 2020 10:32:54 -0800
Subject: [PATCH 023/104] Remove outdated user fields

---
 .../migrations/0017_auto_20201130_1819.py     |  1 +
 .../migrations/0018_auto_20201130_1832.py     | 25 +++++++++++++
 .../migrations/0019_auto_20201130_1939.py     | 24 +++++++++++++
 bookwyrm/models/__init__.py                   |  2 +-
 bookwyrm/models/base_model.py                 | 36 +++++++++++--------
 bookwyrm/models/fields.py                     | 25 ++++++-------
 bookwyrm/models/user.py                       | 16 +++++++--
 7 files changed, 96 insertions(+), 33 deletions(-)
 create mode 100644 bookwyrm/migrations/0018_auto_20201130_1832.py
 create mode 100644 bookwyrm/migrations/0019_auto_20201130_1939.py

diff --git a/bookwyrm/migrations/0017_auto_20201130_1819.py b/bookwyrm/migrations/0017_auto_20201130_1819.py
index bab454cb..ce9f1cc7 100644
--- a/bookwyrm/migrations/0017_auto_20201130_1819.py
+++ b/bookwyrm/migrations/0017_auto_20201130_1819.py
@@ -13,6 +13,7 @@ def copy_rsa_keys(app_registry, schema_editor):
     for user in users.objects.using(db_alias):
         if user.public_key or user.private_key:
             user.key_pair = keypair.objects.create(
+                remote_id='%s/#main-key' % user.remote_id,
                 private_key=user.private_key,
                 public_key=user.public_key
             )
diff --git a/bookwyrm/migrations/0018_auto_20201130_1832.py b/bookwyrm/migrations/0018_auto_20201130_1832.py
new file mode 100644
index 00000000..278446cf
--- /dev/null
+++ b/bookwyrm/migrations/0018_auto_20201130_1832.py
@@ -0,0 +1,25 @@
+# Generated by Django 3.0.7 on 2020-11-30 18:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0017_auto_20201130_1819'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='user',
+            name='following',
+        ),
+        migrations.RemoveField(
+            model_name='user',
+            name='private_key',
+        ),
+        migrations.RemoveField(
+            model_name='user',
+            name='public_key',
+        ),
+    ]
diff --git a/bookwyrm/migrations/0019_auto_20201130_1939.py b/bookwyrm/migrations/0019_auto_20201130_1939.py
new file mode 100644
index 00000000..c69bdbce
--- /dev/null
+++ b/bookwyrm/migrations/0019_auto_20201130_1939.py
@@ -0,0 +1,24 @@
+# Generated by Django 3.0.7 on 2020-11-30 19:39
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0018_auto_20201130_1832'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='name',
+            field=bookwyrm.models.fields.CharField(default='', max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='summary',
+            field=bookwyrm.models.fields.TextField(default=''),
+        ),
+    ]
diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py
index 9a4d6014..86bdf219 100644
--- a/bookwyrm/models/__init__.py
+++ b/bookwyrm/models/__init__.py
@@ -14,7 +14,7 @@ from .attachment import Image
 
 from .tag import Tag, UserTag
 
-from .user import User
+from .user import User, KeyPair
 from .relationship import UserFollows, UserFollowRequest, UserBlocks
 from .federated_server import FederatedServer
 
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 90b889e0..91fda5a3 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -1,5 +1,4 @@
 ''' base model with default fields '''
-from datetime import datetime
 from base64 import b64encode
 from dataclasses import dataclass
 from typing import Callable
@@ -10,9 +9,6 @@ from Crypto.PublicKey import RSA
 from Crypto.Signature import pkcs1_15
 from Crypto.Hash import SHA256
 from django.db import models
-from django.db.models.fields.files import ImageFileDescriptor
-from django.db.models.fields.related_descriptors \
-        import ManyToManyDescriptor, ReverseManyToOneDescriptor
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
@@ -56,24 +52,34 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
         instance.save()
 
 
+def get_field_name(field):
+    ''' model_field_name to activitypubFieldName '''
+    if field.activitypub_field:
+        return field.activitypub_field
+    name = field.name.split('.')[-1]
+    components = name.split('_')
+    return components[0] + ''.join(x.title() for x in components[1:])
+
+
 class ActivitypubMixin:
     ''' add this mixin for models that are AP serializable '''
     activity_serializer = lambda: {}
 
-    def to_activity(self, pure=False):
+    def to_activity(self):
         ''' convert from a model to an activity '''
         activity = {}
-        for field in self.__class__._meta.fields:
-            key, value = field.to_activity(getattr(self, field.name))
-            activity[key] = value
-        for related_object in self.__class__.meta.related_objects:
-            # TODO: check if it's serializable
-            related_model = related_object.related_model
-            key = related_object.name
-            related_values = getattr(self, key)
-            activity[key] = [i.remote_id for i in related_values]
+        for field in self.__class__._meta.get_fields():
+            if not hasattr(field, 'to_activity'):
+                continue
+            key = get_field_name(field)
+            value = field.to_activity(getattr(self, field.name))
+            if value is not None:
+                activity[key] = value
+        if hasattr(self, 'serialize_reverse_fields'):
+            for field_name in self.serialize_reverse_fields:
+                activity[field_name] = getattr(self, field_name).remote_id
 
-        return self.activity_serializer(**activity_json).serialize()
+        return self.activity_serializer(**activity).serialize()
 
 
     def to_create_activity(self, user, pure=False):
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index c231039c..10e1b100 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -21,25 +21,22 @@ def validate_remote_id(value):
         )
 
 
-def to_camel_case(snake_string):
-    ''' model_field_name to activitypubFieldName '''
-    components = snake_string.split('_')
-    return components[0] + ''.join(x.title() for x in components[1:])
-
-
 class ActivitypubFieldMixin:
     ''' make a database field serializable '''
     def __init__(self, *args, \
             activitypub_field=None, activitypub_wrapper=None, **kwargs):
-        self.activitypub_wrapper = activitypub_wrapper
-        self.activitypub_field = activitypub_field
+        if activitypub_wrapper:
+            self.activitypub_wrapper = activitypub_field
+            self.activitypub_field = activitypub_wrapper
+        else:
+            self.activitypub_field = activitypub_field
         super().__init__(*args, **kwargs)
 
     def to_activity(self, value):
         ''' formatter to convert a model value into activitypub '''
-        if self.activitypub_wrapper:
+        if hasattr(self, 'activitypub_wrapper'):
             value = {self.activitypub_wrapper: value}
-        return (self.activitypub_field, value)
+        return value
 
     def from_activity(self, activity_data):
         ''' formatter to convert activitypub into a model value '''
@@ -96,11 +93,9 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
 
 class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
     ''' activitypub-aware foreign key field '''
-    def __init__(self, *args, **kwargs):
-        super(ActivitypubFieldMixin, self).__init__(*args, **kwargs)
-
     def to_activity(self, value):
-        return value.remote_id
+        return value.to_activity()
+
     def from_activity(self, activity_data):
         pass# TODO
 
@@ -113,7 +108,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
 
     def to_activity(self, value):
         if self.link_only:
-            return '%s/followers' % self.instance.remote_id
+            return '%s/followers' % value.instance.remote_id
         return [i.remote_id for i in value]
 
     def from_activity(self, activity_data):
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index 19746eb1..f480476f 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -26,10 +26,12 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         'KeyPair',
         on_delete=models.CASCADE,
         blank=True, null=True,
+        activitypub_field='publicKey',
         related_name='owner'
     )
     inbox = fields.RemoteIdField(unique=True)
     shared_inbox = fields.RemoteIdField(
+        activitypub_field='sharedInbox',
         activitypub_wrapper='endpoints', null=True)
     federated_server = models.ForeignKey(
         'FederatedServer',
@@ -38,7 +40,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         blank=True,
     )
     outbox = fields.RemoteIdField(unique=True)
-    summary = fields.TextField(blank=True, null=True)
+    summary = fields.TextField(default='')
     local = models.BooleanField(default=False)
     bookwyrm_user = fields.BooleanField(default=True)
     localname = models.CharField(
@@ -47,7 +49,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         unique=True
     )
     # name is your display name, which you can change at will
-    name = fields.CharField(max_length=100, blank=True, null=True)
+    name = fields.CharField(max_length=100, default='')
     avatar = fields.ImageField(
         upload_to='avatars/', blank=True, null=True, activitypub_field='icon')
     followers = fields.ManyToManyField(
@@ -87,6 +89,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     manually_approves_followers = fields.BooleanField(default=False)
 
     activity_serializer = activitypub.Person
+    serialize_related = []
 
     def to_outbox(self, **kwargs):
         ''' an ordered collection of statuses '''
@@ -159,6 +162,7 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
         blank=True, null=True, activitypub_field='publicKeyPem')
 
     activity_serializer = activitypub.PublicKey
+    serialize_reverse_fields = ['owner']
 
     def get_remote_id(self):
         # self.owner is set by the OneToOneField on User
@@ -169,6 +173,14 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
         self.private_key, self.public_key = create_key_pair()
         return super().save(*args, **kwargs)
 
+    def to_activity(self):
+        ''' override default AP serializer to add context object
+            idk if this is the best way to go about this '''
+        activity_object = super().to_activity()
+        del activity_object['@context']
+        del activity_object['type']
+        return activity_object
+
 
 @receiver(models.signals.post_save, sender=User)
 def execute_after_save(sender, instance, created, *args, **kwargs):

From 3966c84e08931908e6e3759afb5368a49e53ae37 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 30 Nov 2020 14:24:31 -0800
Subject: [PATCH 024/104] Updates status model and serializer

---
 bookwyrm/activitypub/note.py  |  10 +-
 bookwyrm/models/attachment.py |  15 ++-
 bookwyrm/models/base_model.py |  55 ++++------
 bookwyrm/models/book.py       |   1 +
 bookwyrm/models/fields.py     |  57 +++++++++--
 bookwyrm/models/status.py     | 185 ++++++++++++++--------------------
 bookwyrm/models/user.py       |  10 +-
 7 files changed, 167 insertions(+), 166 deletions(-)

diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py
index aeb078dc..df28bf8d 100644
--- a/bookwyrm/activitypub/note.py
+++ b/bookwyrm/activitypub/note.py
@@ -8,7 +8,6 @@ from .image import Image
 @dataclass(init=False)
 class Tombstone(ActivityObject):
     ''' the placeholder for a deleted status '''
-    url: str
     published: str
     deleted: str
     type: str = 'Tombstone'
@@ -17,14 +16,13 @@ class Tombstone(ActivityObject):
 @dataclass(init=False)
 class Note(ActivityObject):
     ''' Note activity '''
-    url: str
-    inReplyTo: str
     published: str
     attributedTo: str
-    to: List[str]
-    cc: List[str]
     content: str
-    replies: Dict
+    to: List[str] = field(default_factory=lambda: [])
+    cc: List[str] = field(default_factory=lambda: [])
+    replies: Dict = field(default_factory=lambda: {})
+    inReplyTo: str = ''
     tag: List[Link] = field(default_factory=lambda: [])
     attachment: List[Image] = field(default_factory=lambda: [])
     sensitive: bool = False
diff --git a/bookwyrm/models/attachment.py b/bookwyrm/models/attachment.py
index 7329e65d..6a92240d 100644
--- a/bookwyrm/models/attachment.py
+++ b/bookwyrm/models/attachment.py
@@ -3,30 +3,27 @@ from django.db import models
 
 from bookwyrm import activitypub
 from .base_model import ActivitypubMixin
-from .base_model import ActivityMapping, BookWyrmModel
+from .base_model import BookWyrmModel
+from . import fields
 
 
 class Attachment(ActivitypubMixin, BookWyrmModel):
     ''' an image (or, in the future, video etc) associated with a status '''
-    status = models.ForeignKey(
+    status = fields.ForeignKey(
         'Status',
         on_delete=models.CASCADE,
         related_name='attachments',
         null=True
     )
+    reverse_unfurl = True
     class Meta:
         ''' one day we'll have other types of attachments besides images '''
         abstract = True
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('url', 'image'),
-        ActivityMapping('name', 'caption'),
-    ]
 
 class Image(Attachment):
     ''' an image attachment '''
-    image = models.ImageField(upload_to='status/', null=True, blank=True)
-    caption = models.TextField(null=True, blank=True)
+    image = fields.ImageField(upload_to='status/', null=True, blank=True)
+    caption = fields.TextField(null=True, blank=True)
 
     activity_serializer = activitypub.Image
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 91fda5a3..34c4e5bc 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -61,9 +61,19 @@ def get_field_name(field):
     return components[0] + ''.join(x.title() for x in components[1:])
 
 
+def unfurl_related_field(related_field):
+    ''' load reverse lookups (like public key owner or Status attachment '''
+    if hasattr(related_field, 'all'):
+        return [unfurl_related_field(i) for i in related_field.all()]
+    if related_field.reverse_unfurl:
+        return related_field.to_activity()
+    return related_field.remote_id
+
+
 class ActivitypubMixin:
     ''' add this mixin for models that are AP serializable '''
     activity_serializer = lambda: {}
+    reverse_unfurl = False
 
     def to_activity(self):
         ''' convert from a model to an activity '''
@@ -73,18 +83,24 @@ class ActivitypubMixin:
                 continue
             key = get_field_name(field)
             value = field.to_activity(getattr(self, field.name))
-            if value is not None:
+            if value is None:
+                continue
+
+            if key in activity and isinstance(activity[key], list):
+                activity[key] += value
+            else:
                 activity[key] = value
         if hasattr(self, 'serialize_reverse_fields'):
             for field_name in self.serialize_reverse_fields:
-                activity[field_name] = getattr(self, field_name).remote_id
+                related_field = getattr(self, field_name)
+                activity[field_name] = unfurl_related_field(related_field)
 
         return self.activity_serializer(**activity).serialize()
 
 
-    def to_create_activity(self, user, pure=False):
+    def to_create_activity(self, user):
         ''' returns the object wrapped in a Create activity '''
-        activity_object = self.to_activity(pure=pure)
+        activity_object = self.to_activity()
 
         signer = pkcs1_15.new(RSA.import_key(user.private_key))
         content = activity_object['content']
@@ -100,8 +116,8 @@ class ActivitypubMixin:
         return activitypub.Create(
             id=create_id,
             actor=user.remote_id,
-            to=['%s/followers' % user.remote_id],
-            cc=['https://www.w3.org/ns/activitystreams#Public'],
+            to=activity_object['to'],
+            cc=activity_object['cc'],
             object=activity_object,
             signature=signature,
         ).serialize()
@@ -245,30 +261,3 @@ class ActivityMapping:
     model_key: str
     activity_formatter: Callable = lambda x: x
     model_formatter: Callable = lambda x: x
-
-
-def tag_formatter(items, name_field, activity_type):
-    ''' helper function to format lists of foreign keys into Tags '''
-    tags = []
-    for item in items.all():
-        tags.append(activitypub.Link(
-            href=item.remote_id,
-            name=getattr(item, name_field),
-            type=activity_type
-        ))
-    return tags
-
-
-def image_formatter(image):
-    ''' convert images into activitypub json '''
-    if image and hasattr(image, 'url'):
-        url = image.url
-    else:
-        return None
-    url = 'https://%s%s' % (DOMAIN, url)
-    return activitypub.Image(url=url)
-
-
-def image_attachments_formatter(images):
-    ''' create a list of image attachments '''
-    return [image_formatter(i) for i in images]
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index ffc9dd8a..fc7c9e91 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -155,6 +155,7 @@ class Edition(Book):
         'Work', on_delete=models.PROTECT, null=True, related_name='editions')
 
     activity_serializer = activitypub.Edition
+    name_field = 'title'
 
     def save(self, *args, **kwargs):
         ''' calculate isbn 10/13 '''
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 10e1b100..e419c9b8 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -71,6 +71,7 @@ class UsernameField(ActivitypubFieldMixin, models.CharField):
         )
 
     def deconstruct(self):
+        ''' implementation of models.Field deconstruct '''
         name, path, args, kwargs = super().deconstruct()
         del kwargs['verbose_name']
         del kwargs['max_length']
@@ -86,6 +87,8 @@ class UsernameField(ActivitypubFieldMixin, models.CharField):
 class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
     ''' activitypub-aware foreign key field '''
     def to_activity(self, value):
+        if not value:
+            return None
         return value.remote_id
     def from_activity(self, activity_data):
         pass# TODO
@@ -94,6 +97,10 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
 class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
     ''' activitypub-aware foreign key field '''
     def to_activity(self, value):
+        print('HIIIII')
+        print(value)
+        if not value:
+            return None
         return value.to_activity()
 
     def from_activity(self, activity_data):
@@ -113,20 +120,45 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
 
     def from_activity(self, activity_data):
         if self.link_only:
-            return
-        values = super().from_activity(self, activity_data)
+            return None
+        values = super().from_activity(activity_data)
         return values# TODO
 
+class TagField(ManyToManyField):
+    ''' special case of many to many that uses Tags '''
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.activitypub_field = 'tag'
+
+    def to_activity(self, value):
+        tags = []
+        for item in value.all():
+            activity_type = item.__class__.__name__
+            if activity_type == 'User':
+                activity_type = 'Mention'
+            tags.append(activitypub.Link(
+                href=item.remote_id,
+                name=getattr(item, item.name_field),
+                type=activity_type
+            ))
+        return tags
+
+
+def image_serializer(value):
+    ''' helper for serializing images '''
+    print(value)
+    if value and hasattr(value, 'url'):
+        url = value.url
+    else:
+        return None
+    url = 'https://%s%s' % (DOMAIN, url)
+    return activitypub.Image(url=url)
+
 
 class ImageField(ActivitypubFieldMixin, models.ImageField):
     ''' activitypub-aware image field '''
     def to_activity(self, value):
-        if value and hasattr(value, 'url'):
-            url = value.url
-        else:
-            return None
-        url = 'https://%s%s' % (DOMAIN, url)
-        return activitypub.Image(url=url)
+        return image_serializer(value)
 
     def from_activity(self, activity_data):
         image_slug = super().from_activity(activity_data)
@@ -150,6 +182,12 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
         return [image_name, image_content]
 
 
+class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
+    ''' activitypub-aware datetime field '''
+    def to_activity(self, value):
+        return value.isoformat()
+
+
 class CharField(ActivitypubFieldMixin, models.CharField):
     ''' activitypub-aware char field '''
 
@@ -158,3 +196,6 @@ class TextField(ActivitypubFieldMixin, models.TextField):
 
 class BooleanField(ActivitypubFieldMixin, models.BooleanField):
     ''' activitypub-aware boolean field '''
+
+class IntegerField(ActivitypubFieldMixin, models.IntegerField):
+    ''' activitypub-aware boolean field '''
diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index a520164a..c47a33bb 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -6,26 +6,27 @@ from model_utils.managers import InheritanceManager
 
 from bookwyrm import activitypub
 from .base_model import ActivitypubMixin, OrderedCollectionPageMixin
-from .base_model import ActivityMapping, BookWyrmModel, PrivacyLevels
-from .base_model import tag_formatter, image_attachments_formatter
-
+from .base_model import BookWyrmModel, PrivacyLevels
+from . import fields
+from .fields import image_serializer
 
 class Status(OrderedCollectionPageMixin, BookWyrmModel):
     ''' any post, like a reply to a review, etc '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    content = models.TextField(blank=True, null=True)
-    mention_users = models.ManyToManyField('User', related_name='mention_user')
-    mention_books = models.ManyToManyField(
-        'Edition', related_name='mention_book')
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='attributedTo')
+    content = fields.TextField(blank=True, null=True)
+    mention_users = fields.TagField('User', related_name='mention_user')
+    mention_books = fields.TagField('Edition', related_name='mention_book')
     local = models.BooleanField(default=True)
     privacy = models.CharField(
         max_length=255,
         default='public',
         choices=PrivacyLevels.choices
     )
-    sensitive = models.BooleanField(default=False)
+    sensitive = fields.BooleanField(default=False)
     # the created date can't be this, because of receiving federated posts
-    published_date = models.DateTimeField(default=timezone.now)
+    published_date = fields.DateTimeField(
+        default=timezone.now, activitypub_field='published')
     deleted = models.BooleanField(default=False)
     deleted_date = models.DateTimeField(blank=True, null=True)
     favorites = models.ManyToManyField(
@@ -35,79 +36,21 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         through_fields=('status', 'user'),
         related_name='user_favorites'
     )
-    reply_parent = models.ForeignKey(
+    reply_parent = fields.ForeignKey(
         'self',
         null=True,
-        on_delete=models.PROTECT
+        on_delete=models.PROTECT,
+        activitypub_field='inReplyTo',
     )
     objects = InheritanceManager()
 
-    # ---- activitypub serialization settings for this model ----- #
-    @property
-    def ap_to(self):
-        ''' should be related to post privacy I think '''
-        return ['https://www.w3.org/ns/activitystreams#Public']
-
-    @property
-    def ap_cc(self):
-        ''' should be related to post privacy I think '''
-        return [self.user.ap_followers]
-
     @property
     def ap_replies(self):
         ''' structured replies block '''
         return self.to_replies()
 
-    @property
-    def ap_status_image(self):
-        ''' attach a book cover, if relevent '''
-        if hasattr(self, 'book'):
-            return self.book.ap_cover
-        if self.mention_books.first():
-            return self.mention_books.first().ap_cover
-        return None
-
-
-    shared_mappings = [
-        ActivityMapping('url', 'remote_id', lambda x: None),
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('inReplyTo', 'reply_parent'),
-        ActivityMapping('published', 'published_date'),
-        ActivityMapping('attributedTo', 'user'),
-        ActivityMapping('to', 'ap_to'),
-        ActivityMapping('cc', 'ap_cc'),
-        ActivityMapping('replies', 'ap_replies'),
-        ActivityMapping(
-            'tag', 'mention_books',
-            lambda x: tag_formatter(x, 'title', 'Book'),
-        ),
-        ActivityMapping(
-            'tag', 'mention_users',
-            lambda x: tag_formatter(x, 'username', 'Mention'),
-        ),
-        ActivityMapping(
-            'attachment', 'attachments',
-            lambda x: image_attachments_formatter(x.all()),
-        )
-    ]
-
-    # serializing to bookwyrm expanded activitypub
-    activity_mappings = shared_mappings + [
-        ActivityMapping('name', 'name'),
-        ActivityMapping('inReplyToBook', 'book'),
-        ActivityMapping('rating', 'rating'),
-        ActivityMapping('quote', 'quote'),
-        ActivityMapping('content', 'content'),
-    ]
-
-    # for serializing to standard activitypub without extended types
-    pure_activity_mappings = shared_mappings + [
-        ActivityMapping('name', 'ap_pure_name'),
-        ActivityMapping('content', 'ap_pure_content'),
-        ActivityMapping('attachment', 'ap_status_image'),
-    ]
-
     activity_serializer = activitypub.Note
+    serialize_reverse_fields = ['attachments']
 
     #----- replies collection activitypub ----#
     @classmethod
@@ -138,7 +81,43 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
                 deleted=self.deleted_date.isoformat(),
                 published=self.deleted_date.isoformat()
             ).serialize()
-        return ActivitypubMixin.to_activity(self, pure=pure)
+        activity = ActivitypubMixin.to_activity(self)
+        activity['replies'] = self.to_replies()
+
+        # privacy controls
+        public = 'https://www.w3.org/ns/activitystreams#Public'
+        mentions = [u.remote_id for u in self.mention_users.all()]
+        # this is a link to the followers list:
+        followers = self.user.__class__._meta.get_field('followers')\
+                .to_activity(self.user.followers)
+        if self.privacy == 'public':
+            activity['to'] = [public]
+            activity['cc'] = [followers] + mentions
+        elif self.privacy == 'unlisted':
+            activity['to'] = [followers]
+            activity['cc'] = [public] + mentions
+        elif self.privacy == 'followers':
+            activity['to'] = [followers]
+            activity['cc'] = mentions
+        if self.privacy == 'direct':
+            activity['to'] = mentions
+            activity['cc'] = []
+
+        # "pure" serialization for non-bookwyrm instances
+        if pure:
+            activity['content'] = self.pure_content
+            if 'name' in activity:
+                activity['name'] = self.pure_name
+            activity['type'] = self.pure_type
+            activity['attachment'] = [
+                image_serializer(b.cover) for b in self.mention_books.all() \
+                        if b.cover]
+            if hasattr(self, 'book'):
+                activity['attachment'].append(
+                    image_serializer(self.book.cover)
+                )
+        return activity
+
 
     def save(self, *args, **kwargs):
         ''' update user active time '''
@@ -151,40 +130,40 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
 class GeneratedNote(Status):
     ''' these are app-generated messages about user activity '''
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
         message = self.content
         books = ', '.join(
-            '<a href="%s">"%s"</a>' % (self.book.remote_id, self.book.title) \
+            '<a href="%s">"%s"</a>' % (book.remote_id, book.title) \
             for book in self.mention_books.all()
         )
-        return '%s %s' % (message, books)
+        return '%s %s %s' % (self.user.display_name, message, books)
 
     activity_serializer = activitypub.GeneratedNote
-    pure_activity_serializer = activitypub.Note
+    pure_type = 'Note'
 
 
 class Comment(Status):
     ''' like a review but without a rating and transient '''
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
+    book = fields.ForeignKey('Edition', on_delete=models.PROTECT)
 
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
         return self.content + '<br><br>(comment on <a href="%s">"%s"</a>)' % \
                 (self.book.remote_id, self.book.title)
 
     activity_serializer = activitypub.Comment
-    pure_activity_serializer = activitypub.Note
+    pure_type = 'Note'
 
 
 class Quotation(Status):
     ''' like a review but without a rating and transient '''
-    quote = models.TextField()
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
+    quote = fields.TextField()
+    book = fields.ForeignKey('Edition', on_delete=models.PROTECT)
 
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
         return '"%s"<br>-- <a href="%s">"%s"</a><br><br>%s' % (
             self.quote,
@@ -194,14 +173,14 @@ class Quotation(Status):
         )
 
     activity_serializer = activitypub.Quotation
-    pure_activity_serializer = activitypub.Note
+    pure_type = 'Note'
 
 
 class Review(Status):
     ''' a book review '''
-    name = models.CharField(max_length=255, null=True)
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
-    rating = models.IntegerField(
+    name = fields.CharField(max_length=255, null=True)
+    book = fields.ForeignKey('Edition', on_delete=models.PROTECT)
+    rating = fields.IntegerField(
         default=None,
         null=True,
         blank=True,
@@ -209,7 +188,7 @@ class Review(Status):
     )
 
     @property
-    def ap_pure_name(self):
+    def pure_name(self):
         ''' clarify review names for mastodon serialization '''
         if self.rating:
             return 'Review of "%s" (%d stars): %s' % (
@@ -223,26 +202,21 @@ class Review(Status):
         )
 
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
         return self.content + '<br><br>(<a href="%s">"%s"</a>)' % \
                 (self.book.remote_id, self.book.title)
 
     activity_serializer = activitypub.Review
-    pure_activity_serializer = activitypub.Article
+    pure_type = 'Article'
 
 
 class Favorite(ActivitypubMixin, BookWyrmModel):
     ''' fav'ing a post '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    status = models.ForeignKey('Status', on_delete=models.PROTECT)
-
-    # ---- activitypub serialization settings for this model ----- #
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('actor', 'user'),
-        ActivityMapping('object', 'status'),
-    ]
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='actor')
+    status = fields.ForeignKey(
+        'Status', on_delete=models.PROTECT, activitypub_field='object')
 
     activity_serializer = activitypub.Like
 
@@ -252,7 +226,6 @@ class Favorite(ActivitypubMixin, BookWyrmModel):
         self.user.save()
         super().save(*args, **kwargs)
 
-
     class Meta:
         ''' can't fav things twice '''
         unique_together = ('user', 'status')
@@ -260,16 +233,12 @@ class Favorite(ActivitypubMixin, BookWyrmModel):
 
 class Boost(Status):
     ''' boost'ing a post '''
-    boosted_status = models.ForeignKey(
+    boosted_status = fields.ForeignKey(
         'Status',
         on_delete=models.PROTECT,
-        related_name="boosters")
-
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('actor', 'user'),
-        ActivityMapping('object', 'boosted_status'),
-    ]
+        related_name='boosters',
+        activitypub_field='object',
+    )
 
     activity_serializer = activitypub.Boost
 
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index f480476f..95dd1e79 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -88,8 +88,14 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     last_active_date = models.DateTimeField(auto_now=True)
     manually_approves_followers = fields.BooleanField(default=False)
 
+    @property
+    def display_name(self):
+        ''' show the cleanest version of the user's name possible '''
+        if self.name != '':
+            return self.name
+        return self.localname or self.username
+
     activity_serializer = activitypub.Person
-    serialize_related = []
 
     def to_outbox(self, **kwargs):
         ''' an ordered collection of statuses '''
@@ -112,7 +118,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         return self.to_ordered_collection(self.followers, \
                 remote_id=remote_id, id_only=True, **kwargs)
 
-    def to_activity(self, pure=False):
+    def to_activity(self):
         ''' override default AP serializer to add context object
             idk if this is the best way to go about this '''
         activity_object = super().to_activity()

From 77aead722d7bb4fd4ffcfadf243f2c496ded20ce Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 30 Nov 2020 14:40:26 -0800
Subject: [PATCH 025/104] serialize book and author models

---
 bookwyrm/models/author.py | 28 ++++-------
 bookwyrm/models/book.py   | 98 +++++++++++++--------------------------
 bookwyrm/models/fields.py | 11 +++--
 3 files changed, 49 insertions(+), 88 deletions(-)

diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py
index fdaf73d9..5b70b57f 100644
--- a/bookwyrm/models/author.py
+++ b/bookwyrm/models/author.py
@@ -4,29 +4,29 @@ from django.utils import timezone
 
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
-from bookwyrm.utils.fields import ArrayField
 
-from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel
+from .base_model import ActivitypubMixin, BookWyrmModel
+from . import fields
 
 
 class Author(ActivitypubMixin, BookWyrmModel):
     ''' basic biographic info '''
     origin_id = models.CharField(max_length=255, null=True)
     ''' copy of an author from OL '''
-    openlibrary_key = models.CharField(max_length=255, blank=True, null=True)
+    openlibrary_key = fields.CharField(max_length=255, blank=True, null=True)
     sync = models.BooleanField(default=True)
     last_sync_date = models.DateTimeField(default=timezone.now)
-    wikipedia_link = models.CharField(max_length=255, blank=True, null=True)
+    wikipedia_link = fields.CharField(max_length=255, blank=True, null=True)
     # idk probably other keys would be useful here?
-    born = models.DateTimeField(blank=True, null=True)
-    died = models.DateTimeField(blank=True, null=True)
-    name = models.CharField(max_length=255)
+    born = fields.DateTimeField(blank=True, null=True)
+    died = fields.DateTimeField(blank=True, null=True)
+    name = fields.CharField(max_length=255)
     last_name = models.CharField(max_length=255, blank=True, null=True)
     first_name = models.CharField(max_length=255, blank=True, null=True)
-    aliases = ArrayField(
+    aliases = fields.ArrayField(
         models.CharField(max_length=255), blank=True, default=list
     )
-    bio = models.TextField(null=True, blank=True)
+    bio = fields.TextField(null=True, blank=True)
 
     def save(self, *args, **kwargs):
         ''' can't be abstract for query reasons, but you shouldn't USE it '''
@@ -52,14 +52,4 @@ class Author(ActivitypubMixin, BookWyrmModel):
             return self.first_name + ' ' + self.last_name
         return self.last_name or self.first_name
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('name', 'name'),
-        ActivityMapping('born', 'born'),
-        ActivityMapping('died', 'died'),
-        ActivityMapping('aliases', 'aliases'),
-        ActivityMapping('bio', 'bio'),
-        ActivityMapping('openlibraryKey', 'openlibrary_key'),
-        ActivityMapping('wikipediaLink', 'wikipedia_link'),
-    ]
     activity_serializer = activitypub.Author
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index fc7c9e91..da532561 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -7,18 +7,18 @@ from model_utils.managers import InheritanceManager
 
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
-from bookwyrm.utils.fields import ArrayField
 
-from .base_model import ActivityMapping, BookWyrmModel
+from .base_model import BookWyrmModel
 from .base_model import ActivitypubMixin, OrderedCollectionPageMixin
+from . import fields
 
 class Book(ActivitypubMixin, BookWyrmModel):
     ''' a generic book, which can mean either an edition or a work '''
     origin_id = models.CharField(max_length=255, null=True, blank=True)
     # these identifiers apply to both works and editions
-    openlibrary_key = models.CharField(max_length=255, blank=True, null=True)
-    librarything_key = models.CharField(max_length=255, blank=True, null=True)
-    goodreads_key = models.CharField(max_length=255, blank=True, null=True)
+    openlibrary_key = fields.CharField(max_length=255, blank=True, null=True)
+    librarything_key = fields.CharField(max_length=255, blank=True, null=True)
+    goodreads_key = fields.CharField(max_length=255, blank=True, null=True)
 
     # info about where the data comes from and where/if to sync
     sync = models.BooleanField(default=True)
@@ -30,66 +30,31 @@ class Book(ActivitypubMixin, BookWyrmModel):
     # TODO: edit history
 
     # book/work metadata
-    title = models.CharField(max_length=255)
-    sort_title = models.CharField(max_length=255, blank=True, null=True)
-    subtitle = models.CharField(max_length=255, blank=True, null=True)
-    description = models.TextField(blank=True, null=True)
-    languages = ArrayField(
+    title = fields.CharField(max_length=255)
+    sort_title = fields.CharField(max_length=255, blank=True, null=True)
+    subtitle = fields.CharField(max_length=255, blank=True, null=True)
+    description = fields.TextField(blank=True, null=True)
+    languages = fields.ArrayField(
         models.CharField(max_length=255), blank=True, default=list
     )
-    series = models.CharField(max_length=255, blank=True, null=True)
-    series_number = models.CharField(max_length=255, blank=True, null=True)
-    subjects = ArrayField(
+    series = fields.CharField(max_length=255, blank=True, null=True)
+    series_number = fields.CharField(max_length=255, blank=True, null=True)
+    subjects = fields.ArrayField(
         models.CharField(max_length=255), blank=True, null=True, default=list
     )
-    subject_places = ArrayField(
+    subject_places = fields.ArrayField(
         models.CharField(max_length=255), blank=True, null=True, default=list
     )
     # TODO: include an annotation about the type of authorship (ie, translator)
-    authors = models.ManyToManyField('Author')
+    authors = fields.ManyToManyField('Author')
     # preformatted authorship string for search and easier display
     author_text = models.CharField(max_length=255, blank=True, null=True)
-    cover = models.ImageField(upload_to='covers/', blank=True, null=True)
-    first_published_date = models.DateTimeField(blank=True, null=True)
-    published_date = models.DateTimeField(blank=True, null=True)
+    cover = fields.ImageField(upload_to='covers/', blank=True, null=True)
+    first_published_date = fields.DateTimeField(blank=True, null=True)
+    published_date = fields.DateTimeField(blank=True, null=True)
+
     objects = InheritanceManager()
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-
-        ActivityMapping('authors', 'authors'),
-        ActivityMapping('firstPublishedDate', 'firstpublished_date'),
-        ActivityMapping('publishedDate', 'published_date'),
-
-        ActivityMapping('title', 'title'),
-        ActivityMapping('sortTitle', 'sort_title'),
-        ActivityMapping('subtitle', 'subtitle'),
-        ActivityMapping('description', 'description'),
-        ActivityMapping('languages', 'languages'),
-        ActivityMapping('series', 'series'),
-        ActivityMapping('seriesNumber', 'series_number'),
-        ActivityMapping('subjects', 'subjects'),
-        ActivityMapping('subjectPlaces', 'subject_places'),
-
-        ActivityMapping('openlibraryKey', 'openlibrary_key'),
-        ActivityMapping('librarythingKey', 'librarything_key'),
-        ActivityMapping('goodreadsKey', 'goodreads_key'),
-
-        ActivityMapping('work', 'parent_work'),
-        ActivityMapping('isbn10', 'isbn_10'),
-        ActivityMapping('isbn13', 'isbn_13'),
-        ActivityMapping('oclcNumber', 'oclc_number'),
-        ActivityMapping('asin', 'asin'),
-        ActivityMapping('pages', 'pages'),
-        ActivityMapping('physicalFormat', 'physical_format'),
-        ActivityMapping('publishers', 'publishers'),
-
-        ActivityMapping('lccn', 'lccn'),
-        ActivityMapping('editions', 'editions'),
-        ActivityMapping('defaultEdition', 'default_edition'),
-        ActivityMapping('cover', 'cover'),
-    ]
-
     def save(self, *args, **kwargs):
         ''' can't be abstract for query reasons, but you shouldn't USE it '''
         if not isinstance(self, Edition) and not isinstance(self, Work):
@@ -118,9 +83,9 @@ class Book(ActivitypubMixin, BookWyrmModel):
 class Work(OrderedCollectionPageMixin, Book):
     ''' a work (an abstract concept of a book that manifests in an edition) '''
     # library of congress catalog control number
-    lccn = models.CharField(max_length=255, blank=True, null=True)
+    lccn = fields.CharField(max_length=255, blank=True, null=True)
     # this has to be nullable but should never be null
-    default_edition = models.ForeignKey(
+    default_edition = fields.ForeignKey(
         'Edition',
         on_delete=models.PROTECT,
         null=True
@@ -131,18 +96,19 @@ class Work(OrderedCollectionPageMixin, Book):
         return self.default_edition or self.editions.first()
 
     activity_serializer = activitypub.Work
+    serialize_reverse_fields = ['editions']
 
 
 class Edition(Book):
     ''' an edition of a book '''
     # these identifiers only apply to editions, not works
-    isbn_10 = models.CharField(max_length=255, blank=True, null=True)
-    isbn_13 = models.CharField(max_length=255, blank=True, null=True)
-    oclc_number = models.CharField(max_length=255, blank=True, null=True)
-    asin = models.CharField(max_length=255, blank=True, null=True)
-    pages = models.IntegerField(blank=True, null=True)
-    physical_format = models.CharField(max_length=255, blank=True, null=True)
-    publishers = ArrayField(
+    isbn_10 = fields.CharField(max_length=255, blank=True, null=True)
+    isbn_13 = fields.CharField(max_length=255, blank=True, null=True)
+    oclc_number = fields.CharField(max_length=255, blank=True, null=True)
+    asin = fields.CharField(max_length=255, blank=True, null=True)
+    pages = fields.IntegerField(blank=True, null=True)
+    physical_format = fields.CharField(max_length=255, blank=True, null=True)
+    publishers = fields.ArrayField(
         models.CharField(max_length=255), blank=True, default=list
     )
     shelves = models.ManyToManyField(
@@ -151,8 +117,9 @@ class Edition(Book):
         through='ShelfBook',
         through_fields=('book', 'shelf')
     )
-    parent_work = models.ForeignKey(
-        'Work', on_delete=models.PROTECT, null=True, related_name='editions')
+    parent_work = fields.ForeignKey(
+        'Work', on_delete=models.PROTECT, null=True,
+        related_name='editions', activitypub_field='work')
 
     activity_serializer = activitypub.Edition
     name_field = 'title'
@@ -167,7 +134,6 @@ class Edition(Book):
         return super().save(*args, **kwargs)
 
 
-
 def isbn_10_to_13(isbn_10):
     ''' convert an isbn 10 into an isbn 13 '''
     isbn_10 = re.sub(r'[^0-9X]', '', isbn_10)
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index e419c9b8..b03c5492 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -3,6 +3,7 @@ import re
 from uuid import uuid4
 
 from django.contrib.auth.models import AbstractUser
+from django.contrib.postgres.fields import ArrayField as DjangoArrayField
 from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
 from django.db import models
@@ -97,8 +98,6 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
 class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
     ''' activitypub-aware foreign key field '''
     def to_activity(self, value):
-        print('HIIIII')
-        print(value)
         if not value:
             return None
         return value.to_activity()
@@ -116,7 +115,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
     def to_activity(self, value):
         if self.link_only:
             return '%s/followers' % value.instance.remote_id
-        return [i.remote_id for i in value]
+        return [i.remote_id for i in value.all()]
 
     def from_activity(self, activity_data):
         if self.link_only:
@@ -185,8 +184,14 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
 class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
     ''' activitypub-aware datetime field '''
     def to_activity(self, value):
+        if not value:
+            return None
         return value.isoformat()
 
+class ArrayField(ActivitypubFieldMixin, DjangoArrayField):
+    ''' activitypub-aware array field '''
+    def to_activity(self, value):
+        return [str(i) for i in value]
 
 class CharField(ActivitypubFieldMixin, models.CharField):
     ''' activitypub-aware char field '''

From 1ec2f204864e5b8da3d0fbeb4f7c4a9f007472d9 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 30 Nov 2020 14:54:45 -0800
Subject: [PATCH 026/104] avoid naming clash is to_activity for field vs model

---
 bookwyrm/models/base_model.py   |  6 +++---
 bookwyrm/models/fields.py       | 18 ++++++++---------
 bookwyrm/models/relationship.py | 18 ++++++++---------
 bookwyrm/models/shelf.py        | 34 +++++++++++++--------------------
 bookwyrm/models/status.py       |  2 +-
 5 files changed, 34 insertions(+), 44 deletions(-)

diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 34c4e5bc..af65a36a 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -66,7 +66,7 @@ def unfurl_related_field(related_field):
     if hasattr(related_field, 'all'):
         return [unfurl_related_field(i) for i in related_field.all()]
     if related_field.reverse_unfurl:
-        return related_field.to_activity()
+        return related_field.field_to_activity()
     return related_field.remote_id
 
 
@@ -79,10 +79,10 @@ class ActivitypubMixin:
         ''' convert from a model to an activity '''
         activity = {}
         for field in self.__class__._meta.get_fields():
-            if not hasattr(field, 'to_activity'):
+            if not hasattr(field, 'field_to_activity'):
                 continue
             key = get_field_name(field)
-            value = field.to_activity(getattr(self, field.name))
+            value = field.field_to_activity(getattr(self, field.name))
             if value is None:
                 continue
 
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index b03c5492..197b6e8e 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -33,7 +33,7 @@ class ActivitypubFieldMixin:
             self.activitypub_field = activitypub_field
         super().__init__(*args, **kwargs)
 
-    def to_activity(self, value):
+    def field_to_activity(self, value):
         ''' formatter to convert a model value into activitypub '''
         if hasattr(self, 'activitypub_wrapper'):
             value = {self.activitypub_wrapper: value}
@@ -81,13 +81,13 @@ class UsernameField(ActivitypubFieldMixin, models.CharField):
         del kwargs['error_messages']
         return name, path, args, kwargs
 
-    def to_activity(self, value):
+    def field_to_activity(self, value):
         return value.split('@')[0]
 
 
 class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
     ''' activitypub-aware foreign key field '''
-    def to_activity(self, value):
+    def field_to_activity(self, value):
         if not value:
             return None
         return value.remote_id
@@ -97,7 +97,7 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
 
 class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
     ''' activitypub-aware foreign key field '''
-    def to_activity(self, value):
+    def field_to_activity(self, value):
         if not value:
             return None
         return value.to_activity()
@@ -112,7 +112,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
         self.link_only = link_only
         super().__init__(*args, **kwargs)
 
-    def to_activity(self, value):
+    def field_to_activity(self, value):
         if self.link_only:
             return '%s/followers' % value.instance.remote_id
         return [i.remote_id for i in value.all()]
@@ -129,7 +129,7 @@ class TagField(ManyToManyField):
         super().__init__(*args, **kwargs)
         self.activitypub_field = 'tag'
 
-    def to_activity(self, value):
+    def field_to_activity(self, value):
         tags = []
         for item in value.all():
             activity_type = item.__class__.__name__
@@ -156,7 +156,7 @@ def image_serializer(value):
 
 class ImageField(ActivitypubFieldMixin, models.ImageField):
     ''' activitypub-aware image field '''
-    def to_activity(self, value):
+    def field_to_activity(self, value):
         return image_serializer(value)
 
     def from_activity(self, activity_data):
@@ -183,14 +183,14 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
 
 class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
     ''' activitypub-aware datetime field '''
-    def to_activity(self, value):
+    def field_to_activity(self, value):
         if not value:
             return None
         return value.isoformat()
 
 class ArrayField(ActivitypubFieldMixin, DjangoArrayField):
     ''' activitypub-aware array field '''
-    def to_activity(self, value):
+    def field_to_activity(self, value):
         return [str(i) for i in value]
 
 class CharField(ActivitypubFieldMixin, models.CharField):
diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py
index dbf99778..8913b9ab 100644
--- a/bookwyrm/models/relationship.py
+++ b/bookwyrm/models/relationship.py
@@ -2,20 +2,23 @@
 from django.db import models
 
 from bookwyrm import activitypub
-from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel
+from .base_model import ActivitypubMixin, BookWyrmModel
+from . import fields
 
 
 class UserRelationship(ActivitypubMixin, BookWyrmModel):
     ''' many-to-many through table for followers '''
-    user_subject = models.ForeignKey(
+    user_subject = fields.ForeignKey(
         'User',
         on_delete=models.PROTECT,
-        related_name='%(class)s_user_subject'
+        related_name='%(class)s_user_subject',
+        activitypub_field='actor',
     )
-    user_object = models.ForeignKey(
+    user_object = fields.ForeignKey(
         'User',
         on_delete=models.PROTECT,
-        related_name='%(class)s_user_object'
+        related_name='%(class)s_user_object',
+        activitypub_field='object',
     )
 
     class Meta:
@@ -32,11 +35,6 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel):
             )
         ]
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('actor', 'user_subject'),
-        ActivityMapping('object', 'user_object'),
-    ]
     activity_serializer = activitypub.Follow
 
     def get_remote_id(self, status=None):
diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py
index b8d5ea17..fc63d198 100644
--- a/bookwyrm/models/shelf.py
+++ b/bookwyrm/models/shelf.py
@@ -3,17 +3,19 @@ import re
 from django.db import models
 
 from bookwyrm import activitypub
-from .base_model import ActivityMapping, BookWyrmModel
+from .base_model import BookWyrmModel
 from .base_model import OrderedCollectionMixin, PrivacyLevels
+from . import fields
 
 
 class Shelf(OrderedCollectionMixin, BookWyrmModel):
     ''' a list of books owned by a user '''
-    name = models.CharField(max_length=100)
+    name = fields.CharField(max_length=100)
     identifier = models.CharField(max_length=100)
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='owner')
     editable = models.BooleanField(default=True)
-    privacy = models.CharField(
+    privacy = fields.CharField(
         max_length=255,
         default='public',
         choices=PrivacyLevels.choices
@@ -48,31 +50,21 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
         ''' user/shelf unqiueness '''
         unique_together = ('user', 'identifier')
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('owner', 'user'),
-        ActivityMapping('name', 'name'),
-    ]
-
 
 class ShelfBook(BookWyrmModel):
     ''' many to many join table for books and shelves '''
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
-    shelf = models.ForeignKey('Shelf', on_delete=models.PROTECT)
-    added_by = models.ForeignKey(
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='object')
+    shelf = fields.ForeignKey(
+        'Shelf', on_delete=models.PROTECT, activitypub_field='target')
+    added_by = fields.ForeignKey(
         'User',
         blank=True,
         null=True,
-        on_delete=models.PROTECT
+        on_delete=models.PROTECT,
+        activitypub_field='actor'
     )
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('actor', 'added_by'),
-        ActivityMapping('object', 'book'),
-        ActivityMapping('target', 'shelf'),
-    ]
-
     activity_serializer = activitypub.AddBook
 
     def to_add_activity(self, user):
diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index c47a33bb..371a6345 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -89,7 +89,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         mentions = [u.remote_id for u in self.mention_users.all()]
         # this is a link to the followers list:
         followers = self.user.__class__._meta.get_field('followers')\
-                .to_activity(self.user.followers)
+                .field_to_activity(self.user.followers)
         if self.privacy == 'public':
             activity['to'] = [public]
             activity['cc'] = [followers] + mentions

From eb6206252d7509b45f79fe06a769adb3f7c56f88 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 30 Nov 2020 19:01:43 -0800
Subject: [PATCH 027/104] cleans up ordered collection mixin

---
 bookwyrm/models/base_model.py | 126 ++++++++++------------------------
 bookwyrm/models/fields.py     |  17 +++--
 bookwyrm/models/tag.py        |  26 +++----
 bookwyrm/models/user.py       |   4 +-
 4 files changed, 59 insertions(+), 114 deletions(-)

diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index af65a36a..0f317dbc 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -1,18 +1,16 @@
 ''' base model with default fields '''
 from base64 import b64encode
-from dataclasses import dataclass
-from typing import Callable
 from uuid import uuid4
-from urllib.parse import urlencode
 
 from Crypto.PublicKey import RSA
 from Crypto.Signature import pkcs1_15
 from Crypto.Hash import SHA256
+from django.core.paginator import Paginator
 from django.db import models
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import DOMAIN, PAGE_LENGTH
 from .fields import RemoteIdField
 
 
@@ -52,15 +50,6 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
         instance.save()
 
 
-def get_field_name(field):
-    ''' model_field_name to activitypubFieldName '''
-    if field.activitypub_field:
-        return field.activitypub_field
-    name = field.name.split('.')[-1]
-    components = name.split('_')
-    return components[0] + ''.join(x.title() for x in components[1:])
-
-
 def unfurl_related_field(related_field):
     ''' load reverse lookups (like public key owner or Status attachment '''
     if hasattr(related_field, 'all'):
@@ -78,15 +67,16 @@ class ActivitypubMixin:
     def to_activity(self):
         ''' convert from a model to an activity '''
         activity = {}
-        for field in self.__class__._meta.get_fields():
+        for field in self._meta.get_fields():
             if not hasattr(field, 'field_to_activity'):
                 continue
-            key = get_field_name(field)
             value = field.field_to_activity(getattr(self, field.name))
             if value is None:
                 continue
 
+            key = field.get_activitypub_field()
             if key in activity and isinstance(activity[key], list):
+                # handles tags on status, which accumulate across fields
                 activity[key] += value
             else:
                 activity[key] = value
@@ -125,15 +115,12 @@ class ActivitypubMixin:
 
     def to_delete_activity(self, user):
         ''' notice of deletion '''
-        # this should be a tombstone
-        activity_object = self.to_activity()
-
         return activitypub.Delete(
             id=self.remote_id + '/activity',
             actor=user.remote_id,
             to=['%s/followers' % user.remote_id],
             cc=['https://www.w3.org/ns/activitystreams#Public'],
-            object=activity_object,
+            object=self.to_activity(),
         ).serialize()
 
 
@@ -165,81 +152,53 @@ class OrderedCollectionPageMixin(ActivitypubMixin):
         ''' this can be overriden if there's a special remote id, ie outbox '''
         return self.remote_id
 
-    def page(self, min_id=None, max_id=None):
-        ''' helper function to create the pagination url '''
-        params = {'page': 'true'}
-        if min_id:
-            params['min_id'] = min_id
-        if max_id:
-            params['max_id'] = max_id
-        return '?%s' % urlencode(params)
-
-    def next_page(self, items):
-        ''' use the max id of the last item '''
-        if not items.count():
-            return ''
-        return self.page(max_id=items[items.count() - 1].id)
-
-    def prev_page(self, items):
-        ''' use the min id of the first item '''
-        if not items.count():
-            return ''
-        return self.page(min_id=items[0].id)
-
-    def to_ordered_collection_page(self, queryset, remote_id, \
-            id_only=False, min_id=None, max_id=None):
-        ''' serialize and pagiante a queryset '''
-        # TODO: weird place to define this
-        limit = 20
-        # filters for use in the django queryset min/max
-        filters = {}
-        if min_id is not None:
-            filters['id__gt'] = min_id
-        if max_id is not None:
-            filters['id__lte'] = max_id
-        page_id = self.page(min_id=min_id, max_id=max_id)
-
-        items = queryset.filter(
-            **filters
-        ).all()[:limit]
-
-        if id_only:
-            page = [s.remote_id for s in items]
-        else:
-            page = [s.to_activity() for s in items]
-        return activitypub.OrderedCollectionPage(
-            id='%s%s' % (remote_id, page_id),
-            partOf=remote_id,
-            orderedItems=page,
-            next='%s%s' % (remote_id, self.next_page(items)),
-            prev='%s%s' % (remote_id, self.prev_page(items))
-        ).serialize()
 
     def to_ordered_collection(self, queryset, \
             remote_id=None, page=False, **kwargs):
         ''' an ordered collection of whatevers '''
         remote_id = remote_id or self.remote_id
         if page:
-            return self.to_ordered_collection_page(
+            return to_ordered_collection_page(
                 queryset, remote_id, **kwargs)
-        name = ''
-        if hasattr(self, 'name'):
-            name = self.name
-        owner = ''
-        if hasattr(self, 'user'):
-            owner = self.user.remote_id
+        name = self.name if hasattr(self, 'name') else None
+        owner = self.user.remote_id if hasattr(self, 'user') else ''
 
-        size = queryset.count()
+        paginated = Paginator(queryset, PAGE_LENGTH)
         return activitypub.OrderedCollection(
             id=remote_id,
-            totalItems=size,
+            totalItems=paginated.count,
             name=name,
             owner=owner,
-            first='%s%s' % (remote_id, self.page()),
-            last='%s%s' % (remote_id, self.page(min_id=0))
+            first='%s?page=1' % remote_id,
+            last='%s?page=%d' % (remote_id, paginated.num_pages)
         ).serialize()
 
 
+def to_ordered_collection_page(queryset, remote_id, id_only=False, page=1):
+    ''' serialize and pagiante a queryset '''
+    paginated = Paginator(queryset, PAGE_LENGTH)
+
+    activity_page = paginated.page(page)
+    if id_only:
+        items = [s.remote_id for s in activity_page.object_list]
+    else:
+        items = [s.to_activity() for s in activity_page.object_list]
+
+    prev_page = next_page = None
+    if activity_page.has_next():
+        next_page = '%s?page=%d' % (remote_id, activity_page.next_page_number())
+    if activity_page.has_previous():
+        prev_page = '%s?page=%d' % \
+                (remote_id, activity_page.previous_page_number())
+    return activitypub.OrderedCollectionPage(
+        id='%s?page=%s' % (remote_id, page),
+        partOf=remote_id,
+        orderedItems=items,
+        next=next_page,
+        prev=prev_page
+    ).serialize()
+
+
 class OrderedCollectionMixin(OrderedCollectionPageMixin):
     ''' extends activitypub models to work as ordered collections '''
     @property
@@ -252,12 +211,3 @@ class OrderedCollectionMixin(OrderedCollectionPageMixin):
     def to_activity(self, **kwargs):
         ''' an ordered collection of the specified model queryset  '''
         return self.to_ordered_collection(self.collection_queryset, **kwargs)
-
-
-@dataclass(frozen=True)
-class ActivityMapping:
-    ''' translate between an activitypub json field and a model field '''
-    activity_key: str
-    model_key: str
-    activity_formatter: Callable = lambda x: x
-    model_formatter: Callable = lambda x: x
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 197b6e8e..ab0f5892 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -36,7 +36,7 @@ class ActivitypubFieldMixin:
     def field_to_activity(self, value):
         ''' formatter to convert a model value into activitypub '''
         if hasattr(self, 'activitypub_wrapper'):
-            value = {self.activitypub_wrapper: value}
+            return {self.activitypub_wrapper: value}
         return value
 
     def from_activity(self, activity_data):
@@ -46,6 +46,14 @@ class ActivitypubFieldMixin:
             value = value.get(self.activitypub_wrapper)
         return value
 
+    def get_activitypub_field(self):
+        ''' model_field_name to activitypubFieldName '''
+        if self.activitypub_field:
+            return self.activitypub_field
+        name = self.name.split('.')[-1]
+        components = name.split('_')
+        return components[0] + ''.join(x.title() for x in components[1:])
+
 
 class RemoteIdField(ActivitypubFieldMixin, models.CharField):
     ''' a url that serves as a unique identifier '''
@@ -91,8 +99,6 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
         if not value:
             return None
         return value.remote_id
-    def from_activity(self, activity_data):
-        pass# TODO
 
 
 class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
@@ -102,9 +108,6 @@ class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
             return None
         return value.to_activity()
 
-    def from_activity(self, activity_data):
-        pass# TODO
-
 
 class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
     ''' activitypub-aware many to many field '''
@@ -123,6 +126,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
         values = super().from_activity(activity_data)
         return values# TODO
 
+
 class TagField(ManyToManyField):
     ''' special case of many to many that uses Tags '''
     def __init__(self, *args, **kwargs):
@@ -145,7 +149,6 @@ class TagField(ManyToManyField):
 
 def image_serializer(value):
     ''' helper for serializing images '''
-    print(value)
     if value and hasattr(value, 'url'):
         url = value.url
     else:
diff --git a/bookwyrm/models/tag.py b/bookwyrm/models/tag.py
index 8b7efceb..940b4192 100644
--- a/bookwyrm/models/tag.py
+++ b/bookwyrm/models/tag.py
@@ -5,19 +5,15 @@ from django.db import models
 
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
-from .base_model import OrderedCollectionMixin, BookWyrmModel, ActivityMapping
+from .base_model import OrderedCollectionMixin, BookWyrmModel
+from . import fields
 
 
 class Tag(OrderedCollectionMixin, BookWyrmModel):
     ''' freeform tags for books '''
-    name = models.CharField(max_length=100, unique=True)
+    name = fields.CharField(max_length=100, unique=True)
     identifier = models.CharField(max_length=100)
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('name', 'name'),
-    ]
-
     @classmethod
     def book_queryset(cls, identifier):
         ''' county of books associated with this tag '''
@@ -44,16 +40,12 @@ class Tag(OrderedCollectionMixin, BookWyrmModel):
 
 class UserTag(BookWyrmModel):
     ''' an instance of a tag on a book by a user '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
-    tag = models.ForeignKey('Tag', on_delete=models.PROTECT)
-
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('actor', 'user'),
-        ActivityMapping('object', 'book'),
-        ActivityMapping('target', 'tag'),
-    ]
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='actor')
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='object')
+    tag = fields.ForeignKey(
+        'Tag', on_delete=models.PROTECT, activitypub_field='target')
 
     activity_serializer = activitypub.AddBook
 
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index 95dd1e79..0097fcde 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -109,13 +109,13 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     def to_following_activity(self, **kwargs):
         ''' activitypub following list '''
         remote_id = '%s/following' % self.remote_id
-        return self.to_ordered_collection(self.following, \
+        return self.to_ordered_collection(self.following.all(), \
                 remote_id=remote_id, id_only=True, **kwargs)
 
     def to_followers_activity(self, **kwargs):
         ''' activitypub followers list '''
         remote_id = '%s/followers' % self.remote_id
-        return self.to_ordered_collection(self.followers, \
+        return self.to_ordered_collection(self.followers.all(), \
                 remote_id=remote_id, id_only=True, **kwargs)
 
     def to_activity(self):

From fee5846aa88293c22778ce6e2529ab4679580f40 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 30 Nov 2020 19:33:50 -0800
Subject: [PATCH 028/104] Fixes generating new key paris for user

and the broadcast test
---
 bookwyrm/broadcast.py                    |  2 +-
 bookwyrm/incoming.py                     |  8 ++++----
 bookwyrm/models/user.py                  |  5 +++--
 bookwyrm/tests/models/test_user_model.py |  4 ++--
 bookwyrm/tests/test_broadcast.py         | 11 +++++------
 5 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/bookwyrm/broadcast.py b/bookwyrm/broadcast.py
index a1eebaee..a98b6774 100644
--- a/bookwyrm/broadcast.py
+++ b/bookwyrm/broadcast.py
@@ -65,7 +65,7 @@ def sign_and_send(sender, data, destination):
     ''' crpyto whatever and http junk '''
     now = http_date()
 
-    if not sender.private_key:
+    if not sender.key_pair.private_key:
         # this shouldn't happen. it would be bad if it happened.
         raise ValueError('No private key found for sender')
 
diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index 017be19d..b5b2523d 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -98,15 +98,15 @@ def has_valid_signature(request, activity):
         remote_user = activitypub.resolve_remote_id(models.User, key_actor)
 
         try:
-            signature.verify(remote_user.public_key, request)
+            signature.verify(remote_user.key_pair.public_key, request)
         except ValueError:
-            old_key = remote_user.public_key
+            old_key = remote_user.key_pair.public_key
             activitypub.resolve_remote_id(
                 models.User, remote_user, refresh=True
             )
-            if remote_user.public_key == old_key:
+            if remote_user.key_pair.public_key == old_key:
                 raise # Key unchanged.
-            signature.verify(remote_user.public_key, request)
+            signature.verify(remote_user.key_pair.public_key, request)
     except (ValueError, requests.exceptions.HTTPError):
         return False
     return True
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index 0097fcde..a200ec51 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -155,8 +155,6 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         self.inbox = '%s/inbox' % self.remote_id
         self.shared_inbox = 'https://%s/inbox' % DOMAIN
         self.outbox = '%s/outbox' % self.remote_id
-        if not self.key_pair:
-            self.key_pair = KeyPair.objects.create()
 
         return super().save(*args, **kwargs)
 
@@ -197,6 +195,9 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
     if not instance.local:
         set_remote_server.delay(instance.id)
 
+    instance.key_pair = KeyPair.objects.create(
+        remote_id='%s/#main-key' % instance.remote_id)
+
     shelves = [{
         'name': 'To Read',
         'identifier': 'to-read',
diff --git a/bookwyrm/tests/models/test_user_model.py b/bookwyrm/tests/models/test_user_model.py
index e82f91be..c648bf6b 100644
--- a/bookwyrm/tests/models/test_user_model.py
+++ b/bookwyrm/tests/models/test_user_model.py
@@ -19,8 +19,8 @@ class User(TestCase):
         self.assertEqual(self.user.shared_inbox, 'https://%s/inbox' % DOMAIN)
         self.assertEqual(self.user.inbox, '%s/inbox' % expected_id)
         self.assertEqual(self.user.outbox, '%s/outbox' % expected_id)
-        self.assertIsNotNone(self.user.private_key)
-        self.assertIsNotNone(self.user.public_key)
+        self.assertIsNotNone(self.user.key_pair.private_key)
+        self.assertIsNotNone(self.user.key_pair.public_key)
 
 
     def test_user_shelves(self):
diff --git a/bookwyrm/tests/test_broadcast.py b/bookwyrm/tests/test_broadcast.py
index ae4e6145..b8051219 100644
--- a/bookwyrm/tests/test_broadcast.py
+++ b/bookwyrm/tests/test_broadcast.py
@@ -6,13 +6,12 @@ from bookwyrm import models, broadcast
 
 class Book(TestCase):
     def setUp(self):
-        with patch('bookwyrm.models.user.get_remote_reviews.delay'):
-            self.user = models.User.objects.create_user(
-                'mouse', 'mouse@mouse.mouse', 'mouseword')
+        self.user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
 
-            local_follower = models.User.objects.create_user(
-                'joe', 'joe@mouse.mouse', 'jeoword')
-            self.user.followers.add(local_follower)
+        local_follower = models.User.objects.create_user(
+            'joe', 'joe@mouse.mouse', 'jeoword', local=True)
+        self.user.followers.add(local_follower)
 
         with patch('bookwyrm.models.user.set_remote_server.delay'):
             follower = models.User.objects.create_user(

From 1610d81ce673f99d7e2b2690486a58a59c0a01d1 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 30 Nov 2020 19:53:42 -0800
Subject: [PATCH 029/104] fixes some of the signing test issues

---
 bookwyrm/incoming.py           |  2 ++
 bookwyrm/signatures.py         |  2 +-
 bookwyrm/tests/test_signing.py | 19 +++++++++++--------
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index b5b2523d..9a7c6e63 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -96,6 +96,8 @@ def has_valid_signature(request, activity):
             raise ValueError("Wrong actor created signature.")
 
         remote_user = activitypub.resolve_remote_id(models.User, key_actor)
+        if not remote_user:
+            return False
 
         try:
             signature.verify(remote_user.key_pair.public_key, request)
diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py
index dbb88d8a..a3e1fccc 100644
--- a/bookwyrm/signatures.py
+++ b/bookwyrm/signatures.py
@@ -31,7 +31,7 @@ def make_signature(sender, destination, date, digest):
         'digest: %s' % digest,
     ]
     message_to_sign = '\n'.join(signature_headers)
-    signer = pkcs1_15.new(RSA.import_key(sender.private_key))
+    signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key))
     signed_message = signer.sign(SHA256.new(message_to_sign.encode('utf8')))
     signature = {
         'keyId': '%s#main-key' % sender.remote_id,
diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py
index 129a4333..58e8cb7f 100644
--- a/bookwyrm/tests/test_signing.py
+++ b/bookwyrm/tests/test_signing.py
@@ -25,20 +25,23 @@ def get_follow_data(follower, followee):
     ).serialize()
     return json.dumps(follow_activity)
 
-Sender = namedtuple('Sender', ('remote_id', 'private_key', 'public_key'))
+KeyPair = namedtuple('KeyPair', ('private_key', 'public_key'))
+Sender = namedtuple('Sender', ('remote_id', 'key_pair'))
 
 class Signature(TestCase):
     def setUp(self):
-        self.mouse = User.objects.create_user('mouse', 'mouse@example.com', '')
-        self.rat = User.objects.create_user('rat', 'rat@example.com', '')
-        self.cat = User.objects.create_user('cat', 'cat@example.com', '')
+        self.mouse = User.objects.create_user(
+            'mouse', 'mouse@example.com', '', local=True)
+        self.rat = User.objects.create_user(
+            'rat', 'rat@example.com', '', local=True)
+        self.cat = User.objects.create_user(
+            'cat', 'cat@example.com', '', local=True)
 
         private_key, public_key = create_key_pair()
 
         self.fake_remote = Sender(
             'http://localhost/user/remote',
-            private_key,
-            public_key,
+            KeyPair(private_key, public_key)
         )
 
     def send(self, signature, now, data, digest):
@@ -89,7 +92,7 @@ class Signature(TestCase):
         datafile = pathlib.Path(__file__).parent.joinpath('data/ap_user.json')
         data = json.loads(datafile.read_bytes())
         data['id'] = self.fake_remote.remote_id
-        data['publicKey']['publicKeyPem'] = self.fake_remote.public_key
+        data['publicKey']['publicKeyPem'] = self.fake_remote.key_pair.public_key
         del data['icon'] # Avoid having to return an avatar.
         responses.add(
             responses.GET,
@@ -116,7 +119,7 @@ class Signature(TestCase):
         datafile = pathlib.Path(__file__).parent.joinpath('data/ap_user.json')
         data = json.loads(datafile.read_bytes())
         data['id'] = self.fake_remote.remote_id
-        data['publicKey']['publicKeyPem'] = self.fake_remote.public_key
+        data['publicKey']['publicKeyPem'] = self.fake_remote.key_pair.public_key
         del data['icon'] # Avoid having to return an avatar.
         responses.add(
             responses.GET,

From a85043b351d9b3f70c3860c4733def6ba77635f0 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 12:35:57 -0800
Subject: [PATCH 030/104] Updates to_model to use fields

---
 bookwyrm/activitypub/base_activity.py         | 82 ++++++-------------
 bookwyrm/connectors/abstract_connector.py     |  3 +-
 .../migrations/0019_auto_20201130_1939.py     | 12 +++
 bookwyrm/models/__init__.py                   |  5 ++
 bookwyrm/models/base_model.py                 |  2 +
 bookwyrm/models/fields.py                     | 36 ++++++--
 bookwyrm/models/user.py                       |  4 +-
 7 files changed, 78 insertions(+), 66 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 9e1b5b82..786a2aab 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -1,20 +1,13 @@
 ''' basics for an activitypub serializer '''
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
-from uuid import uuid4
 
-import dateutil.parser
-from dateutil.parser import ParserError
-from django.core.files.base import ContentFile
 from django.db.models.fields.related_descriptors \
     import ForwardManyToOneDescriptor, ManyToManyDescriptor, \
         ReverseManyToOneDescriptor
-from django.db.models.fields import DateTimeField
 from django.db.models.fields.files import ImageFileDescriptor
-from django.db.models.query_utils import DeferredAttribute
-from django.utils import timezone
 
-from bookwyrm.connectors import ConnectorException, get_data, get_image
+from bookwyrm.connectors import ConnectorException, get_data
 
 class ActivitySerializerError(ValueError):
     ''' routine problems serializing activitypub json '''
@@ -84,35 +77,28 @@ class ActivityObject:
         # check for an existing instance, if we're not updating a known obj
         if not instance:
             instance = find_existing_by_remote_id(model, self.id)
+	# TODO: deduplicate books by identifiers
 
-        model_fields = [m.name for m in model._meta.get_fields()]
         mapped_fields = {}
         many_to_many_fields = {}
         one_to_many_fields = {}
         image_fields = {}
 
-        for mapping in model.activity_mappings:
-            if mapping.model_key not in model_fields:
+        for field in model._meta.get_fields():
+            if not hasattr(field, 'field_to_activity'):
                 continue
+            activitypub_field = field.get_activitypub_field()
+            value = field.field_from_activity(getattr(self, activitypub_field))
+            if value is None:
+                continue
+
             # value is None if there's a default that isn't supplied
             # in the activity but is supplied in the formatter
-            value = None
-            if mapping.activity_key:
-                value = getattr(self, mapping.activity_key)
-            model_field = getattr(model, mapping.model_key)
+            value = getattr(self, activitypub_field)
+            model_field = getattr(model, field.name)
 
-            formatted_value = mapping.model_formatter(value)
-            if isinstance(model_field, DeferredAttribute) and \
-                    isinstance(model_field.field, DateTimeField):
-                try:
-                    date_value = dateutil.parser.parse(formatted_value)
-                    try:
-                        formatted_value = timezone.make_aware(date_value)
-                    except ValueError:
-                        formatted_value = date_value
-                except (ParserError, TypeError):
-                    formatted_value = None
-            elif isinstance(model_field, ForwardManyToOneDescriptor):
+            formatted_value = field.field_from_activity(value)
+            if isinstance(model_field, ForwardManyToOneDescriptor):
                 if not formatted_value:
                     continue
                 # foreign key remote id reolver (work on Edition, for example)
@@ -120,25 +106,30 @@ class ActivityObject:
                 if isinstance(formatted_value, dict) and \
                         formatted_value.get('id'):
                     # if the AP field is a serialized object (as in Add)
-                    remote_id = formatted_value['id']
+                    # or PublicKey
+                    related_model = field.related_model
+                    related_activity = related_model.activity_serializer
+                    mapped_fields[field.name] = related_activity(
+                        **formatted_value
+                    ).to_model(related_model)
                 else:
                     # if the field is just a remote_id (as in every other case)
                     remote_id = formatted_value
-                reference = resolve_remote_id(fk_model, remote_id)
-                mapped_fields[mapping.model_key] = reference
+                    reference = resolve_remote_id(fk_model, remote_id)
+                    mapped_fields[field.name] = reference
             elif isinstance(model_field, ManyToManyDescriptor):
                 # status mentions book/users
-                many_to_many_fields[mapping.model_key] = formatted_value
+                many_to_many_fields[field.name] = formatted_value
             elif isinstance(model_field, ReverseManyToOneDescriptor):
                 # attachments on Status, for example
-                one_to_many_fields[mapping.model_key] = formatted_value
+                one_to_many_fields[field.name] = formatted_value
             elif isinstance(model_field, ImageFileDescriptor):
                 # image fields need custom handling
-                image_fields[mapping.model_key] = formatted_value
+                image_fields[field.name] = formatted_value
             else:
                 if formatted_value == MISSING:
                     formatted_value = None
-                mapped_fields[mapping.model_key] = formatted_value
+                mapped_fields[field.name] = formatted_value
 
         if instance:
             # updating an existing model instance
@@ -154,7 +145,6 @@ class ActivityObject:
 
         # add images
         for (model_key, value) in image_fields.items():
-            formatted_value = image_formatter(value)
             if not formatted_value:
                 continue
             getattr(instance, model_key).save(*formatted_value, save=True)
@@ -243,25 +233,3 @@ def resolve_remote_id(model, remote_id, refresh=False):
     item = model.activity_serializer(**data)
     # if we're refreshing, "result" will be set and we'll update it
     return item.to_model(model, instance=result)
-
-
-def image_formatter(image_slug):
-    ''' helper function to load images and format them for a model '''
-    # when it's an inline image (User avatar/icon, Book cover), it's a json
-    # blob, but when it's an attached image, it's just a url
-    if isinstance(image_slug, dict):
-        url = image_slug.get('url')
-    elif isinstance(image_slug, str):
-        url = image_slug
-    else:
-        return None
-    if not url:
-        return None
-
-    response = get_image(url)
-    if not response:
-        return None
-
-    image_name = str(uuid4()) + '.' + url.split('.')[-1]
-    image_content = ContentFile(response.content)
-    return [image_name, image_content]
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index 4e756d8c..8c5d9658 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -8,6 +8,7 @@ from django.db import transaction
 from dateutil import parser
 import requests
 from requests import HTTPError
+from requests.exceptions import SSLError
 
 from bookwyrm import models
 
@@ -322,7 +323,7 @@ def get_image(url):
     ''' wrapper for requesting an image '''
     try:
         resp = requests.get(url)
-    except RequestError:
+    except (RequestError, SSLError):
         return None
     if not resp.ok:
         return None
diff --git a/bookwyrm/migrations/0019_auto_20201130_1939.py b/bookwyrm/migrations/0019_auto_20201130_1939.py
index c69bdbce..11cf6a3b 100644
--- a/bookwyrm/migrations/0019_auto_20201130_1939.py
+++ b/bookwyrm/migrations/0019_auto_20201130_1939.py
@@ -3,6 +3,17 @@
 import bookwyrm.models.fields
 from django.db import migrations
 
+def update_notnull(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    users = app_registry.get_model('bookwyrm', 'User')
+    for user in users.objects.using(db_alias):
+        if user.name and user.summary:
+            continue
+        if not user.summary:
+            user.summary = ''
+        if not user.name:
+            user.name = ''
+        user.save()
 
 class Migration(migrations.Migration):
 
@@ -11,6 +22,7 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.RunPython(update_notnull),
         migrations.AlterField(
             model_name='user',
             name='name',
diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py
index 86bdf219..b9a2814e 100644
--- a/bookwyrm/models/__init__.py
+++ b/bookwyrm/models/__init__.py
@@ -25,3 +25,8 @@ from .site import SiteSettings, SiteInvite, PasswordReset
 cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass)
 activity_models = {c[1].activity_serializer.__name__: c[1] \
     for c in cls_members if hasattr(c[1], 'activity_serializer')}
+
+def to_activity(activity_json):
+    ''' link up models and activities '''
+    activity_type = activity_json.get('type')
+    return activity_models[activity_type].to_activity(activity_json)
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 0f317dbc..520864c2 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -80,7 +80,9 @@ class ActivitypubMixin:
                 activity[key] += value
             else:
                 activity[key] = value
+
         if hasattr(self, 'serialize_reverse_fields'):
+            # for example, editions of a work
             for field_name in self.serialize_reverse_fields:
                 related_field = getattr(self, field_name)
                 activity[field_name] = unfurl_related_field(related_field)
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index ab0f5892..960f3605 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -2,11 +2,14 @@
 import re
 from uuid import uuid4
 
+import dateutil.parser
+from dateutil.parser import ParserError
 from django.contrib.auth.models import AbstractUser
 from django.contrib.postgres.fields import ArrayField as DjangoArrayField
 from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
 from django.db import models
+from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
@@ -39,10 +42,9 @@ class ActivitypubFieldMixin:
             return {self.activitypub_wrapper: value}
         return value
 
-    def from_activity(self, activity_data):
+    def field_from_activity(self, value):
         ''' formatter to convert activitypub into a model value '''
-        value = activity_data.get(self.activitypub_field)
-        if self.activitypub_wrapper:
+        if hasattr(self, 'activitypub_wrapper'):
             value = value.get(self.activitypub_wrapper)
         return value
 
@@ -100,6 +102,16 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
             return None
         return value.remote_id
 
+    def field_from_activity(self, value):
+        if isinstance(value, dict) and value.get('id'):
+            # if the AP field is a serialized object (as in Add)
+            remote_id = value['id']
+        else:
+            # if the field is just a remote_id (as in every other case)
+            remote_id = value
+
+        return resolve_remote_id(remote_id)
+
 
 class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
     ''' activitypub-aware foreign key field '''
@@ -120,10 +132,10 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
             return '%s/followers' % value.instance.remote_id
         return [i.remote_id for i in value.all()]
 
-    def from_activity(self, activity_data):
+    def field_from_activity(self, valueactivity_data):
         if self.link_only:
             return None
-        values = super().from_activity(activity_data)
+        values = super().field_from_activity(values)
         return values# TODO
 
 
@@ -162,8 +174,8 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
     def field_to_activity(self, value):
         return image_serializer(value)
 
-    def from_activity(self, activity_data):
-        image_slug = super().from_activity(activity_data)
+    def field_from_activity(self, value):
+        image_slug = value
         # when it's an inline image (User avatar/icon, Book cover), it's a json
         # blob, but when it's an attached image, it's just a url
         if isinstance(image_slug, dict):
@@ -191,6 +203,16 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
             return None
         return value.isoformat()
 
+    def field_from_activity(self, value):
+        try:
+            date_value = dateutil.parser.parse(value)
+            try:
+                return timezone.make_aware(date_value)
+            except ValueError:
+                return date_value
+        except (ParserError, TypeError):
+            return None
+
 class ArrayField(ActivitypubFieldMixin, DjangoArrayField):
     ''' activitypub-aware array field '''
     def field_to_activity(self, value):
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index a200ec51..ecfc3459 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -174,7 +174,8 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
 
     def save(self, *args, **kwargs):
         ''' create a key pair '''
-        self.private_key, self.public_key = create_key_pair()
+        if not self.public_key:
+            self.private_key, self.public_key = create_key_pair()
         return super().save(*args, **kwargs)
 
     def to_activity(self):
@@ -194,6 +195,7 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
 
     if not instance.local:
         set_remote_server.delay(instance.id)
+        return
 
     instance.key_pair = KeyPair.objects.create(
         remote_id='%s/#main-key' % instance.remote_id)

From 77a1fc26f13cc4b133784c73d9d3c42ea801c41e Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 12:45:01 -0800
Subject: [PATCH 031/104] Save key pair when creating new User

---
 bookwyrm/models/user.py        | 1 +
 bookwyrm/tests/test_signing.py | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index ecfc3459..ac729fd3 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -199,6 +199,7 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
 
     instance.key_pair = KeyPair.objects.create(
         remote_id='%s/#main-key' % instance.remote_id)
+    instance.save()
 
     shelves = [{
         'name': 'To Read',
diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py
index 58e8cb7f..7427dbdf 100644
--- a/bookwyrm/tests/test_signing.py
+++ b/bookwyrm/tests/test_signing.py
@@ -138,10 +138,10 @@ class Signature(TestCase):
         )
 
         # Second and subsequent fetches get a different key:
-        new_private_key, new_public_key = create_key_pair()
+        key_pair = KeyPair(*create_key_pair())
         new_sender = Sender(
-            self.fake_remote.remote_id, new_private_key, new_public_key)
-        data['publicKey']['publicKeyPem'] = new_public_key
+            self.fake_remote.remote_id, key_pair)
+        data['publicKey']['publicKeyPem'] = key_pair.public_key
         responses.add(
             responses.GET,
             self.fake_remote.remote_id,

From 6d137ccadadc3389bb70f592632c2f1853737714 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 12:50:21 -0800
Subject: [PATCH 032/104] mock celery task in test signing

---
 bookwyrm/tests/test_signing.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py
index 7427dbdf..f2997ff9 100644
--- a/bookwyrm/tests/test_signing.py
+++ b/bookwyrm/tests/test_signing.py
@@ -73,8 +73,9 @@ class Signature(TestCase):
         digest = digest or make_digest(data)
         signature = make_signature(
             signer or sender, self.rat.inbox, now, digest)
-        with patch('bookwyrm.incoming.handle_follow.delay') as _:
-            return self.send(signature, now, send_data or data, digest)
+        with patch('bookwyrm.incoming.handle_follow.delay'):
+            with patch('bookwyrm.models.user.set_remote_server.delay'):
+                return self.send(signature, now, send_data or data, digest)
 
     def test_correct_signature(self):
         response = self.send_test_request(sender=self.mouse)

From e87236d78f0ef8984f85483b19f49e44b78cddd8 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 12:57:44 -0800
Subject: [PATCH 033/104] Remove unnecessary user create action

---
 bookwyrm/tests/activitypub/test_person.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/bookwyrm/tests/activitypub/test_person.py b/bookwyrm/tests/activitypub/test_person.py
index bec9e19b..3e0d74e0 100644
--- a/bookwyrm/tests/activitypub/test_person.py
+++ b/bookwyrm/tests/activitypub/test_person.py
@@ -2,14 +2,11 @@ import json
 import pathlib
 from django.test import TestCase
 
-from bookwyrm import activitypub, models
+from bookwyrm import activitypub
 
 
 class Person(TestCase):
     def setUp(self):
-        self.user = models.User.objects.create_user(
-            'rat', 'rat@rat.com', 'ratword',
-        )
         datafile = pathlib.Path(__file__).parent.joinpath(
             '../data/ap_user.json'
         )

From 3a751273012c9f77bf0614b7c18340e2f17edd3a Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 13:00:04 -0800
Subject: [PATCH 034/104] Removes half-baked field serializers

---
 bookwyrm/models/fields.py | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 960f3605..2b0b3b47 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -102,16 +102,6 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
             return None
         return value.remote_id
 
-    def field_from_activity(self, value):
-        if isinstance(value, dict) and value.get('id'):
-            # if the AP field is a serialized object (as in Add)
-            remote_id = value['id']
-        else:
-            # if the field is just a remote_id (as in every other case)
-            remote_id = value
-
-        return resolve_remote_id(remote_id)
-
 
 class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
     ''' activitypub-aware foreign key field '''
@@ -132,12 +122,6 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
             return '%s/followers' % value.instance.remote_id
         return [i.remote_id for i in value.all()]
 
-    def field_from_activity(self, valueactivity_data):
-        if self.link_only:
-            return None
-        values = super().field_from_activity(values)
-        return values# TODO
-
 
 class TagField(ManyToManyField):
     ''' special case of many to many that uses Tags '''

From de7e64932a166a7525a15b18222646b541c72b57 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 13:14:04 -0800
Subject: [PATCH 035/104] Fixes name of book field on status

---
 bookwyrm/models/status.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index 371a6345..6fa9f995 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -145,7 +145,8 @@ class GeneratedNote(Status):
 
 class Comment(Status):
     ''' like a review but without a rating and transient '''
-    book = fields.ForeignKey('Edition', on_delete=models.PROTECT)
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook')
 
     @property
     def pure_content(self):
@@ -160,7 +161,8 @@ class Comment(Status):
 class Quotation(Status):
     ''' like a review but without a rating and transient '''
     quote = fields.TextField()
-    book = fields.ForeignKey('Edition', on_delete=models.PROTECT)
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook')
 
     @property
     def pure_content(self):
@@ -179,7 +181,8 @@ class Quotation(Status):
 class Review(Status):
     ''' a book review '''
     name = fields.CharField(max_length=255, null=True)
-    book = fields.ForeignKey('Edition', on_delete=models.PROTECT)
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook')
     rating = fields.IntegerField(
         default=None,
         null=True,

From 9c6db1cc0e27f7206a17584cd86eb28cd43e420d Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 13:22:50 -0800
Subject: [PATCH 036/104] Updates connector tests

---
 .../connectors/test_abstract_connector.py     |  4 ++--
 .../connectors/test_bookwyrm_connector.py     | 21 ++++++-------------
 2 files changed, 8 insertions(+), 17 deletions(-)

diff --git a/bookwyrm/tests/connectors/test_abstract_connector.py b/bookwyrm/tests/connectors/test_abstract_connector.py
index c01b00db..f05645ab 100644
--- a/bookwyrm/tests/connectors/test_abstract_connector.py
+++ b/bookwyrm/tests/connectors/test_abstract_connector.py
@@ -3,7 +3,7 @@ from django.test import TestCase
 
 from bookwyrm import models
 from bookwyrm.connectors.abstract_connector import Mapping
-from bookwyrm.connectors.bookwyrm_connector import Connector
+from bookwyrm.connectors.openlibrary import Connector
 
 
 class AbstractConnector(TestCase):
@@ -12,7 +12,7 @@ class AbstractConnector(TestCase):
 
         models.Connector.objects.create(
             identifier='example.com',
-            connector_file='bookwyrm_connector',
+            connector_file='openlibrary',
             base_url='https://example.com',
             books_url='https:/example.com',
             covers_url='https://example.com',
diff --git a/bookwyrm/tests/connectors/test_bookwyrm_connector.py b/bookwyrm/tests/connectors/test_bookwyrm_connector.py
index c41b454c..8d866ca2 100644
--- a/bookwyrm/tests/connectors/test_bookwyrm_connector.py
+++ b/bookwyrm/tests/connectors/test_bookwyrm_connector.py
@@ -1,16 +1,17 @@
 ''' testing book data connectors '''
-from dateutil import parser
-from django.test import TestCase
 import json
 import pathlib
+from django.test import TestCase
 
 from bookwyrm import models
 from bookwyrm.connectors.bookwyrm_connector import Connector
-from bookwyrm.connectors.abstract_connector import SearchResult, get_date
+from bookwyrm.connectors.abstract_connector import SearchResult
 
 
 class BookWyrmConnector(TestCase):
+    ''' this connector doesn't do much, just search '''
     def setUp(self):
+        ''' create the connector '''
         models.Connector.objects.create(
             identifier='example.com',
             connector_file='bookwyrm_connector',
@@ -29,13 +30,9 @@ class BookWyrmConnector(TestCase):
         self.edition_data = json.loads(edition_file.read_bytes())
 
 
-    def test_is_work_data(self):
-        self.assertEqual(self.connector.is_work_data(self.work_data), True)
-        self.assertEqual(self.connector.is_work_data(self.edition_data), False)
-
-
     def test_format_search_result(self):
-        datafile = pathlib.Path(__file__).parent.joinpath('../data/fr_search.json')
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/fr_search.json')
         search_data = json.loads(datafile.read_bytes())
         results = self.connector.parse_search_data(search_data)
         self.assertIsInstance(results, list)
@@ -46,9 +43,3 @@ class BookWyrmConnector(TestCase):
         self.assertEqual(result.key, 'https://example.com/book/122')
         self.assertEqual(result.author, 'Susanna Clarke')
         self.assertEqual(result.year, 2017)
-
-
-    def test_get_date(self):
-        date = get_date(self.edition_data['published_date'])
-        expected = parser.parse("2020-09-15T00:00:00+00:00")
-        self.assertEqual(date, expected)

From b1640c5dc9718d5f55c6c772f75d571fd5f7da17 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 13:35:51 -0800
Subject: [PATCH 037/104] Sets mocks up for incoming tests

---
 bookwyrm/tests/incoming/test_favorite.py      | 19 +++++++++++--------
 bookwyrm/tests/incoming/test_follow.py        | 18 ++++++++++--------
 bookwyrm/tests/incoming/test_follow_accept.py | 19 +++++++++++--------
 bookwyrm/tests/incoming/test_update_user.py   | 15 +++++++++------
 4 files changed, 41 insertions(+), 30 deletions(-)

diff --git a/bookwyrm/tests/incoming/test_favorite.py b/bookwyrm/tests/incoming/test_favorite.py
index c528d38c..912657da 100644
--- a/bookwyrm/tests/incoming/test_favorite.py
+++ b/bookwyrm/tests/incoming/test_favorite.py
@@ -1,5 +1,6 @@
 import json
 import pathlib
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models, incoming
@@ -7,15 +8,17 @@ from bookwyrm import models, incoming
 
 class Favorite(TestCase):
     def setUp(self):
-        self.remote_user = models.User.objects.create_user(
-            'rat', 'rat@rat.com', 'ratword',
-            local=False,
-            remote_id='https://example.com/users/rat',
-            inbox='https://example.com/users/rat/inbox',
-            outbox='https://example.com/users/rat/outbox',
-        )
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            with patch('bookwyrm.models.user.get_remote_reviews.delay'):
+                self.remote_user = models.User.objects.create_user(
+                    'rat', 'rat@rat.com', 'ratword',
+                    local=False,
+                    remote_id='https://example.com/users/rat',
+                    inbox='https://example.com/users/rat/inbox',
+                    outbox='https://example.com/users/rat/outbox',
+                )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword',
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True,
             remote_id='http://local.com/user/mouse')
 
         self.status = models.Status.objects.create(
diff --git a/bookwyrm/tests/incoming/test_follow.py b/bookwyrm/tests/incoming/test_follow.py
index 50e47d88..8e0c7598 100644
--- a/bookwyrm/tests/incoming/test_follow.py
+++ b/bookwyrm/tests/incoming/test_follow.py
@@ -6,15 +6,17 @@ from bookwyrm import models, incoming
 
 class IncomingFollow(TestCase):
     def setUp(self):
-        self.remote_user = models.User.objects.create_user(
-            'rat', 'rat@rat.com', 'ratword',
-            local=False,
-            remote_id='https://example.com/users/rat',
-            inbox='https://example.com/users/rat/inbox',
-            outbox='https://example.com/users/rat/outbox',
-        )
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            with patch('bookwyrm.models.user.get_remote_reviews.delay'):
+                self.remote_user = models.User.objects.create_user(
+                    'rat', 'rat@rat.com', 'ratword',
+                    local=False,
+                    remote_id='https://example.com/users/rat',
+                    inbox='https://example.com/users/rat/inbox',
+                    outbox='https://example.com/users/rat/outbox',
+                )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword')
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
         self.local_user.remote_id = 'http://local.com/user/mouse'
         self.local_user.save()
 
diff --git a/bookwyrm/tests/incoming/test_follow_accept.py b/bookwyrm/tests/incoming/test_follow_accept.py
index ba88bb40..d6e048fb 100644
--- a/bookwyrm/tests/incoming/test_follow_accept.py
+++ b/bookwyrm/tests/incoming/test_follow_accept.py
@@ -1,3 +1,4 @@
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models, incoming
@@ -5,15 +6,17 @@ from bookwyrm import models, incoming
 
 class IncomingFollowAccept(TestCase):
     def setUp(self):
-        self.remote_user = models.User.objects.create_user(
-            'rat', 'rat@rat.com', 'ratword',
-            local=False,
-            remote_id='https://example.com/users/rat',
-            inbox='https://example.com/users/rat/inbox',
-            outbox='https://example.com/users/rat/outbox',
-        )
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            with patch('bookwyrm.models.user.get_remote_reviews.delay'):
+                self.remote_user = models.User.objects.create_user(
+                    'rat', 'rat@rat.com', 'ratword',
+                    local=False,
+                    remote_id='https://example.com/users/rat',
+                    inbox='https://example.com/users/rat/inbox',
+                    outbox='https://example.com/users/rat/outbox',
+                )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword')
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
         self.local_user.remote_id = 'http://local.com/user/mouse'
         self.local_user.save()
 
diff --git a/bookwyrm/tests/incoming/test_update_user.py b/bookwyrm/tests/incoming/test_update_user.py
index 7ac038eb..5e8b5f2e 100644
--- a/bookwyrm/tests/incoming/test_update_user.py
+++ b/bookwyrm/tests/incoming/test_update_user.py
@@ -1,6 +1,7 @@
 ''' when a remote user changes their profile '''
 import json
 import pathlib
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models, incoming
@@ -8,12 +9,14 @@ from bookwyrm import models, incoming
 
 class UpdateUser(TestCase):
     def setUp(self):
-        self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword',
-            remote_id='https://example.com/user/mouse',
-            local=False,
-            localname='mouse'
-        )
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            with patch('bookwyrm.models.user.get_remote_reviews.delay'):
+                self.user = models.User.objects.create_user(
+                    'mouse', 'mouse@mouse.com', 'mouseword',
+                    remote_id='https://example.com/user/mouse',
+                    local=False,
+                    localname='mouse'
+                )
 
         datafile = pathlib.Path(__file__).parent.joinpath(
             '../data/ap_user.json'

From 27c45c05848755864fea0b17511e051267347b4d Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 13:42:02 -0800
Subject: [PATCH 038/104] Catch json decode error in loading data

---
 bookwyrm/connectors/abstract_connector.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index 8c5d9658..a696232d 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -315,7 +315,11 @@ def get_data(url):
         raise ConnectorException()
     if not resp.ok:
         resp.raise_for_status()
-    data = resp.json()
+    try:
+        data = resp.json()
+    except ValueError:
+        raise ConnectorException()
+
     return data
 
 

From bbbfbe721e03ef2bb6d88ffae6a2876fbf9d8599 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 16:23:13 -0800
Subject: [PATCH 039/104] Removes update user test

---
 bookwyrm/tests/incoming/test_follow.py      | 21 -------------
 bookwyrm/tests/incoming/test_update_user.py | 34 ---------------------
 2 files changed, 55 deletions(-)
 delete mode 100644 bookwyrm/tests/incoming/test_update_user.py

diff --git a/bookwyrm/tests/incoming/test_follow.py b/bookwyrm/tests/incoming/test_follow.py
index 8e0c7598..799907da 100644
--- a/bookwyrm/tests/incoming/test_follow.py
+++ b/bookwyrm/tests/incoming/test_follow.py
@@ -75,24 +75,3 @@ class IncomingFollow(TestCase):
         # the follow relationship should not exist
         follow = models.UserFollows.objects.all()
         self.assertEqual(list(follow), [])
-
-
-    def test_nonexistent_user_follow(self):
-        activity = {
-            "@context": "https://www.w3.org/ns/activitystreams",
-            "id": "https://example.com/users/rat/follows/123",
-            "type": "Follow",
-            "actor": "https://example.com/users/rat",
-            "object": "http://local.com/user/nonexistent-user"
-        }
-
-        with patch('bookwyrm.broadcast.broadcast_task.delay') as _:
-            incoming.handle_follow(activity)
-
-        # do nothing
-        notifications = models.Notification.objects.all()
-        self.assertEqual(list(notifications), [])
-        requests = models.UserFollowRequest.objects.all()
-        self.assertEqual(list(requests), [])
-        follows = models.UserFollows.objects.all()
-        self.assertEqual(list(follows), [])
diff --git a/bookwyrm/tests/incoming/test_update_user.py b/bookwyrm/tests/incoming/test_update_user.py
deleted file mode 100644
index 5e8b5f2e..00000000
--- a/bookwyrm/tests/incoming/test_update_user.py
+++ /dev/null
@@ -1,34 +0,0 @@
-''' when a remote user changes their profile '''
-import json
-import pathlib
-from unittest.mock import patch
-from django.test import TestCase
-
-from bookwyrm import models, incoming
-
-
-class UpdateUser(TestCase):
-    def setUp(self):
-        with patch('bookwyrm.models.user.set_remote_server.delay'):
-            with patch('bookwyrm.models.user.get_remote_reviews.delay'):
-                self.user = models.User.objects.create_user(
-                    'mouse', 'mouse@mouse.com', 'mouseword',
-                    remote_id='https://example.com/user/mouse',
-                    local=False,
-                    localname='mouse'
-                )
-
-        datafile = pathlib.Path(__file__).parent.joinpath(
-            '../data/ap_user.json'
-        )
-        self.user_data = json.loads(datafile.read_bytes())
-
-    def test_handle_update_user(self):
-        self.assertIsNone(self.user.name)
-        self.assertEqual(self.user.localname, 'mouse')
-
-        incoming.handle_update_user({'object': self.user_data})
-        self.user = models.User.objects.get(id=self.user.id)
-
-        self.assertEqual(self.user.name, 'MOUSE?? MOUSE!!')
-        self.assertEqual(self.user.localname, 'mouse')

From 646ced80ce7963565677a83b8b32ba9f772a7795 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 17:18:23 -0800
Subject: [PATCH 040/104] Test fixes

---
 bookwyrm/tests/models/test_base_model.py      |  2 +-
 bookwyrm/tests/models/test_book_model.py      |  4 ++--
 bookwyrm/tests/models/test_import_model.py    |  2 +-
 .../tests/models/test_relationship_models.py  | 18 +++++++-------
 bookwyrm/tests/models/test_status_model.py    | 12 +---------
 bookwyrm/tests/models/test_user_model.py      |  3 +--
 bookwyrm/tests/outgoing/test_follow.py        | 24 ++++++++++---------
 7 files changed, 29 insertions(+), 36 deletions(-)

diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py
index 14f8b7fb..7e383a46 100644
--- a/bookwyrm/tests/models/test_base_model.py
+++ b/bookwyrm/tests/models/test_base_model.py
@@ -15,7 +15,7 @@ class BaseModel(TestCase):
 
     def test_remote_id_with_user(self):
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword')
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
         instance = BookWyrmModel()
         instance.user = user
         instance.id = 1
diff --git a/bookwyrm/tests/models/test_book_model.py b/bookwyrm/tests/models/test_book_model.py
index d01eea2e..12564992 100644
--- a/bookwyrm/tests/models/test_book_model.py
+++ b/bookwyrm/tests/models/test_book_model.py
@@ -24,7 +24,7 @@ class Book(TestCase):
     def test_remote_id(self):
         remote_id = 'https://%s/book/%d' % (settings.DOMAIN, self.work.id)
         self.assertEqual(self.work.get_remote_id(), remote_id)
-        self.assertEqual(self.work.remote_id, 'https://example.com/book/1')
+        self.assertEqual(self.work.remote_id, remote_id)
 
     def test_create_book(self):
         ''' you shouldn't be able to create Books (only editions and works) '''
@@ -59,7 +59,7 @@ class Book(TestCase):
 class Shelf(TestCase):
     def setUp(self):
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
         models.Shelf.objects.create(
             name='Test Shelf', identifier='test-shelf', user=user)
 
diff --git a/bookwyrm/tests/models/test_import_model.py b/bookwyrm/tests/models/test_import_model.py
index b56d9a44..c703d08a 100644
--- a/bookwyrm/tests/models/test_import_model.py
+++ b/bookwyrm/tests/models/test_import_model.py
@@ -52,7 +52,7 @@ class ImportJob(TestCase):
         unknown_read_data['Date Read'] = ''
 
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
         job = models.ImportJob.objects.create(user=user)
         models.ImportItem.objects.create(
             job=job, index=1, data=currently_reading_data)
diff --git a/bookwyrm/tests/models/test_relationship_models.py b/bookwyrm/tests/models/test_relationship_models.py
index 1e763f59..c5c619a0 100644
--- a/bookwyrm/tests/models/test_relationship_models.py
+++ b/bookwyrm/tests/models/test_relationship_models.py
@@ -1,4 +1,5 @@
 ''' testing models '''
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models
@@ -6,15 +7,16 @@ from bookwyrm import models
 
 class Relationship(TestCase):
     def setUp(self):
-        self.remote_user = models.User.objects.create_user(
-            'rat', 'rat@rat.com', 'ratword',
-            local=False,
-            remote_id='https://example.com/users/rat',
-            inbox='https://example.com/users/rat/inbox',
-            outbox='https://example.com/users/rat/outbox',
-        )
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            self.remote_user = models.User.objects.create_user(
+                'rat', 'rat@rat.com', 'ratword',
+                local=False,
+                remote_id='https://example.com/users/rat',
+                inbox='https://example.com/users/rat/inbox',
+                outbox='https://example.com/users/rat/outbox',
+            )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword')
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
         self.local_user.remote_id = 'http://local.com/user/mouse'
         self.local_user.save()
 
diff --git a/bookwyrm/tests/models/test_status_model.py b/bookwyrm/tests/models/test_status_model.py
index 719adfe5..7c625197 100644
--- a/bookwyrm/tests/models/test_status_model.py
+++ b/bookwyrm/tests/models/test_status_model.py
@@ -7,7 +7,7 @@ from bookwyrm import models, settings
 class Status(TestCase):
     def setUp(self):
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
         book = models.Edition.objects.create(title='Example Edition')
 
         models.Status.objects.create(user=user, content='Blah blah')
@@ -40,13 +40,3 @@ class Status(TestCase):
         expected_id = 'https://%s/user/mouse/review/%d' % \
                 (settings.DOMAIN, review.id)
         self.assertEqual(review.remote_id, expected_id)
-
-
-class Tag(TestCase):
-    def test_tag(self):
-        book = models.Edition.objects.create(title='Example Edition')
-        user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
-        tag = models.Tag.objects.create(user=user, book=book, name='t/est tag')
-        self.assertEqual(tag.identifier, 't%2Fest+tag')
-
diff --git a/bookwyrm/tests/models/test_user_model.py b/bookwyrm/tests/models/test_user_model.py
index c648bf6b..dabf760c 100644
--- a/bookwyrm/tests/models/test_user_model.py
+++ b/bookwyrm/tests/models/test_user_model.py
@@ -8,7 +8,7 @@ from bookwyrm.settings import DOMAIN
 class User(TestCase):
     def setUp(self):
         self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
 
     def test_computed_fields(self):
         ''' username instead of id here '''
@@ -53,7 +53,6 @@ class User(TestCase):
         self.assertEqual(activity['name'], self.user.name)
         self.assertEqual(activity['inbox'], self.user.inbox)
         self.assertEqual(activity['outbox'], self.user.outbox)
-        self.assertEqual(activity['followers'], self.user.ap_followers)
         self.assertEqual(activity['bookwyrmUser'], True)
         self.assertEqual(activity['discoverable'], True)
         self.assertEqual(activity['type'], 'Person')
diff --git a/bookwyrm/tests/outgoing/test_follow.py b/bookwyrm/tests/outgoing/test_follow.py
index 89e677ab..d27db876 100644
--- a/bookwyrm/tests/outgoing/test_follow.py
+++ b/bookwyrm/tests/outgoing/test_follow.py
@@ -2,17 +2,19 @@ from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models, outgoing
+from bookwyrm.settings import DOMAIN
 
 
 class Following(TestCase):
     def setUp(self):
-        self.remote_user = models.User.objects.create_user(
-            'rat', 'rat@rat.com', 'ratword',
-            local=False,
-            remote_id='https://example.com/users/rat',
-            inbox='https://example.com/users/rat/inbox',
-            outbox='https://example.com/users/rat/outbox',
-        )
+        with patch('bookwyrm.models.user.set_remote_server'):
+            self.remote_user = models.User.objects.create_user(
+                'rat', 'rat@rat.com', 'ratword',
+                local=False,
+                remote_id='https://example.com/users/rat',
+                inbox='https://example.com/users/rat/inbox',
+                outbox='https://example.com/users/rat/outbox',
+            )
         self.local_user = models.User.objects.create_user(
             'mouse', 'mouse@mouse.com', 'mouseword',
             local=True,
@@ -23,7 +25,7 @@ class Following(TestCase):
     def test_handle_follow(self):
         self.assertEqual(models.UserFollowRequest.objects.count(), 0)
 
-        with patch('bookwyrm.broadcast.broadcast_task.delay') as _:
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
             outgoing.handle_follow(self.local_user, self.remote_user)
 
         rel = models.UserFollowRequest.objects.get()
@@ -36,7 +38,7 @@ class Following(TestCase):
     def test_handle_unfollow(self):
         self.remote_user.followers.add(self.local_user)
         self.assertEqual(self.remote_user.followers.count(), 1)
-        with patch('bookwyrm.broadcast.broadcast_task.delay') as _:
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
             outgoing.handle_unfollow(self.local_user, self.remote_user)
 
         self.assertEqual(self.remote_user.followers.count(), 0)
@@ -49,7 +51,7 @@ class Following(TestCase):
         )
         rel_id = rel.id
 
-        with patch('bookwyrm.broadcast.broadcast_task.delay') as _:
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
             outgoing.handle_accept(rel)
         # request should be deleted
         self.assertEqual(
@@ -66,7 +68,7 @@ class Following(TestCase):
         )
         rel_id = rel.id
 
-        with patch('bookwyrm.broadcast.broadcast_task.delay') as _:
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
             outgoing.handle_reject(rel)
         # request should be deleted
         self.assertEqual(

From d92fb5333307bcc373438c842514dcf367035f24 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Thu, 3 Dec 2020 17:23:08 -0800
Subject: [PATCH 041/104] Handle absent remote ids

it oughtent get to this state, but...
---
 bookwyrm/models/base_model.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 520864c2..91724e62 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -87,6 +87,8 @@ class ActivitypubMixin:
                 related_field = getattr(self, field_name)
                 activity[field_name] = unfurl_related_field(related_field)
 
+        if not activity.get('id'):
+            activity['id'] = self.get_remote_id()
         return self.activity_serializer(**activity).serialize()
 
 

From ae8d0e197443a78a30e67d86370f08dcf937de8e Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 08:07:47 -0800
Subject: [PATCH 042/104] Adds sort order for outbox

---
 bookwyrm/models/user.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index ac729fd3..6395987b 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -102,7 +102,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         queryset = Status.objects.filter(
             user=self,
             deleted=False,
-        ).select_subclasses()
+        ).select_subclasses().order_by('-published_date')
         return self.to_ordered_collection(queryset, \
                 remote_id=self.outbox, **kwargs)
 

From de6147ecfa6b5da176bb6eeb20aa6f808583b4ad Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 08:29:54 -0800
Subject: [PATCH 043/104] Ignore many to many activitypub serialization of
 non-lists

---
 bookwyrm/activitypub/base_activity.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 786a2aab..ebc026b9 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -151,8 +151,9 @@ class ActivityObject:
 
         # add many to many fields
         for (model_key, values) in many_to_many_fields.items():
-            # mention books, mention users
-            if values == MISSING:
+            # mention books, mention users, followers
+            if values == MISSING or not isinstance(values, list):
+                # user followers is a link to an orderedcollection, skip it
                 continue
             model_field = getattr(instance, model_key)
             model = model_field.model

From 1ae3830ae48c135b80155b409dc71293be1a2df9 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 08:42:34 -0800
Subject: [PATCH 044/104] Removes test state of signatures

---
 bookwyrm/signatures.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py
index a3e1fccc..ff281664 100644
--- a/bookwyrm/signatures.py
+++ b/bookwyrm/signatures.py
@@ -89,7 +89,7 @@ class Signature:
 
     def verify(self, public_key, request):
         ''' verify rsa signature '''
-        if False:#http_date_age(request.headers['date']) > MAX_SIGNATURE_AGE:
+        if http_date_age(request.headers['date']) > MAX_SIGNATURE_AGE:
             raise ValueError(
                 "Request too old: %s" % (request.headers['date'],))
         public_key = RSA.import_key(public_key)

From 9989641f4c6f947b7aca8e5642f72ba1d19af5df Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 09:46:40 -0800
Subject: [PATCH 045/104] fixes bug in update user public key

---
 bookwyrm/incoming.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index 9a7c6e63..29c0be98 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -104,7 +104,7 @@ def has_valid_signature(request, activity):
         except ValueError:
             old_key = remote_user.key_pair.public_key
             activitypub.resolve_remote_id(
-                models.User, remote_user, refresh=True
+                models.User, remote_user.remote_id, refresh=True
             )
             if remote_user.key_pair.public_key == old_key:
                 raise # Key unchanged.

From 2e4aff90a36a021fa47b7d87ad7ecaeca3b4093e Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 11:22:08 -0800
Subject: [PATCH 046/104] Fixes signing Create activities

and some tests for the base_model
---
 bookwyrm/models/base_model.py            |  2 +-
 bookwyrm/models/status.py                |  5 ---
 bookwyrm/tests/models/test_base_model.py | 54 ++++++++++++++++++++++--
 3 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 91724e62..121b5f75 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -96,7 +96,7 @@ class ActivitypubMixin:
         ''' returns the object wrapped in a Create activity '''
         activity_object = self.to_activity()
 
-        signer = pkcs1_15.new(RSA.import_key(user.private_key))
+        signer = pkcs1_15.new(RSA.import_key(user.key_pair.private_key))
         content = activity_object['content']
         signed_message = signer.sign(SHA256.new(content.encode('utf8')))
         create_id = self.remote_id + '/activity'
diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index 6fa9f995..a2b873bb 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -44,11 +44,6 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
     )
     objects = InheritanceManager()
 
-    @property
-    def ap_replies(self):
-        ''' structured replies block '''
-        return self.to_replies()
-
     activity_serializer = activitypub.Note
     serialize_reverse_fields = ['attachments']
 
diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py
index 7e383a46..9568ae6a 100644
--- a/bookwyrm/tests/models/test_base_model.py
+++ b/bookwyrm/tests/models/test_base_model.py
@@ -1,14 +1,15 @@
 ''' testing models '''
+from collections import namedtuple
 from django.test import TestCase
 
 from bookwyrm import models
-from bookwyrm.models.base_model import BookWyrmModel
+from bookwyrm.models import base_model
+from bookwyrm.models.base_model import ActivitypubMixin
 from bookwyrm.settings import DOMAIN
 
-
 class BaseModel(TestCase):
     def test_remote_id(self):
-        instance = BookWyrmModel()
+        instance = base_model.BookWyrmModel()
         instance.id = 1
         expected = instance.get_remote_id()
         self.assertEqual(expected, 'https://%s/bookwyrmmodel/1' % DOMAIN)
@@ -16,10 +17,55 @@ class BaseModel(TestCase):
     def test_remote_id_with_user(self):
         user = models.User.objects.create_user(
             'mouse', 'mouse@mouse.com', 'mouseword', local=True)
-        instance = BookWyrmModel()
+        instance = base_model.BookWyrmModel()
         instance.user = user
         instance.id = 1
         expected = instance.get_remote_id()
         self.assertEqual(
             expected,
             'https://%s/user/mouse/bookwyrmmodel/1' % DOMAIN)
+
+    def test_execute_after_save(self):
+        ''' this function sets remote ids after creation '''
+        # using Work because it BookWrymModel is abstract and this requires save
+        # Work is a relatively not-fancy model.
+        instance = models.Work.objects.create(title='work title')
+        instance.remote_id = None
+        base_model.execute_after_save(None, instance, True)
+        self.assertEqual(
+            instance.remote_id,
+            'https://%s/book/%d' % (DOMAIN, instance.id)
+        )
+
+        # shouldn't set remote_id if it's not created
+        instance.remote_id = None
+        base_model.execute_after_save(None, instance, False)
+        self.assertIsNone(instance.remote_id)
+
+    def test_to_create_activity(self):
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+
+        object_activity = {
+            'to': 'to field', 'cc': 'cc field',
+            'content': 'hi',
+            'published': '2020-12-04T17:52:22.623807+00:00',
+        }
+        MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
+        mock_self = MockSelf(
+            'https://example.com/status/1',
+            lambda *args: object_activity
+        )
+        activity = ActivitypubMixin.to_create_activity(mock_self, user)
+        self.assertEqual(
+            activity['id'],
+            'https://example.com/status/1/activity'
+        )
+        self.assertEqual(activity['actor'], user.remote_id)
+        self.assertEqual(activity['to'], 'to field')
+        self.assertEqual(activity['cc'], 'cc field')
+        self.assertEqual(activity['object'], object_activity)
+        self.assertEqual(
+            activity['signature'].creator,
+            '%s#main-key' % user.remote_id
+        )

From 39307ce1cdeda140637f9c4b69dfa059053043cf Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 11:46:16 -0800
Subject: [PATCH 047/104] Fixes remote_id on Update activities

---
 bookwyrm/models/base_model.py            |  2 +-
 bookwyrm/tests/models/test_base_model.py | 49 ++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 121b5f75..94fbc015 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -130,7 +130,7 @@ class ActivitypubMixin:
 
     def to_update_activity(self, user):
         ''' wrapper for Updates to an activity '''
-        activity_id = '%s#update/%s' % (user.remote_id, uuid4())
+        activity_id = '%s#update/%s' % (self.remote_id, uuid4())
         return activitypub.Update(
             id=activity_id,
             actor=user.remote_id,
diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py
index 9568ae6a..40306d91 100644
--- a/bookwyrm/tests/models/test_base_model.py
+++ b/bookwyrm/tests/models/test_base_model.py
@@ -1,5 +1,6 @@
 ''' testing models '''
 from collections import namedtuple
+import re
 from django.test import TestCase
 
 from bookwyrm import models
@@ -62,6 +63,7 @@ class BaseModel(TestCase):
             'https://example.com/status/1/activity'
         )
         self.assertEqual(activity['actor'], user.remote_id)
+        self.assertEqual(activity['type'], 'Create')
         self.assertEqual(activity['to'], 'to field')
         self.assertEqual(activity['cc'], 'cc field')
         self.assertEqual(activity['object'], object_activity)
@@ -69,3 +71,50 @@ class BaseModel(TestCase):
             activity['signature'].creator,
             '%s#main-key' % user.remote_id
         )
+
+    def test_to_delete_activity(self):
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+
+        MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
+        mock_self = MockSelf(
+            'https://example.com/status/1',
+            lambda *args: {}
+        )
+        activity = ActivitypubMixin.to_delete_activity(mock_self, user)
+        self.assertEqual(
+            activity['id'],
+            'https://example.com/status/1/activity'
+        )
+        self.assertEqual(activity['actor'], user.remote_id)
+        self.assertEqual(activity['type'], 'Delete')
+        self.assertEqual(
+            activity['to'],
+            ['%s/followers' % user.remote_id])
+        self.assertEqual(
+            activity['cc'],
+            ['https://www.w3.org/ns/activitystreams#Public'])
+
+    def test_to_update_activity(self):
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+
+        MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
+        mock_self = MockSelf(
+            'https://example.com/status/1',
+            lambda *args: {}
+        )
+        activity = ActivitypubMixin.to_update_activity(mock_self, user)
+        print(activity['id'])
+        self.assertIsNotNone(
+            re.match(
+                r'^https:\/\/example\.com\/status\/1#update\/.*',
+                activity['id']
+            )
+        )
+        self.assertEqual(activity['actor'], user.remote_id)
+        self.assertEqual(activity['type'], 'Update')
+        self.assertEqual(
+            activity['to'],
+            ['https://www.w3.org/ns/activitystreams#Public'])
+        self.assertEqual(activity['object'], {})

From 800ddf2a6b9144508a69807be32790fe7abc9e9d Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 11:52:01 -0800
Subject: [PATCH 048/104] fixes inconsistency in to_undo activity helper

---
 bookwyrm/models/base_model.py            |  4 ++--
 bookwyrm/tests/models/test_base_model.py | 19 ++++++++++++++++++-
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 94fbc015..d3c9471f 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -142,10 +142,10 @@ class ActivitypubMixin:
     def to_undo_activity(self, user):
         ''' undo an action '''
         return activitypub.Undo(
-            id='%s#undo' % user.remote_id,
+            id='%s#undo' % self.remote_id,
             actor=user.remote_id,
             object=self.to_activity()
-        )
+        ).serialize()
 
 
 class OrderedCollectionPageMixin(ActivitypubMixin):
diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py
index 40306d91..b187284a 100644
--- a/bookwyrm/tests/models/test_base_model.py
+++ b/bookwyrm/tests/models/test_base_model.py
@@ -105,7 +105,6 @@ class BaseModel(TestCase):
             lambda *args: {}
         )
         activity = ActivitypubMixin.to_update_activity(mock_self, user)
-        print(activity['id'])
         self.assertIsNotNone(
             re.match(
                 r'^https:\/\/example\.com\/status\/1#update\/.*',
@@ -118,3 +117,21 @@ class BaseModel(TestCase):
             activity['to'],
             ['https://www.w3.org/ns/activitystreams#Public'])
         self.assertEqual(activity['object'], {})
+
+    def test_to_undo_activity(self):
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+
+        MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
+        mock_self = MockSelf(
+            'https://example.com/status/1',
+            lambda *args: {}
+        )
+        activity = ActivitypubMixin.to_undo_activity(mock_self, user)
+        self.assertEqual(
+            activity['id'],
+            'https://example.com/status/1#undo'
+        )
+        self.assertEqual(activity['actor'], user.remote_id)
+        self.assertEqual(activity['type'], 'Undo')
+        self.assertEqual(activity['object'], {})

From 9c9da35d9a20baf7b3f9db50547a0aa3121c8e99 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 15:14:26 -0800
Subject: [PATCH 049/104] Tests base_model to_activity

---
 bookwyrm/tests/models/test_base_model.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py
index b187284a..a807c47f 100644
--- a/bookwyrm/tests/models/test_base_model.py
+++ b/bookwyrm/tests/models/test_base_model.py
@@ -1,8 +1,10 @@
 ''' testing models '''
 from collections import namedtuple
+from dataclasses import dataclass
 import re
 from django.test import TestCase
 
+from bookwyrm.activitypub.base_activity import ActivityObject
 from bookwyrm import models
 from bookwyrm.models import base_model
 from bookwyrm.models.base_model import ActivitypubMixin
@@ -135,3 +137,21 @@ class BaseModel(TestCase):
         self.assertEqual(activity['actor'], user.remote_id)
         self.assertEqual(activity['type'], 'Undo')
         self.assertEqual(activity['object'], {})
+
+
+    def test_to_activity(self):
+        @dataclass(init=False)
+        class TestActivity(ActivityObject):
+            type: str = 'Test'
+
+        class TestModel(ActivitypubMixin, base_model.BookWyrmModel):
+            pass
+
+        instance = TestModel()
+        instance.remote_id = 'https://www.example.com/test'
+        instance.activity_serializer = TestActivity
+
+        activity = instance.to_activity()
+        self.assertIsInstance(activity, dict)
+        self.assertEqual(activity['id'], 'https://www.example.com/test')
+        self.assertEqual(activity['type'], 'Test')

From 142a39cf5586807a6794e02e85b3325cad69a59b Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 16:26:07 -0800
Subject: [PATCH 050/104] Updates remote user when refreshing key

---
 bookwyrm/activitypub/base_activity.py | 11 ++++++-----
 bookwyrm/incoming.py                  |  2 +-
 bookwyrm/tests/test_signing.py        | 19 ++++++-------------
 3 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index ebc026b9..0a34d067 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -106,7 +106,7 @@ class ActivityObject:
                 if isinstance(formatted_value, dict) and \
                         formatted_value.get('id'):
                     # if the AP field is a serialized object (as in Add)
-                    # or PublicKey
+                    # or KeyPair (even though it's OneToOne)
                     related_model = field.related_model
                     related_activity = related_model.activity_serializer
                     mapped_fields[field.name] = related_activity(
@@ -131,6 +131,7 @@ class ActivityObject:
                     formatted_value = None
                 mapped_fields[field.name] = formatted_value
 
+
         if instance:
             # updating an existing model instance
             for k, v in mapped_fields.items():
@@ -178,13 +179,13 @@ class ActivityObject:
             if values == MISSING:
                 continue
             model_field = getattr(instance, model_key)
-            model = model_field.model
+            related_model = model_field.model
             for item in values:
                 if isinstance(item, str):
-                    item = resolve_remote_id(model, item)
+                    item = resolve_remote_id(related_model, item)
                 else:
-                    item = model.activity_serializer(**item)
-                    item = item.to_model(model)
+                    item = related_model.activity_serializer(**item)
+                    item = item.to_model(related_model)
                 field_name = instance.__class__.__name__.lower()
                 setattr(item, field_name, instance)
                 item.save()
diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index 29c0be98..bbbebf0f 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -103,7 +103,7 @@ def has_valid_signature(request, activity):
             signature.verify(remote_user.key_pair.public_key, request)
         except ValueError:
             old_key = remote_user.key_pair.public_key
-            activitypub.resolve_remote_id(
+            remote_user = activitypub.resolve_remote_id(
                 models.User, remote_user.remote_id, refresh=True
             )
             if remote_user.key_pair.public_key == old_key:
diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py
index f2997ff9..bf252764 100644
--- a/bookwyrm/tests/test_signing.py
+++ b/bookwyrm/tests/test_signing.py
@@ -111,7 +111,7 @@ class Signature(TestCase):
             status=200
         )
 
-        with patch('bookwyrm.models.user.get_remote_reviews.delay') as _:
+        with patch('bookwyrm.models.user.get_remote_reviews.delay'):
             response = self.send_test_request(sender=self.fake_remote)
             self.assertEqual(response.status_code, 200)
 
@@ -131,17 +131,10 @@ class Signature(TestCase):
             responses.GET,
             'https://localhost/.well-known/nodeinfo',
             status=404)
-        responses.add(
-            responses.GET,
-            'https://example.com/user/mouse/outbox?page=true',
-            json={'orderedItems': []},
-            status=200
-        )
 
         # Second and subsequent fetches get a different key:
         key_pair = KeyPair(*create_key_pair())
-        new_sender = Sender(
-            self.fake_remote.remote_id, key_pair)
+        new_sender = Sender(self.fake_remote.remote_id, key_pair)
         data['publicKey']['publicKeyPem'] = key_pair.public_key
         responses.add(
             responses.GET,
@@ -149,7 +142,7 @@ class Signature(TestCase):
             json=data,
             status=200)
 
-        with patch('bookwyrm.models.user.get_remote_reviews.delay') as _:
+        with patch('bookwyrm.models.user.get_remote_reviews.delay'):
             # Key correct:
             response = self.send_test_request(sender=self.fake_remote)
             self.assertEqual(response.status_code, 200)
@@ -181,7 +174,7 @@ class Signature(TestCase):
     @pytest.mark.integration
     def test_changed_data(self):
         '''Message data must match the digest header.'''
-        with patch('bookwyrm.activitypub.resolve_remote_id') as _:
+        with patch('bookwyrm.activitypub.resolve_remote_id'):
             response = self.send_test_request(
                 self.mouse,
                 send_data=get_follow_data(self.mouse, self.cat))
@@ -189,7 +182,7 @@ class Signature(TestCase):
 
     @pytest.mark.integration
     def test_invalid_digest(self):
-        with patch('bookwyrm.activitypub.resolve_remote_id') as _:
+        with patch('bookwyrm.activitypub.resolve_remote_id'):
             response = self.send_test_request(
                 self.mouse,
                 digest='SHA-256=AAAAAAAAAAAAAAAAAA')
@@ -198,7 +191,7 @@ class Signature(TestCase):
     @pytest.mark.integration
     def test_old_message(self):
         '''Old messages should be rejected to prevent replay attacks.'''
-        with patch('bookwyrm.activitypub.resolve_remote_id') as _:
+        with patch('bookwyrm.activitypub.resolve_remote_id'):
             response = self.send_test_request(
                 self.mouse,
                 date=http_date(time.time() - 301)

From aa6e312cfb03d4ef0685dc31a81df5640c93aa36 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 17:38:39 -0800
Subject: [PATCH 051/104] Starts adding tests for custom model fields

---
 bookwyrm/tests/models/test_fields.py | 77 ++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 bookwyrm/tests/models/test_fields.py

diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
new file mode 100644
index 00000000..1e98c44d
--- /dev/null
+++ b/bookwyrm/tests/models/test_fields.py
@@ -0,0 +1,77 @@
+''' testing models '''
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.test import TestCase
+
+from bookwyrm.models import fields
+
+class ActivitypubFields(TestCase):
+    def test_validate_remote_id(self):
+        self.assertIsNone(fields.validate_remote_id(
+            'http://www.example.com'
+        ))
+        self.assertIsNone(fields.validate_remote_id(
+            'https://www.example.com'
+        ))
+        self.assertIsNone(fields.validate_remote_id(
+            'http://example.com/dlfjg-23/x'
+        ))
+        self.assertRaises(
+            ValidationError, fields.validate_remote_id,
+            'http:/example.com/dlfjg-23/x'
+        )
+        self.assertRaises(
+            ValidationError, fields.validate_remote_id,
+            'www.example.com/dlfjg-23/x'
+        )
+        self.assertRaises(
+            ValidationError, fields.validate_remote_id,
+            'http://www.example.com/dlfjg 23/x'
+        )
+
+    def test_activitypub_field_mixin(self):
+        instance = fields.ActivitypubFieldMixin()
+        self.assertEqual(instance.field_to_activity('fish'), 'fish')
+        self.assertEqual(instance.field_from_activity('fish'), 'fish')
+
+        instance = fields.ActivitypubFieldMixin(
+            activitypub_wrapper='endpoints', activitypub_field='outbox'
+        )
+        self.assertEqual(
+            instance.field_to_activity('fish'),
+            {'outbox': 'fish'}
+        )
+        self.assertEqual(
+            instance.field_from_activity({'outbox': 'fish'}),
+            'fish'
+        )
+        self.assertEqual(instance.get_activitypub_field(), 'endpoints')
+
+        instance = fields.ActivitypubFieldMixin()
+        instance.name = 'snake_case_name'
+        self.assertEqual(instance.get_activitypub_field(), 'snakeCaseName')
+
+    def test_remote_id_field(self):
+        instance = fields.RemoteIdField()
+        self.assertEqual(instance.max_length, 255)
+
+        with self.assertRaises(ValidationError):
+            instance.run_validators('http://www.example.com/dlfjg 23/x')
+
+    def test_username_field(self):
+        instance = fields.UsernameField()
+        self.assertEqual(instance.activitypub_field, 'preferredUsername')
+        self.assertEqual(instance.max_length, 150)
+        self.assertEqual(instance.unique, True)
+        with self.assertRaises(ValidationError):
+            instance.run_validators('one two')
+            instance.run_validators('a*&')
+            instance.run_validators('trailingwhite ')
+        self.assertIsNone(instance.run_validators('aksdhf'))
+
+        self.assertEqual(instance.field_to_activity('test@example.com'), 'test')
+
+    def test_foreign_key(self):
+        instance = fields.ForeignKey('User', on_delete=models.CASCADE)
+        item = namedtuple('Serializable', ('to_activity'))(lambda: {'a': 'b'})
+        self.assertEqual(instance.field_to_activity(item), {'a': 'b'})

From f116ce378d990849440948c148cc8874dec544b1 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 17:42:01 -0800
Subject: [PATCH 052/104] Fixes foreign key test

---
 bookwyrm/tests/models/test_fields.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index 1e98c44d..5e29a527 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -1,4 +1,6 @@
 ''' testing models '''
+from collections import namedtuple
+
 from django.core.exceptions import ValidationError
 from django.db import models
 from django.test import TestCase
@@ -73,5 +75,6 @@ class ActivitypubFields(TestCase):
 
     def test_foreign_key(self):
         instance = fields.ForeignKey('User', on_delete=models.CASCADE)
-        item = namedtuple('Serializable', ('to_activity'))(lambda: {'a': 'b'})
-        self.assertEqual(instance.field_to_activity(item), {'a': 'b'})
+        Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
+        item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c')
+        self.assertEqual(instance.field_to_activity(item), 'https://e.b/c')

From 8a900689d3288bb1bbdf27bb8de21b8e6ad1f37c Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 4 Dec 2020 17:57:14 -0800
Subject: [PATCH 053/104] Generalizes link format in many to many field

---
 bookwyrm/models/fields.py            |  2 +-
 bookwyrm/tests/models/test_fields.py | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 2b0b3b47..b5e321ee 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -119,7 +119,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
 
     def field_to_activity(self, value):
         if self.link_only:
-            return '%s/followers' % value.instance.remote_id
+            return '%s/%s' % (value.instance.remote_id, self.name)
         return [i.remote_id for i in value.all()]
 
 
diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index 5e29a527..e7393096 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -78,3 +78,28 @@ class ActivitypubFields(TestCase):
         Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
         item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c')
         self.assertEqual(instance.field_to_activity(item), 'https://e.b/c')
+
+    def test_one_to_one_field(self):
+        instance = fields.OneToOneField('User', on_delete=models.CASCADE)
+        Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
+        item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c')
+        self.assertEqual(instance.field_to_activity(item), {'a': 'b'})
+
+    def test_many_to_many_field(self):
+        instance = fields.ManyToManyField('User')
+
+        Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
+        Queryset = namedtuple('Queryset', ('all', 'instance'))
+        item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c')
+        another_item = Serializable(lambda: {}, 'example.com')
+
+        items = Queryset(lambda: [item], another_item)
+
+        self.assertEqual(instance.field_to_activity(items), ['https://e.b/c'])
+
+        instance = fields.ManyToManyField('User', link_only=True)
+        instance.name = 'snake_case'
+        self.assertEqual(
+            instance.field_to_activity(items),
+            'example.com/snake_case'
+        )

From 05cde33a0c3a962f04ad7d201b73a3d68b6b06d1 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 5 Dec 2020 14:40:23 -0800
Subject: [PATCH 054/104] Adds tests for remaining nontrivial model fields

---
 bookwyrm/tests/models/test_fields.py | 86 +++++++++++++++++++++++++++-
 1 file changed, 85 insertions(+), 1 deletion(-)

diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index e7393096..2b2d3939 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -1,11 +1,20 @@
 ''' testing models '''
+from io import BytesIO
 from collections import namedtuple
+import pathlib
+import re
+
+from PIL import Image
+import responses
 
 from django.core.exceptions import ValidationError
+from django.core.files.base import ContentFile
 from django.db import models
 from django.test import TestCase
+from django.utils import timezone
 
-from bookwyrm.models import fields
+from bookwyrm.models import fields, User
+from bookwyrm.settings import DOMAIN
 
 class ActivitypubFields(TestCase):
     def test_validate_remote_id(self):
@@ -103,3 +112,78 @@ class ActivitypubFields(TestCase):
             instance.field_to_activity(items),
             'example.com/snake_case'
         )
+
+    def test_tag_field(self):
+        instance = fields.TagField('User')
+
+        Serializable = namedtuple(
+            'Serializable',
+            ('to_activity', 'remote_id', 'name_field', 'name')
+        )
+        Queryset = namedtuple('Queryset', ('all', 'instance'))
+        item = Serializable(
+            lambda: {'a': 'b'}, 'https://e.b/c', 'name', 'Name')
+        another_item = Serializable(
+            lambda: {}, 'example.com', '', '')
+        items = Queryset(lambda: [item], another_item)
+
+        result = instance.field_to_activity(items)
+        self.assertIsInstance(result, list)
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0].href, 'https://e.b/c')
+        self.assertEqual(result[0].name, 'Name')
+        self.assertEqual(result[0].type, 'Serializable')
+
+
+    @responses.activate
+    def test_image_field(self):
+        user = User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        image_file = pathlib.Path(__file__).parent.joinpath(
+            '../../static/images/default_avi.jpg')
+        image = Image.open(image_file)
+        output = BytesIO()
+        image.save(output, format=image.format)
+        user.avatar.save(
+            'test.jpg',
+            ContentFile(output.getvalue())
+        )
+
+        output = fields.image_serializer(user.avatar)
+        self.assertIsNotNone(
+            re.match(
+                r'https://%s/images/avatars/test_[A-z0-9]+\.jpg' % DOMAIN,
+                output.url,
+            )
+        )
+        self.assertEqual(output.type, 'Image')
+
+        instance = fields.ImageField()
+
+        self.assertEqual(instance.field_to_activity(user.avatar), output)
+
+        responses.add(
+            responses.GET,
+            'http://www.example.com/image.jpg',
+            body=user.avatar.file.read(),
+            status=200)
+        loaded_image = instance.field_from_activity(
+            'http://www.example.com/image.jpg')
+        self.assertIsInstance(loaded_image, list)
+        self.assertIsNotNone(re.match(r'.*\.jpg', loaded_image[0]))
+        self.assertIsInstance(loaded_image[1], ContentFile)
+
+
+    def test_datetime_field(self):
+        instance = fields.DateTimeField()
+        now = timezone.now()
+        self.assertEqual(instance.field_to_activity(now), now.isoformat())
+        self.assertEqual(
+            instance.field_from_activity(now.isoformat()), now
+        )
+        self.assertEqual(instance.field_from_activity('bip'), None)
+
+
+    def test_array_field(self):
+        instance = fields.ArrayField(fields.IntegerField)
+        self.assertEqual(instance.field_to_activity([0, 1]), ['0', '1'])

From ef1558628f7041be2c6f293bba787a898f6176df Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 5 Dec 2020 14:48:47 -0800
Subject: [PATCH 055/104] Fixes transient failure in image field test

---
 bookwyrm/tests/models/test_fields.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index 2b2d3939..a85f3234 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -152,7 +152,7 @@ class ActivitypubFields(TestCase):
         output = fields.image_serializer(user.avatar)
         self.assertIsNotNone(
             re.match(
-                r'https://%s/images/avatars/test_[A-z0-9]+\.jpg' % DOMAIN,
+                r'https://%s/images/avatars/test_.*\.jpg' % DOMAIN,
                 output.url,
             )
         )

From 8500a7cfe12b432dd97d4ae7df2e153089d2c3a1 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 5 Dec 2020 15:28:41 -0800
Subject: [PATCH 056/104] Unit test fails in CI but not local

---
 bookwyrm/tests/models/test_fields.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index a85f3234..b157aab3 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -152,7 +152,7 @@ class ActivitypubFields(TestCase):
         output = fields.image_serializer(user.avatar)
         self.assertIsNotNone(
             re.match(
-                r'https://%s/images/avatars/test_.*\.jpg' % DOMAIN,
+                r'.*\.jpg',
                 output.url,
             )
         )
@@ -170,7 +170,6 @@ class ActivitypubFields(TestCase):
         loaded_image = instance.field_from_activity(
             'http://www.example.com/image.jpg')
         self.assertIsInstance(loaded_image, list)
-        self.assertIsNotNone(re.match(r'.*\.jpg', loaded_image[0]))
         self.assertIsInstance(loaded_image[1], ContentFile)
 
 

From 7a90aa8f6c95e14d91e685d9567ddd581f381ac0 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 5 Dec 2020 21:33:48 -0800
Subject: [PATCH 057/104] Start moving serializing from to_model to fields

---
 bookwyrm/activitypub/base_activity.py | 63 +++++----------------------
 bookwyrm/models/fields.py             | 43 +++++++++++++++++-
 bookwyrm/tests/models/test_fields.py  | 17 +++++++-
 3 files changed, 69 insertions(+), 54 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 0a34d067..4de4dc6b 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -92,44 +92,24 @@ class ActivityObject:
             if value is None:
                 continue
 
-            # value is None if there's a default that isn't supplied
-            # in the activity but is supplied in the formatter
-            value = getattr(self, activitypub_field)
             model_field = getattr(model, field.name)
 
-            formatted_value = field.field_from_activity(value)
             if isinstance(model_field, ForwardManyToOneDescriptor):
-                if not formatted_value:
-                    continue
-                # foreign key remote id reolver (work on Edition, for example)
-                fk_model = model_field.field.related_model
-                if isinstance(formatted_value, dict) and \
-                        formatted_value.get('id'):
-                    # if the AP field is a serialized object (as in Add)
-                    # or KeyPair (even though it's OneToOne)
-                    related_model = field.related_model
-                    related_activity = related_model.activity_serializer
-                    mapped_fields[field.name] = related_activity(
-                        **formatted_value
-                    ).to_model(related_model)
-                else:
-                    # if the field is just a remote_id (as in every other case)
-                    remote_id = formatted_value
-                    reference = resolve_remote_id(fk_model, remote_id)
-                    mapped_fields[field.name] = reference
-            elif isinstance(model_field, ManyToManyDescriptor):
+                mapped_fields[field.name] = value
+            if isinstance(model_field, ManyToManyDescriptor):
                 # status mentions book/users
-                many_to_many_fields[field.name] = formatted_value
+                many_to_many_fields[field.name] = value
             elif isinstance(model_field, ReverseManyToOneDescriptor):
                 # attachments on Status, for example
-                one_to_many_fields[field.name] = formatted_value
+                one_to_many_fields[field.name] = value
             elif isinstance(model_field, ImageFileDescriptor):
                 # image fields need custom handling
-                image_fields[field.name] = formatted_value
+                print(model_field, field.name, value)
+                image_fields[field.name] = value
             else:
-                if formatted_value == MISSING:
-                    formatted_value = None
-                mapped_fields[field.name] = formatted_value
+                if value == MISSING:
+                    value = None
+                mapped_fields[field.name] = value
 
 
         if instance:
@@ -146,33 +126,14 @@ class ActivityObject:
 
         # add images
         for (model_key, value) in image_fields.items():
-            if not formatted_value:
+            if not value:
                 continue
-            getattr(instance, model_key).save(*formatted_value, save=True)
+            getattr(instance, model_key).save(*value, save=True)
 
         # add many to many fields
         for (model_key, values) in many_to_many_fields.items():
             # mention books, mention users, followers
-            if values == MISSING or not isinstance(values, list):
-                # user followers is a link to an orderedcollection, skip it
-                continue
-            model_field = getattr(instance, model_key)
-            model = model_field.model
-            items = []
-            for link in values:
-                if isinstance(link, dict):
-                    # check that the Type matches the model (Status
-                    # tags contain both user mentions and book tags)
-                    if not model.activity_serializer.type == \
-                            link.get('type'):
-                        continue
-                    remote_id = link.get('href')
-                else:
-                    remote_id = link
-                items.append(
-                    resolve_remote_id(model, remote_id)
-                )
-            getattr(instance, model_key).set(items)
+            getattr(instance, model_key).set(values)
 
         # add one to many fields
         for (model_key, values) in one_to_many_fields.items():
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index b5e321ee..e1636f84 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -57,6 +57,27 @@ class ActivitypubFieldMixin:
         return components[0] + ''.join(x.title() for x in components[1:])
 
 
+class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
+    ''' default (de)serialization for foreign key and one to one '''
+    def field_from_activity(self, value):
+        if not value:
+            return None
+
+        related_model = self.related_model
+        if isinstance(value, dict) and value.get('id'):
+            # this is an activitypub object, which we can deserialize
+            activity_serializer = related_model.activity_serializer
+            return activity_serializer(**value).to_model(related_model)
+        try:
+            # make sure the value looks like a remote id
+            validate_remote_id(value)
+        except ValidationError:
+            # we don't know what this is, ignore it
+            return None
+        # gets or creates the model field from the remote id
+        return activitypub.resolve_remote_id(related_model, value)
+
+
 class RemoteIdField(ActivitypubFieldMixin, models.CharField):
     ''' a url that serves as a unique identifier '''
     def __init__(self, *args, max_length=255, validators=None, **kwargs):
@@ -95,7 +116,7 @@ class UsernameField(ActivitypubFieldMixin, models.CharField):
         return value.split('@')[0]
 
 
-class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
+class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey):
     ''' activitypub-aware foreign key field '''
     def field_to_activity(self, value):
         if not value:
@@ -103,7 +124,7 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
         return value.remote_id
 
 
-class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
+class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField):
     ''' activitypub-aware foreign key field '''
     def field_to_activity(self, value):
         if not value:
@@ -122,6 +143,15 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
             return '%s/%s' % (value.instance.remote_id, self.name)
         return [i.remote_id for i in value.all()]
 
+    def field_from_activity(self, value):
+        items = []
+        for remote_id in value:
+            validate_remote_id(remote_id)
+            items.append(
+                activitypub.resolve_remote_id(self.related_model, remote_id)
+            )
+        return items
+
 
 class TagField(ManyToManyField):
     ''' special case of many to many that uses Tags '''
@@ -142,6 +172,15 @@ class TagField(ManyToManyField):
             ))
         return tags
 
+    def field_from_activity(self, value):
+        items = []
+        for link_json in value:
+            link = activitypub.Link(**link_json)
+            items.append(
+                activitypub.resolve_remote_id(self.related_model, link.href)
+            )
+        return items
+
 
 def image_serializer(value):
     ''' helper for serializing images '''
diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index b157aab3..06ff01e0 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -14,7 +14,7 @@ from django.test import TestCase
 from django.utils import timezone
 
 from bookwyrm.models import fields, User
-from bookwyrm.settings import DOMAIN
+from bookwyrm import activitypub
 
 class ActivitypubFields(TestCase):
     def test_validate_remote_id(self):
@@ -86,8 +86,23 @@ class ActivitypubFields(TestCase):
         instance = fields.ForeignKey('User', on_delete=models.CASCADE)
         Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
         item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c')
+        # returns the remote_id field of the related object
         self.assertEqual(instance.field_to_activity(item), 'https://e.b/c')
 
+    def test_foreign_key_from_activity(self):
+        instance = fields.ForeignKey(User, on_delete=models.CASCADE)
+
+        # test receiving an unknown remote id and loading data TODO
+
+        # test recieving activity json TODO
+
+        # test receiving a remote id of an object in the db
+        user = User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        value = instance.field_from_activity(user.remote_id)
+        self.assertEqual(value, user)
+
+
     def test_one_to_one_field(self):
         instance = fields.OneToOneField('User', on_delete=models.CASCADE)
         Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))

From 69bb3f27516374b003c55da7ae624c211bc8ed3f Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 7 Dec 2020 09:14:30 -0800
Subject: [PATCH 058/104] Fixes validation error in many to many field
 deserializer

---
 bookwyrm/models/fields.py            | 15 +++++++++---
 bookwyrm/tests/models/test_fields.py | 36 ++++++++++++++++++++++++++--
 2 files changed, 46 insertions(+), 5 deletions(-)

diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index e1636f84..0d6c678a 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -92,7 +92,10 @@ class UsernameField(ActivitypubFieldMixin, models.CharField):
     ''' activitypub-aware username field '''
     def __init__(self, activitypub_field='preferredUsername'):
         self.activitypub_field = activitypub_field
-        super(ActivitypubFieldMixin, self).__init__(
+        # I don't totally know why pylint is mad at this, but it makes it work
+        super( #pylint: disable=bad-super-call
+            ActivitypubFieldMixin, self
+        ).__init__(
             _('username'),
             max_length=150,
             unique=True,
@@ -146,7 +149,10 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
     def field_from_activity(self, value):
         items = []
         for remote_id in value:
-            validate_remote_id(remote_id)
+            try:
+                validate_remote_id(remote_id)
+            except ValidationError:
+                return None
             items.append(
                 activitypub.resolve_remote_id(self.related_model, remote_id)
             )
@@ -207,7 +213,10 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
             url = image_slug
         else:
             return None
-        if not url:
+
+        try:
+            validate_remote_id(url)
+        except ValidationError:
             return None
 
         response = get_image(url)
diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index 06ff01e0..8fdb0e2b 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -1,6 +1,7 @@
 ''' testing models '''
 from io import BytesIO
 from collections import namedtuple
+import json
 import pathlib
 import re
 
@@ -14,10 +15,11 @@ from django.test import TestCase
 from django.utils import timezone
 
 from bookwyrm.models import fields, User
-from bookwyrm import activitypub
 
 class ActivitypubFields(TestCase):
+    ''' overwrites standard model feilds to work with activitypub '''
     def test_validate_remote_id(self):
+        ''' should look like a url '''
         self.assertIsNone(fields.validate_remote_id(
             'http://www.example.com'
         ))
@@ -41,6 +43,7 @@ class ActivitypubFields(TestCase):
         )
 
     def test_activitypub_field_mixin(self):
+        ''' generic mixin with super basic to and from functionality '''
         instance = fields.ActivitypubFieldMixin()
         self.assertEqual(instance.field_to_activity('fish'), 'fish')
         self.assertEqual(instance.field_from_activity('fish'), 'fish')
@@ -63,6 +66,7 @@ class ActivitypubFields(TestCase):
         self.assertEqual(instance.get_activitypub_field(), 'snakeCaseName')
 
     def test_remote_id_field(self):
+        ''' just sets some defaults on charfield '''
         instance = fields.RemoteIdField()
         self.assertEqual(instance.max_length, 255)
 
@@ -70,6 +74,7 @@ class ActivitypubFields(TestCase):
             instance.run_validators('http://www.example.com/dlfjg 23/x')
 
     def test_username_field(self):
+        ''' again, just setting defaults on username field '''
         instance = fields.UsernameField()
         self.assertEqual(instance.activitypub_field, 'preferredUsername')
         self.assertEqual(instance.max_length, 150)
@@ -83,6 +88,7 @@ class ActivitypubFields(TestCase):
         self.assertEqual(instance.field_to_activity('test@example.com'), 'test')
 
     def test_foreign_key(self):
+        ''' should be able to format a related model '''
         instance = fields.ForeignKey('User', on_delete=models.CASCADE)
         Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
         item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c')
@@ -90,11 +96,22 @@ class ActivitypubFields(TestCase):
         self.assertEqual(instance.field_to_activity(item), 'https://e.b/c')
 
     def test_foreign_key_from_activity(self):
+        ''' this is the important stuff '''
         instance = fields.ForeignKey(User, on_delete=models.CASCADE)
 
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json'
+        )
+        userdata = json.loads(datafile.read_bytes())
+        del userdata['icon']
         # test receiving an unknown remote id and loading data TODO
 
-        # test recieving activity json TODO
+        # test recieving activity json
+        value = instance.field_from_activity(userdata)
+        self.assertIsInstance(value, User)
+        self.assertEqual(value.remote_id, 'https://example.com/user/mouse')
+        self.assertEqual(value.name, 'MOUSE?? MOUSE!!')
+        # et cetera but we're not testing serializing user json
 
         # test receiving a remote id of an object in the db
         user = User.objects.create_user(
@@ -104,12 +121,14 @@ class ActivitypubFields(TestCase):
 
 
     def test_one_to_one_field(self):
+        ''' a gussied up foreign key '''
         instance = fields.OneToOneField('User', on_delete=models.CASCADE)
         Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
         item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c')
         self.assertEqual(instance.field_to_activity(item), {'a': 'b'})
 
     def test_many_to_many_field(self):
+        ''' lists! '''
         instance = fields.ManyToManyField('User')
 
         Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
@@ -128,7 +147,12 @@ class ActivitypubFields(TestCase):
             'example.com/snake_case'
         )
 
+    def test_many_to_many_field_from_activity(self):
+        ''' resolve related fields for a list '''
+        # TODO
+
     def test_tag_field(self):
+        ''' a special type of many to many field '''
         instance = fields.TagField('User')
 
         Serializable = namedtuple(
@@ -150,8 +174,14 @@ class ActivitypubFields(TestCase):
         self.assertEqual(result[0].type, 'Serializable')
 
 
+    def test_tag_field_from_activity(self):
+        ''' loadin' a list of items from Links '''
+        # TODO
+
+
     @responses.activate
     def test_image_field(self):
+        ''' storing images '''
         user = User.objects.create_user(
             'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
         image_file = pathlib.Path(__file__).parent.joinpath(
@@ -189,6 +219,7 @@ class ActivitypubFields(TestCase):
 
 
     def test_datetime_field(self):
+        ''' this one is pretty simple, it just has to use isoformat '''
         instance = fields.DateTimeField()
         now = timezone.now()
         self.assertEqual(instance.field_to_activity(now), now.isoformat())
@@ -199,5 +230,6 @@ class ActivitypubFields(TestCase):
 
 
     def test_array_field(self):
+        ''' idk why it makes them strings but probably for a good reason '''
         instance = fields.ArrayField(fields.IntegerField)
         self.assertEqual(instance.field_to_activity([0, 1]), ['0', '1'])

From 4599df752d96c1c71c67e590bcd953c89e86af87 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 7 Dec 2020 09:31:12 -0800
Subject: [PATCH 059/104] Adds tests for many to many field deserialization

---
 bookwyrm/models/fields.py            |  2 +-
 bookwyrm/tests/models/test_fields.py | 35 +++++++++++++++++++++++++---
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 0d6c678a..41e34b36 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -152,7 +152,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
             try:
                 validate_remote_id(remote_id)
             except ValidationError:
-                return None
+                continue
             items.append(
                 activitypub.resolve_remote_id(self.related_model, remote_id)
             )
diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index 8fdb0e2b..dec602da 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -95,6 +95,7 @@ class ActivitypubFields(TestCase):
         # returns the remote_id field of the related object
         self.assertEqual(instance.field_to_activity(item), 'https://e.b/c')
 
+    @responses.activate
     def test_foreign_key_from_activity(self):
         ''' this is the important stuff '''
         instance = fields.ForeignKey(User, on_delete=models.CASCADE)
@@ -103,8 +104,16 @@ class ActivitypubFields(TestCase):
             '../data/ap_user.json'
         )
         userdata = json.loads(datafile.read_bytes())
+        # don't try to load the user icon
         del userdata['icon']
-        # test receiving an unknown remote id and loading data TODO
+
+        # test receiving an unknown remote id and loading data
+        responses.add(
+            responses.GET,
+            'https://example.com/user/mouse',
+            json=userdata,
+            status=200)
+        value = instance.field_from_activity('https://example.com/user/mouse')
 
         # test recieving activity json
         value = instance.field_from_activity(userdata)
@@ -147,9 +156,29 @@ class ActivitypubFields(TestCase):
             'example.com/snake_case'
         )
 
+    @responses.activate
     def test_many_to_many_field_from_activity(self):
-        ''' resolve related fields for a list '''
-        # TODO
+        ''' resolve related fields for a list, takes a list of remote ids '''
+        instance = fields.ManyToManyField(User)
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json'
+        )
+        userdata = json.loads(datafile.read_bytes())
+        # don't try to load the user icon
+        del userdata['icon']
+
+        # test receiving an unknown remote id and loading data
+        responses.add(
+            responses.GET,
+            'https://example.com/user/mouse',
+            json=userdata,
+            status=200)
+        value = instance.field_from_activity(
+            ['https://example.com/user/mouse', 'bleh']
+        )
+        self.assertIsInstance(value, list)
+        self.assertEqual(len(value), 1)
+        self.assertIsInstance(value[0], User)
 
     def test_tag_field(self):
         ''' a special type of many to many field '''

From 74ac8d60f84d5c91271e10185ff5196fba1e1c09 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 7 Dec 2020 10:32:31 -0800
Subject: [PATCH 060/104] Starts adding tests for base_activity

includes init and find_existing_by_remote_id
---
 .../tests/activitypub/test_base_activity.py   | 81 +++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 bookwyrm/tests/activitypub/test_base_activity.py

diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py
new file mode 100644
index 00000000..865e1638
--- /dev/null
+++ b/bookwyrm/tests/activitypub/test_base_activity.py
@@ -0,0 +1,81 @@
+''' tests the base functionality for activitypub dataclasses '''
+from dataclasses import dataclass
+from django.test import TestCase
+
+from bookwyrm.activitypub.base_activity import ActivityObject, \
+    find_existing_by_remote_id
+from bookwyrm.activitypub import ActivitySerializerError
+from bookwyrm import models
+
+class BaseActivity(TestCase):
+    ''' the super class for model-linked activitypub dataclasses '''
+    def test_init(self):
+        ''' simple successfuly init '''
+        instance = ActivityObject(id='a', type='b')
+        self.assertTrue(hasattr(instance, 'id'))
+        self.assertTrue(hasattr(instance, 'type'))
+
+    def test_init_missing(self):
+        ''' init with missing required params '''
+        with self.assertRaises(ActivitySerializerError):
+            ActivityObject()
+
+    def test_init_extra_fields(self):
+        ''' init ignoring additional fields '''
+        instance = ActivityObject(id='a', type='b', fish='c')
+        self.assertTrue(hasattr(instance, 'id'))
+        self.assertTrue(hasattr(instance, 'type'))
+
+    def test_init_default_field(self):
+        ''' replace an existing required field with a default field '''
+        @dataclass(init=False)
+        class TestClass(ActivityObject):
+            ''' test class with default field '''
+            type: str = 'TestObject'
+
+        instance = TestClass(id='a')
+        self.assertEqual(instance.id, 'a')
+        self.assertEqual(instance.type, 'TestObject')
+
+    def test_serialize(self):
+        ''' simple function for converting dataclass to dict '''
+        instance = ActivityObject(id='a', type='b')
+        serialized = instance.serialize()
+        self.assertIsInstance(serialized, dict)
+        self.assertEqual(serialized['id'], 'a')
+        self.assertEqual(serialized['type'], 'b')
+
+    def test_find_existing_by_remote_id(self):
+        ''' attempt to match a remote id to an object in the db '''
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        user.remote_id = 'http://example.com/a/b'
+        user.save()
+
+        # uses a different remote id scheme
+        book = models.Edition.objects.create(
+            title='Test Edition', remote_id='http://book.com/book')
+        # this isn't really part of this test directly but it's helpful to state
+        self.assertEqual(book.origin_id, 'http://book.com/book')
+        self.assertNotEqual(book.remote_id, 'http://book.com/book')
+
+        # uses subclasses
+        models.Comment.objects.create(
+            user=user, content='test status', book=book, \
+            remote_id='https://comment.net')
+
+        result = find_existing_by_remote_id(models.User, 'hi')
+        self.assertIsNone(result)
+
+        result = find_existing_by_remote_id(
+            models.User, 'http://example.com/a/b')
+        self.assertEqual(result, user)
+
+        # test using origin id
+        result = find_existing_by_remote_id(
+            models.Edition, 'http://book.com/book')
+        self.assertEqual(result, book)
+
+        # test subclass match
+        result = find_existing_by_remote_id(
+            models.Status, 'https://comment.net')

From f61fcb1261b63a8d7e05605408e2f2d5b8fd5d48 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 7 Dec 2020 10:46:41 -0800
Subject: [PATCH 061/104] Adds tests for resolve_remote_id

---
 .../tests/activitypub/test_base_activity.py   | 39 ++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py
index 865e1638..e30db281 100644
--- a/bookwyrm/tests/activitypub/test_base_activity.py
+++ b/bookwyrm/tests/activitypub/test_base_activity.py
@@ -1,9 +1,14 @@
 ''' tests the base functionality for activitypub dataclasses '''
+import json
+import pathlib
+from unittest.mock import patch
+
 from dataclasses import dataclass
 from django.test import TestCase
+import responses
 
 from bookwyrm.activitypub.base_activity import ActivityObject, \
-    find_existing_by_remote_id
+    find_existing_by_remote_id, resolve_remote_id
 from bookwyrm.activitypub import ActivitySerializerError
 from bookwyrm import models
 
@@ -79,3 +84,35 @@ class BaseActivity(TestCase):
         # test subclass match
         result = find_existing_by_remote_id(
             models.Status, 'https://comment.net')
+
+    @responses.activate
+    def test_resolve_remote_id(self):
+        ''' look up or load remote data '''
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        user.remote_id = 'http://example.com/a/b'
+        user.save()
+
+        # existing item
+        result = resolve_remote_id(models.User, 'http://example.com/a/b')
+        self.assertEqual(result, user)
+
+        # remote item
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json'
+        )
+        userdata = json.loads(datafile.read_bytes())
+        # don't try to load the user icon
+        del userdata['icon']
+        responses.add(
+            responses.GET,
+            'https://example.com/user/mouse',
+            json=userdata,
+            status=200)
+
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            result = resolve_remote_id(
+                models.User, 'https://example.com/user/mouse')
+        self.assertIsInstance(result, models.User)
+        self.assertEqual(result.remote_id, 'https://example.com/user/mouse')
+        self.assertEqual(result.name, 'MOUSE?? MOUSE!!')

From 6817babf3c4fa7753a2efe8a31d37b49549b8a98 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 7 Dec 2020 12:19:15 -0800
Subject: [PATCH 062/104] adds some tests for to_model

---
 bookwyrm/activitypub/base_activity.py         |  2 -
 .../tests/activitypub/test_base_activity.py   | 89 ++++++++++++++-----
 2 files changed, 69 insertions(+), 22 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 4de4dc6b..169d7ee3 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -104,14 +104,12 @@ class ActivityObject:
                 one_to_many_fields[field.name] = value
             elif isinstance(model_field, ImageFileDescriptor):
                 # image fields need custom handling
-                print(model_field, field.name, value)
                 image_fields[field.name] = value
             else:
                 if value == MISSING:
                     value = None
                 mapped_fields[field.name] = value
 
-
         if instance:
             # updating an existing model instance
             for k, v in mapped_fields.items():
diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py
index e30db281..7f915aa3 100644
--- a/bookwyrm/tests/activitypub/test_base_activity.py
+++ b/bookwyrm/tests/activitypub/test_base_activity.py
@@ -1,12 +1,15 @@
 ''' tests the base functionality for activitypub dataclasses '''
+from io import BytesIO
 import json
 import pathlib
 from unittest.mock import patch
 
 from dataclasses import dataclass
 from django.test import TestCase
+from PIL import Image
 import responses
 
+from bookwyrm import activitypub
 from bookwyrm.activitypub.base_activity import ActivityObject, \
     find_existing_by_remote_id, resolve_remote_id
 from bookwyrm.activitypub import ActivitySerializerError
@@ -14,6 +17,20 @@ from bookwyrm import models
 
 class BaseActivity(TestCase):
     ''' the super class for model-linked activitypub dataclasses '''
+    def setUp(self):
+        ''' we're probably going to re-use this so why copy/paste '''
+        self.user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        self.user.remote_id = 'http://example.com/a/b'
+        self.user.save()
+
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json'
+        )
+        self.userdata = json.loads(datafile.read_bytes())
+        # don't try to load the user icon
+        del self.userdata['icon']
+
     def test_init(self):
         ''' simple successfuly init '''
         instance = ActivityObject(id='a', type='b')
@@ -52,11 +69,6 @@ class BaseActivity(TestCase):
 
     def test_find_existing_by_remote_id(self):
         ''' attempt to match a remote id to an object in the db '''
-        user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
-        user.remote_id = 'http://example.com/a/b'
-        user.save()
-
         # uses a different remote id scheme
         book = models.Edition.objects.create(
             title='Test Edition', remote_id='http://book.com/book')
@@ -66,7 +78,7 @@ class BaseActivity(TestCase):
 
         # uses subclasses
         models.Comment.objects.create(
-            user=user, content='test status', book=book, \
+            user=self.user, content='test status', book=book, \
             remote_id='https://comment.net')
 
         result = find_existing_by_remote_id(models.User, 'hi')
@@ -74,7 +86,7 @@ class BaseActivity(TestCase):
 
         result = find_existing_by_remote_id(
             models.User, 'http://example.com/a/b')
-        self.assertEqual(result, user)
+        self.assertEqual(result, self.user)
 
         # test using origin id
         result = find_existing_by_remote_id(
@@ -88,26 +100,15 @@ class BaseActivity(TestCase):
     @responses.activate
     def test_resolve_remote_id(self):
         ''' look up or load remote data '''
-        user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
-        user.remote_id = 'http://example.com/a/b'
-        user.save()
-
         # existing item
         result = resolve_remote_id(models.User, 'http://example.com/a/b')
-        self.assertEqual(result, user)
+        self.assertEqual(result, self.user)
 
         # remote item
-        datafile = pathlib.Path(__file__).parent.joinpath(
-            '../data/ap_user.json'
-        )
-        userdata = json.loads(datafile.read_bytes())
-        # don't try to load the user icon
-        del userdata['icon']
         responses.add(
             responses.GET,
             'https://example.com/user/mouse',
-            json=userdata,
+            json=self.userdata,
             status=200)
 
         with patch('bookwyrm.models.user.set_remote_server.delay'):
@@ -116,3 +117,51 @@ class BaseActivity(TestCase):
         self.assertIsInstance(result, models.User)
         self.assertEqual(result.remote_id, 'https://example.com/user/mouse')
         self.assertEqual(result.name, 'MOUSE?? MOUSE!!')
+
+    def test_to_model(self):
+        ''' the big boy of this module. it feels janky to test this with actual
+        models rather than a test model, but I don't know how to make a test
+        model so here we are. '''
+        instance = ActivityObject(id='a', type='b')
+        with self.assertRaises(ActivitySerializerError):
+            instance.to_model(models.User)
+
+        # test setting simple fields
+        self.assertEqual(self.user.name, '')
+        update_data = activitypub.Person(**self.user.to_activity())
+        update_data.name = 'New Name'
+        update_data.to_model(models.User, self.user)
+
+        self.assertEqual(self.user.name, 'New Name')
+
+    def test_to_model_foreign_key(self):
+        ''' test setting one to one/foreign key '''
+        update_data = activitypub.Person(**self.user.to_activity())
+        update_data.publicKey['publicKeyPem'] = 'hi im secure'
+        update_data.to_model(models.User, self.user)
+        self.assertEqual(self.user.key_pair.public_key, 'hi im secure')
+
+    @responses.activate
+    def test_to_model_image(self):
+        ''' update an image field '''
+        update_data = activitypub.Person(**self.user.to_activity())
+        update_data.icon = {'url': 'http://www.example.com/image.jpg'}
+        image_file = pathlib.Path(__file__).parent.joinpath(
+            '../../static/images/default_avi.jpg')
+        image = Image.open(image_file)
+        output = BytesIO()
+        image.save(output, format=image.format)
+        image_data = output.getvalue()
+        responses.add(
+            responses.GET,
+            'http://www.example.com/image.jpg',
+            body=image_data,
+            status=200)
+
+        self.assertIsNone(self.user.avatar.name)
+        with self.assertRaises(ValueError):
+            self.user.avatar.file #pylint: disable=pointless-statement
+
+        update_data.to_model(models.User, self.user)
+        self.assertIsNotNone(self.user.avatar.name)
+        self.assertIsNotNone(self.user.avatar.file)

From 0a576c325cc70e291f5ab0295adbe88601f995dc Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 7 Dec 2020 13:16:42 -0800
Subject: [PATCH 063/104] Fixes deserializing tags of varied types

---
 bookwyrm/models/fields.py                     |  4 +++
 .../tests/activitypub/test_base_activity.py   | 36 +++++++++++++++----
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 41e34b36..1a6138fd 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -182,6 +182,10 @@ class TagField(ManyToManyField):
         items = []
         for link_json in value:
             link = activitypub.Link(**link_json)
+            tag_type = link.type if link.type != 'Mention' else 'Person'
+            if tag_type != self.related_model.activity_serializer.type:
+                # tags can contain multiple types
+                continue
             items.append(
                 activitypub.resolve_remote_id(self.related_model, link.href)
             )
diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py
index 7f915aa3..44581144 100644
--- a/bookwyrm/tests/activitypub/test_base_activity.py
+++ b/bookwyrm/tests/activitypub/test_base_activity.py
@@ -31,6 +31,9 @@ class BaseActivity(TestCase):
         # don't try to load the user icon
         del self.userdata['icon']
 
+        self.book = models.Edition.objects.create(
+            title='Test Edition', remote_id='http://book.com/book')
+
     def test_init(self):
         ''' simple successfuly init '''
         instance = ActivityObject(id='a', type='b')
@@ -70,15 +73,13 @@ class BaseActivity(TestCase):
     def test_find_existing_by_remote_id(self):
         ''' attempt to match a remote id to an object in the db '''
         # uses a different remote id scheme
-        book = models.Edition.objects.create(
-            title='Test Edition', remote_id='http://book.com/book')
         # this isn't really part of this test directly but it's helpful to state
-        self.assertEqual(book.origin_id, 'http://book.com/book')
-        self.assertNotEqual(book.remote_id, 'http://book.com/book')
+        self.assertEqual(self.book.origin_id, 'http://book.com/book')
+        self.assertNotEqual(self.book.remote_id, 'http://book.com/book')
 
         # uses subclasses
         models.Comment.objects.create(
-            user=self.user, content='test status', book=book, \
+            user=self.user, content='test status', book=self.book, \
             remote_id='https://comment.net')
 
         result = find_existing_by_remote_id(models.User, 'hi')
@@ -91,7 +92,7 @@ class BaseActivity(TestCase):
         # test using origin id
         result = find_existing_by_remote_id(
             models.Edition, 'http://book.com/book')
-        self.assertEqual(result, book)
+        self.assertEqual(result, self.book)
 
         # test subclass match
         result = find_existing_by_remote_id(
@@ -165,3 +166,26 @@ class BaseActivity(TestCase):
         update_data.to_model(models.User, self.user)
         self.assertIsNotNone(self.user.avatar.name)
         self.assertIsNotNone(self.user.avatar.file)
+
+    def test_to_model_many_to_many(self):
+        ''' annoying that these all need special handling '''
+        status = models.Status.objects.create(
+            content='test status',
+            user=self.user,
+        )
+        update_data = activitypub.Note(**status.to_activity())
+        update_data.tag = [
+            {
+                'type': 'Mention',
+                'name': 'gerald',
+                'href': 'http://example.com/a/b'
+            },
+            {
+                'type': 'Edition',
+                'name': 'gerald j. books',
+                'href': 'http://book.com/book'
+            },
+        ]
+        update_data.to_model(models.Status, instance=status)
+        self.assertEqual(status.mention_users.first(), self.user)
+        self.assertEqual(status.mention_books.first(), self.book)

From d0c1a68df61f628618a3099ef43fc0f0439aec10 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 7 Dec 2020 13:39:05 -0800
Subject: [PATCH 064/104] Patches celery call in field tests and fixes tag
 field

---
 bookwyrm/models/fields.py            |  2 ++
 bookwyrm/tests/models/test_fields.py | 12 ++++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 1a6138fd..b4d08b04 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -179,6 +179,8 @@ class TagField(ManyToManyField):
         return tags
 
     def field_from_activity(self, value):
+        if not isinstance(value, list):
+            return None
         items = []
         for link_json in value:
             link = activitypub.Link(**link_json)
diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index dec602da..58d9e08c 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -4,6 +4,7 @@ from collections import namedtuple
 import json
 import pathlib
 import re
+from unittest.mock import patch
 
 from PIL import Image
 import responses
@@ -113,7 +114,9 @@ class ActivitypubFields(TestCase):
             'https://example.com/user/mouse',
             json=userdata,
             status=200)
-        value = instance.field_from_activity('https://example.com/user/mouse')
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            value = instance.field_from_activity(
+                'https://example.com/user/mouse')
 
         # test recieving activity json
         value = instance.field_from_activity(userdata)
@@ -173,9 +176,10 @@ class ActivitypubFields(TestCase):
             'https://example.com/user/mouse',
             json=userdata,
             status=200)
-        value = instance.field_from_activity(
-            ['https://example.com/user/mouse', 'bleh']
-        )
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            value = instance.field_from_activity(
+                ['https://example.com/user/mouse', 'bleh']
+            )
         self.assertIsInstance(value, list)
         self.assertEqual(len(value), 1)
         self.assertIsInstance(value[0], User)

From 4d4ee8b8c3db1c396c18a1a21dd6e642427e7650 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Mon, 7 Dec 2020 18:28:42 -0800
Subject: [PATCH 065/104] Starts getting reverse fields working for
 deserialization

also fixes the fields on the image model and runs a long overdue
migration
---
 bookwyrm/activitypub/base_activity.py         |  77 ++--
 .../migrations/0020_auto_20201208_0213.py     | 353 ++++++++++++++++++
 bookwyrm/models/attachment.py                 |   7 +-
 bookwyrm/models/base_model.py                 |   8 +-
 bookwyrm/models/book.py                       |   3 +-
 bookwyrm/models/status.py                     |   3 +-
 bookwyrm/models/user.py                       |   2 +-
 .../tests/activitypub/test_base_activity.py   |  40 +-
 8 files changed, 434 insertions(+), 59 deletions(-)
 create mode 100644 bookwyrm/migrations/0020_auto_20201208_0213.py

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 169d7ee3..a478a41c 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -76,77 +76,68 @@ class ActivityObject:
 
         # check for an existing instance, if we're not updating a known obj
         if not instance:
-            instance = find_existing_by_remote_id(model, self.id)
+            instance = find_existing_by_remote_id(model, self.id) or model()
 	# TODO: deduplicate books by identifiers
 
-        mapped_fields = {}
         many_to_many_fields = {}
-        one_to_many_fields = {}
-        image_fields = {}
-
         for field in model._meta.get_fields():
             if not hasattr(field, 'field_to_activity'):
                 continue
-            activitypub_field = field.get_activitypub_field()
-            value = field.field_from_activity(getattr(self, activitypub_field))
-            if value is None:
+            # call the formatter associated with the model field class
+            value = field.field_from_activity(
+                getattr(self, field.get_activitypub_field())
+            )
+            if value is None or value is MISSING:
                 continue
 
             model_field = getattr(model, field.name)
 
-            if isinstance(model_field, ForwardManyToOneDescriptor):
-                mapped_fields[field.name] = value
             if isinstance(model_field, ManyToManyDescriptor):
-                # status mentions book/users
+                # status mentions book/users for example, stash this for later
                 many_to_many_fields[field.name] = value
-            elif isinstance(model_field, ReverseManyToOneDescriptor):
-                # attachments on Status, for example
-                one_to_many_fields[field.name] = value
             elif isinstance(model_field, ImageFileDescriptor):
                 # image fields need custom handling
-                image_fields[field.name] = value
+                getattr(instance, field.name).save(*value)
             else:
-                if value == MISSING:
-                    value = None
-                mapped_fields[field.name] = value
+                # just a good old fashioned model.field = value
+                setattr(instance, field.name, value)
 
-        if instance:
-            # updating an existing model instance
-            for k, v in mapped_fields.items():
-                setattr(instance, k, v)
-            instance.save()
-        else:
-            # creating a new model instance
-            instance = model.objects.create(**mapped_fields)
+        instance.save()
 
-        # --- these are all fields that can't be saved until after the
-        # instance has an id (after it's been saved). ---------------#
-
-        # add images
-        for (model_key, value) in image_fields.items():
-            if not value:
-                continue
-            getattr(instance, model_key).save(*value, save=True)
-
-        # add many to many fields
+        # add many to many fields, which have to be set post-save
         for (model_key, values) in many_to_many_fields.items():
             # mention books, mention users, followers
             getattr(instance, model_key).set(values)
 
-        # add one to many fields
-        for (model_key, values) in one_to_many_fields.items():
-            if values == MISSING:
+        if not hasattr(model, 'deserialize_reverse_fields'):
+            return instance
+
+        # reversed relationships in the models
+        for (model_field_name, activity_field_name) in \
+                model.deserialize_reverse_fields:
+            if not activity_field_name:
                 continue
-            model_field = getattr(instance, model_key)
-            related_model = model_field.model
+            # attachments on Status, for example
+            values = getattr(self, activity_field_name)
+            if values is None or values is MISSING:
+                continue
+            try:
+                # this is for one to many
+                related_model = getattr(model, model_field_name).field.model
+            except AttributeError:
+                # it's a one to one or foreign key
+                related_model = getattr(model, model_field_name)\
+                        .related.related_model
+                values = [values]
+
             for item in values:
                 if isinstance(item, str):
                     item = resolve_remote_id(related_model, item)
                 else:
                     item = related_model.activity_serializer(**item)
                     item = item.to_model(related_model)
-                field_name = instance.__class__.__name__.lower()
-                setattr(item, field_name, instance)
+                related_name = instance.__class__.__name__.lower()
+                setattr(item, related_name, instance)
                 item.save()
 
         return instance
diff --git a/bookwyrm/migrations/0020_auto_20201208_0213.py b/bookwyrm/migrations/0020_auto_20201208_0213.py
new file mode 100644
index 00000000..9c5345c7
--- /dev/null
+++ b/bookwyrm/migrations/0020_auto_20201208_0213.py
@@ -0,0 +1,353 @@
+# Generated by Django 3.0.7 on 2020-12-08 02:13
+
+import bookwyrm.models.fields
+from django.conf import settings
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0019_auto_20201130_1939'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='author',
+            name='aliases',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='bio',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='born',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='died',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='openlibrary_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='wikipedia_link',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='authors',
+            field=bookwyrm.models.fields.ManyToManyField(to='bookwyrm.Author'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='cover',
+            field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='covers/'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='description',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='first_published_date',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='goodreads_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='languages',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='librarything_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='openlibrary_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='published_date',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='series',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='series_number',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='sort_title',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subject_places',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subjects',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subtitle',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='title',
+            field=bookwyrm.models.fields.CharField(max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='boost',
+            name='boosted_status',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='boosters', to='bookwyrm.Status'),
+        ),
+        migrations.AlterField(
+            model_name='comment',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='asin',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='isbn_10',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='isbn_13',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='oclc_number',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='pages',
+            field=bookwyrm.models.fields.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='parent_work',
+            field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='physical_format',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='publishers',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+        ),
+        migrations.AlterField(
+            model_name='favorite',
+            name='status',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'),
+        ),
+        migrations.AlterField(
+            model_name='favorite',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='image',
+            name='caption',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='image',
+            name='image',
+            field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='status/'),
+        ),
+        migrations.AlterField(
+            model_name='quotation',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='quotation',
+            name='quote',
+            field=bookwyrm.models.fields.TextField(),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='rating',
+            field=bookwyrm.models.fields.IntegerField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(5)]),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='privacy',
+            field=bookwyrm.models.fields.CharField(choices=[('public', 'Public'), ('unlisted', 'Unlisted'), ('followers', 'Followers'), ('direct', 'Direct')], default='public', max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='added_by',
+            field=bookwyrm.models.fields.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='shelf',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Shelf'),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='content',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='mention_books',
+            field=bookwyrm.models.fields.TagField(related_name='mention_book', to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='mention_users',
+            field=bookwyrm.models.fields.TagField(related_name='mention_user', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='published_date',
+            field=bookwyrm.models.fields.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='reply_parent',
+            field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='sensitive',
+            field=bookwyrm.models.fields.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=100, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='userblocks',
+            name='user_object',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_object', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userblocks',
+            name='user_subject',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_subject', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollowrequest',
+            name='user_object',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_object', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollowrequest',
+            name='user_subject',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_subject', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollows',
+            name='user_object',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_object', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollows',
+            name='user_subject',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_subject', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='tag',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag'),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='work',
+            name='default_edition',
+            field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='work',
+            name='lccn',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]
diff --git a/bookwyrm/models/attachment.py b/bookwyrm/models/attachment.py
index 6a92240d..b3337e15 100644
--- a/bookwyrm/models/attachment.py
+++ b/bookwyrm/models/attachment.py
@@ -9,7 +9,7 @@ from . import fields
 
 class Attachment(ActivitypubMixin, BookWyrmModel):
     ''' an image (or, in the future, video etc) associated with a status '''
-    status = fields.ForeignKey(
+    status = models.ForeignKey(
         'Status',
         on_delete=models.CASCADE,
         related_name='attachments',
@@ -23,7 +23,8 @@ class Attachment(ActivitypubMixin, BookWyrmModel):
 
 class Image(Attachment):
     ''' an image attachment '''
-    image = fields.ImageField(upload_to='status/', null=True, blank=True)
-    caption = fields.TextField(null=True, blank=True)
+    image = fields.ImageField(
+        upload_to='status/', null=True, blank=True, activitypub_field='url')
+    caption = fields.TextField(null=True, blank=True, activitypub_field='name')
 
     activity_serializer = activitypub.Image
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index d3c9471f..1e437152 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -83,9 +83,11 @@ class ActivitypubMixin:
 
         if hasattr(self, 'serialize_reverse_fields'):
             # for example, editions of a work
-            for field_name in self.serialize_reverse_fields:
-                related_field = getattr(self, field_name)
-                activity[field_name] = unfurl_related_field(related_field)
+            for model_field_name, activity_field_name in \
+                    self.serialize_reverse_fields:
+                related_field = getattr(self, model_field_name)
+                activity[activity_field_name] = \
+                        unfurl_related_field(related_field)
 
         if not activity.get('id'):
             activity['id'] = self.get_remote_id()
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index da532561..47b8b99e 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -96,7 +96,8 @@ class Work(OrderedCollectionPageMixin, Book):
         return self.default_edition or self.editions.first()
 
     activity_serializer = activitypub.Work
-    serialize_reverse_fields = ['editions']
+    serialize_reverse_fields = [('editions', 'editions')]
+    deserialize_reverse_fields = [('editions', 'editions')]
 
 
 class Edition(Book):
diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index a2b873bb..55036f2c 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -45,7 +45,8 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
     objects = InheritanceManager()
 
     activity_serializer = activitypub.Note
-    serialize_reverse_fields = ['attachments']
+    serialize_reverse_fields = [('attachments', 'attachment')]
+    deserialize_reverse_fields = [('attachments', 'attachment')]
 
     #----- replies collection activitypub ----#
     @classmethod
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index 6395987b..531b0da2 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -166,7 +166,7 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
         blank=True, null=True, activitypub_field='publicKeyPem')
 
     activity_serializer = activitypub.PublicKey
-    serialize_reverse_fields = ['owner']
+    serialize_reverse_fields = [('owner', 'owner')]
 
     def get_remote_id(self):
         # self.owner is set by the OneToOneField on User
diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py
index 44581144..6892f0e4 100644
--- a/bookwyrm/tests/activitypub/test_base_activity.py
+++ b/bookwyrm/tests/activitypub/test_base_activity.py
@@ -34,6 +34,13 @@ class BaseActivity(TestCase):
         self.book = models.Edition.objects.create(
             title='Test Edition', remote_id='http://book.com/book')
 
+        image_file = pathlib.Path(__file__).parent.joinpath(
+            '../../static/images/default_avi.jpg')
+        image = Image.open(image_file)
+        output = BytesIO()
+        image.save(output, format=image.format)
+        self.image_data = output.getvalue()
+
     def test_init(self):
         ''' simple successfuly init '''
         instance = ActivityObject(id='a', type='b')
@@ -147,16 +154,10 @@ class BaseActivity(TestCase):
         ''' update an image field '''
         update_data = activitypub.Person(**self.user.to_activity())
         update_data.icon = {'url': 'http://www.example.com/image.jpg'}
-        image_file = pathlib.Path(__file__).parent.joinpath(
-            '../../static/images/default_avi.jpg')
-        image = Image.open(image_file)
-        output = BytesIO()
-        image.save(output, format=image.format)
-        image_data = output.getvalue()
         responses.add(
             responses.GET,
             'http://www.example.com/image.jpg',
-            body=image_data,
+            body=self.image_data,
             status=200)
 
         self.assertIsNone(self.user.avatar.name)
@@ -189,3 +190,28 @@ class BaseActivity(TestCase):
         update_data.to_model(models.Status, instance=status)
         self.assertEqual(status.mention_users.first(), self.user)
         self.assertEqual(status.mention_books.first(), self.book)
+
+
+    @responses.activate
+    def test_to_model_one_to_many(self):
+        ''' these are reversed relationships, where the secondary object
+        keys the primary object but not vice versa '''
+        status = models.Status.objects.create(
+            content='test status',
+            user=self.user,
+        )
+        update_data = activitypub.Note(**status.to_activity())
+        update_data.attachment = [{
+            'url': 'http://www.example.com/image.jpg',
+            'name': 'alt text',
+            'type': 'Image',
+        }]
+
+        responses.add(
+            responses.GET,
+            'http://www.example.com/image.jpg',
+            body=self.image_data,
+            status=200)
+
+        update_data.to_model(models.Status, instance=status)
+        self.assertIsInstance(status.attachments.first(), models.Image)

From cc42e9d149cdbc5dcf0825bc600e7f58b9919689 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Tue, 8 Dec 2020 09:43:12 -0800
Subject: [PATCH 066/104] Asyncronously set related fields

---
 bookwyrm/activitypub/base_activity.py | 64 ++++++++++++++++++---------
 bookwyrm/models/fields.py             |  9 ++++
 bookwyrm/templates/book.html          |  4 +-
 celerywyrm/celery.py                  |  3 +-
 4 files changed, 57 insertions(+), 23 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index a478a41c..c784c3c1 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -2,12 +2,13 @@
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
 
-from django.db.models.fields.related_descriptors \
-    import ForwardManyToOneDescriptor, ManyToManyDescriptor, \
-        ReverseManyToOneDescriptor
+from django.apps import apps
+from django.db import transaction
 from django.db.models.fields.files import ImageFileDescriptor
+from django.db.models.fields.related_descriptors import ManyToManyDescriptor
 
 from bookwyrm.connectors import ConnectorException, get_data
+from bookwyrm.tasks import app
 
 class ActivitySerializerError(ValueError):
     ''' routine problems serializing activitypub json '''
@@ -64,7 +65,8 @@ class ActivityObject:
             setattr(self, field.name, value)
 
 
-    def to_model(self, model, instance=None):
+    @transaction.atomic
+    def to_model(self, model, instance=None, save=True):
         ''' convert from an activity to a model instance '''
         if not isinstance(self, model.activity_serializer):
             raise ActivitySerializerError(
@@ -97,26 +99,28 @@ class ActivityObject:
                 many_to_many_fields[field.name] = value
             elif isinstance(model_field, ImageFileDescriptor):
                 # image fields need custom handling
-                getattr(instance, field.name).save(*value)
+                getattr(instance, field.name).save(*value, save=save)
             else:
                 # just a good old fashioned model.field = value
                 setattr(instance, field.name, value)
 
+        if not save:
+            # we can't set many to many and reverse fields on an unsaved object
+            return instance
+
         instance.save()
 
         # add many to many fields, which have to be set post-save
         for (model_key, values) in many_to_many_fields.items():
-            # mention books, mention users, followers
+            # mention books/users, for example
             getattr(instance, model_key).set(values)
 
-        if not hasattr(model, 'deserialize_reverse_fields'):
+        if not save or not hasattr(model, 'deserialize_reverse_fields'):
             return instance
 
         # reversed relationships in the models
         for (model_field_name, activity_field_name) in \
                 model.deserialize_reverse_fields:
-            if not activity_field_name:
-                continue
             # attachments on Status, for example
             values = getattr(self, activity_field_name)
             if values is None or values is MISSING:
@@ -131,15 +135,13 @@ class ActivityObject:
                 values = [values]
 
             for item in values:
-                if isinstance(item, str):
-                    item = resolve_remote_id(related_model, item)
-                else:
-                    item = related_model.activity_serializer(**item)
-                    item = item.to_model(related_model)
-                related_name = instance.__class__.__name__.lower()
-                setattr(item, related_name, instance)
-                item.save()
-
+                set_related_field.delay(
+                    related_model.__name__,
+                    instance.__class__.__name__,
+                    instance.__class__.__name__.lower(),
+                    instance.remote_id,
+                    item
+                )
         return instance
 
 
@@ -150,6 +152,28 @@ class ActivityObject:
         return data
 
 
+@app.task
+@transaction.atomic
+def set_related_field(
+        model_name, origin_model_name,
+        related_field_name, related_remote_id, data):
+    ''' load reverse related fields (editions, attachments) without blocking '''
+    model = apps.get_model('bookwyrm.%s' % model_name, require_ready=True)
+    origin_model = apps.get_model(
+        'bookwyrm.%s' % origin_model_name,
+        require_ready=True
+    )
+
+    if isinstance(data, str):
+        item = resolve_remote_id(model, data, save=False)
+    else:
+        item = model.activity_serializer(**data)
+        item = item.to_model(model, save=False)
+    instance = find_existing_by_remote_id(origin_model, related_remote_id)
+    setattr(item, related_field_name, instance)
+    item.save()
+
+
 def find_existing_by_remote_id(model, remote_id):
     ''' check for an existing instance of this id in the db '''
     objects = model.objects
@@ -168,7 +192,7 @@ def find_existing_by_remote_id(model, remote_id):
     return result
 
 
-def resolve_remote_id(model, remote_id, refresh=False):
+def resolve_remote_id(model, remote_id, refresh=False, save=True):
     ''' look up the remote_id in the database or load it remotely '''
     result = find_existing_by_remote_id(model, remote_id)
     if result and not refresh:
@@ -184,4 +208,4 @@ def resolve_remote_id(model, remote_id, refresh=False):
 
     item = model.activity_serializer(**data)
     # if we're refreshing, "result" will be set and we'll update it
-    return item.to_model(model, instance=result)
+    return item.to_model(model, instance=result, save=save)
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index b4d08b04..f480ab89 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -126,6 +126,15 @@ class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey):
             return None
         return value.remote_id
 
+    def field_from_activity(self, value):
+        print(value)
+        try:
+            validate_remote_id(value)
+        except ValidationError:
+            return None
+        return activitypub.resolve_remote_id(self.related_model, value)
+
+
 
 class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField):
     ''' activitypub-aware foreign key field '''
diff --git a/bookwyrm/templates/book.html b/bookwyrm/templates/book.html
index 51fbdafc..8b21b88c 100644
--- a/bookwyrm/templates/book.html
+++ b/bookwyrm/templates/book.html
@@ -86,8 +86,8 @@
                 {% endif %}
 
 
-                {% if book.parent_work.edition_set.count > 1 %}
-                <p><a href="/book/{{ book.parent_work.id }}/editions">{{ book.parent_work.edition_set.count }} editions</a></p>
+                {% if book.parent_work.editions.count > 1 %}
+                <p><a href="/book/{{ book.parent_work.id }}/editions">{{ book.parent_work.editions.count }} editions</a></p>
                 {% endif %}
             </div>
 
diff --git a/celerywyrm/celery.py b/celerywyrm/celery.py
index a47aad32..efa081ee 100644
--- a/celerywyrm/celery.py
+++ b/celerywyrm/celery.py
@@ -19,8 +19,9 @@ app.config_from_object('django.conf:settings', namespace='CELERY')
 
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks()
-app.autodiscover_tasks(['bookwyrm'], related_name='broadcast')
+app.autodiscover_tasks(['bookwyrm'], related_name='activitypub.base_activity')
 app.autodiscover_tasks(['bookwyrm'], related_name='books_manager')
+app.autodiscover_tasks(['bookwyrm'], related_name='broadcast')
 app.autodiscover_tasks(['bookwyrm'], related_name='emailing')
 app.autodiscover_tasks(['bookwyrm'], related_name='goodreads_import')
 app.autodiscover_tasks(['bookwyrm'], related_name='incoming')

From ef2a07884fb3699c3c2b0e807354d50231f7df8a Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Wed, 9 Dec 2020 11:57:29 -0800
Subject: [PATCH 067/104] Throws validation error when remote_id is None

---
 bookwyrm/models/fields.py                    | 3 +--
 bookwyrm/tests/activitypub/test_quotation.py | 4 ++++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index f480ab89..fae3cb57 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -18,7 +18,7 @@ from bookwyrm.connectors import get_image
 
 def validate_remote_id(value):
     ''' make sure the remote_id looks like a url '''
-    if not re.match(r'^http.?:\/\/[^\s]+$', value):
+    if not value or not re.match(r'^http.?:\/\/[^\s]+$', value):
         raise ValidationError(
             _('%(value)s is not a valid remote_id'),
             params={'value': value},
@@ -127,7 +127,6 @@ class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey):
         return value.remote_id
 
     def field_from_activity(self, value):
-        print(value)
         try:
             validate_remote_id(value)
         except ValidationError:
diff --git a/bookwyrm/tests/activitypub/test_quotation.py b/bookwyrm/tests/activitypub/test_quotation.py
index b0699571..60920889 100644
--- a/bookwyrm/tests/activitypub/test_quotation.py
+++ b/bookwyrm/tests/activitypub/test_quotation.py
@@ -1,3 +1,4 @@
+''' quotation activty object serializer class '''
 import json
 import pathlib
 from unittest.mock import patch
@@ -9,6 +10,7 @@ from bookwyrm import activitypub, models
 class Quotation(TestCase):
     ''' we have hecka ways to create statuses '''
     def setUp(self):
+        ''' model objects we'll need '''
         with patch('bookwyrm.models.user.set_remote_server.delay'):
             self.user = models.User.objects.create_user(
                 'mouse', 'mouse@mouse.mouse', 'mouseword',
@@ -28,6 +30,7 @@ class Quotation(TestCase):
 
 
     def test_quotation_activity(self):
+        ''' create a Quoteation ap object from json '''
         quotation = activitypub.Quotation(**self.status_data)
 
         self.assertEqual(quotation.type, 'Quotation')
@@ -41,6 +44,7 @@ class Quotation(TestCase):
 
 
     def test_activity_to_model(self):
+        ''' create a model instance from an activity object '''
         activity = activitypub.Quotation(**self.status_data)
         quotation = activity.to_model(models.Quotation)
 

From 6b9db97ab86bc008298503ea13729d00ad352621 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Wed, 9 Dec 2020 13:11:42 -0800
Subject: [PATCH 068/104] tests set_related_field

---
 .../tests/activitypub/test_base_activity.py   | 31 +++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py
index 6892f0e4..26d63b61 100644
--- a/bookwyrm/tests/activitypub/test_base_activity.py
+++ b/bookwyrm/tests/activitypub/test_base_activity.py
@@ -11,7 +11,7 @@ import responses
 
 from bookwyrm import activitypub
 from bookwyrm.activitypub.base_activity import ActivityObject, \
-    find_existing_by_remote_id, resolve_remote_id
+    find_existing_by_remote_id, resolve_remote_id, set_related_field
 from bookwyrm.activitypub import ActivitySerializerError
 from bookwyrm import models
 
@@ -213,5 +213,32 @@ class BaseActivity(TestCase):
             body=self.image_data,
             status=200)
 
-        update_data.to_model(models.Status, instance=status)
+        # sets the celery task call to the function call
+        with patch(
+                'bookwyrm.activitypub.base_activity.set_related_field.delay'):
+            update_data.to_model(models.Status, instance=status)
+        self.assertIsNone(status.attachments.first())
+
+
+    @responses.activate
+    def test_set_related_field(self):
+        ''' celery task to add back-references to created objects '''
+        status = models.Status.objects.create(
+            content='test status',
+            user=self.user,
+        )
+        data = {
+            'url': 'http://www.example.com/image.jpg',
+            'name': 'alt text',
+            'type': 'Image',
+        }
+        responses.add(
+            responses.GET,
+            'http://www.example.com/image.jpg',
+            body=self.image_data,
+            status=200)
+        set_related_field(
+            'Image', 'Status', 'status', status.remote_id, data)
+
         self.assertIsInstance(status.attachments.first(), models.Image)
+        self.assertIsNotNone(status.attachments.first().image)

From 7204068d2ae60004ae9e8322bd4857f3ea129f23 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Wed, 9 Dec 2020 13:35:36 -0800
Subject: [PATCH 069/104] Removes unnecessary override of field_from_activity
 on foreign key

---
 bookwyrm/models/fields.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index fae3cb57..fb94e1a6 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -126,14 +126,6 @@ class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey):
             return None
         return value.remote_id
 
-    def field_from_activity(self, value):
-        try:
-            validate_remote_id(value)
-        except ValidationError:
-            return None
-        return activitypub.resolve_remote_id(self.related_model, value)
-
-
 
 class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField):
     ''' activitypub-aware foreign key field '''

From 733e0e19acc24e0fc2b61b1f92cfd99afb3076c5 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Fri, 11 Dec 2020 17:39:17 -0800
Subject: [PATCH 070/104] Don't show boost and original status in timeline

Fixes #381
---
 bookwyrm/views.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bookwyrm/views.py b/bookwyrm/views.py
index ab42557c..0fdfa1f9 100644
--- a/bookwyrm/views.py
+++ b/bookwyrm/views.py
@@ -131,7 +131,7 @@ def get_activity_feed(user, filter_level, model=models.Status):
         activities = model.objects
 
     activities = activities.filter(
-        deleted=False
+        deleted=False,
     ).order_by(
         '-published_date'
     )
@@ -160,6 +160,11 @@ def get_activity_feed(user, filter_level, model=models.Status):
             Q(user__in=following, privacy='followers') | Q(privacy='public')
         )
 
+    try:
+        activities = activities.filter(~Q(boosters__in=activities))
+    except ValueError:
+        pass
+
     return activities
 
 

From a176c6cd35c897d69ef418fb9438e0be7b832a7e Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 09:38:05 -0800
Subject: [PATCH 071/104] Creates merge migration

---
 bookwyrm/migrations/0021_merge_20201212_1737.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 bookwyrm/migrations/0021_merge_20201212_1737.py

diff --git a/bookwyrm/migrations/0021_merge_20201212_1737.py b/bookwyrm/migrations/0021_merge_20201212_1737.py
new file mode 100644
index 00000000..4ccf8c8c
--- /dev/null
+++ b/bookwyrm/migrations/0021_merge_20201212_1737.py
@@ -0,0 +1,14 @@
+# Generated by Django 3.0.7 on 2020-12-12 17:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0020_auto_20201208_0213'),
+        ('bookwyrm', '0016_auto_20201211_2026'),
+    ]
+
+    operations = [
+    ]

From 5cf9e24ae5a2f23c434a723e262a83f2b62c90d2 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 09:43:07 -0800
Subject: [PATCH 072/104] Fixes name import in openlibrary

---
 bookwyrm/connectors/openlibrary.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py
index 90bd7f28..28eb1ea0 100644
--- a/bookwyrm/connectors/openlibrary.py
+++ b/bookwyrm/connectors/openlibrary.py
@@ -65,6 +65,7 @@ class Connector(AbstractConnector):
         ]
 
         self.author_mappings = [
+            Mapping('name'),
             Mapping('born', remote_field='birth_date', formatter=get_date),
             Mapping('died', remote_field='death_date', formatter=get_date),
             Mapping('bio', formatter=get_description),
@@ -185,11 +186,6 @@ class Connector(AbstractConnector):
 
         author = models.Author(openlibrary_key=olkey)
         author = update_from_mappings(author, data, self.author_mappings)
-        name = data.get('name')
-        # TODO this is making some BOLD assumption
-        if name:
-            author.last_name = name.split(' ')[-1]
-            author.first_name = ' '.join(name.split(' ')[:-1])
         author.save()
 
         return author

From 31a407d74a3c5aec4aae8fa41c5132c6124be906 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 09:47:27 -0800
Subject: [PATCH 073/104] Use name field only for author name

It feels janky to remove a more granular name designation, but all these
first/last name fields were algorithmically populated by a dubious
process of splitting the name by a space character. If it makes sense to
have first/last name fields, it should be re-added with some
consideration.
---
 bookwyrm/connectors/abstract_connector.py     |  3 +-
 .../migrations/0022_auto_20201212_1744.py     | 30 +++++++++++++++++++
 bookwyrm/models/author.py                     | 12 --------
 bookwyrm/templates/author.html                |  4 +--
 bookwyrm/templates/snippets/authors.html      |  2 +-
 bookwyrm/templates/snippets/shelf.html        |  2 +-
 bookwyrm/tests/activitypub/test_author.py     |  2 --
 7 files changed, 35 insertions(+), 20 deletions(-)
 create mode 100644 bookwyrm/migrations/0022_auto_20201212_1744.py

diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index a696232d..c9f1ad2e 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -178,8 +178,7 @@ class AbstractConnector(AbstractMinimalConnector):
         author_text = []
         for author in self.get_authors_from_data(data):
             book.authors.add(author)
-            if author.display_name:
-                author_text.append(author.display_name)
+            author_text.append(author.name)
         book.author_text = ', '.join(author_text)
         book.save()
 
diff --git a/bookwyrm/migrations/0022_auto_20201212_1744.py b/bookwyrm/migrations/0022_auto_20201212_1744.py
new file mode 100644
index 00000000..0a98597f
--- /dev/null
+++ b/bookwyrm/migrations/0022_auto_20201212_1744.py
@@ -0,0 +1,30 @@
+# Generated by Django 3.0.7 on 2020-12-12 17:44
+
+from django.db import migrations
+
+
+def set_author_name(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    authors = app_registry.get_model('bookwyrm', 'Author')
+    for author in authors.objects.using(db_alias):
+        if not author.name:
+            author.name = '%s %s' % (author.first_name, author.last_name)
+            author.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0021_merge_20201212_1737'),
+    ]
+
+    operations = [
+        migrations.RunPython(set_author_name),
+        migrations.RemoveField(
+            model_name='author',
+            name='first_name',
+        ),
+        migrations.RemoveField(
+            model_name='author',
+            name='last_name',
+        ),
+    ]
diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py
index 5b70b57f..6098447f 100644
--- a/bookwyrm/models/author.py
+++ b/bookwyrm/models/author.py
@@ -21,8 +21,6 @@ class Author(ActivitypubMixin, BookWyrmModel):
     born = fields.DateTimeField(blank=True, null=True)
     died = fields.DateTimeField(blank=True, null=True)
     name = fields.CharField(max_length=255)
-    last_name = models.CharField(max_length=255, blank=True, null=True)
-    first_name = models.CharField(max_length=255, blank=True, null=True)
     aliases = fields.ArrayField(
         models.CharField(max_length=255), blank=True, default=list
     )
@@ -42,14 +40,4 @@ class Author(ActivitypubMixin, BookWyrmModel):
         ''' editions and works both use "book" instead of model_name '''
         return 'https://%s/author/%s' % (DOMAIN, self.id)
 
-    @property
-    def display_name(self):
-        ''' Helper to return a displayable name'''
-        if self.name:
-            return self.name
-        # don't want to return a spurious space if all of these are None
-        if self.first_name and self.last_name:
-            return self.first_name + ' ' + self.last_name
-        return self.last_name or self.first_name
-
     activity_serializer = activitypub.Author
diff --git a/bookwyrm/templates/author.html b/bookwyrm/templates/author.html
index fb4970e4..3e3e0018 100644
--- a/bookwyrm/templates/author.html
+++ b/bookwyrm/templates/author.html
@@ -2,7 +2,7 @@
 {% load fr_display %}
 {% block content %}
 <div class="block">
-    <h1 class="title">{{ author.display_name }}</h1>
+    <h1 class="title">{{ author.name }}</h1>
 
     {% if author.bio %}
     <p>
@@ -12,7 +12,7 @@
 </div>
 
 <div class="block">
-    <h3 class="title is-4">Books by {{ author.display_name }}</h3>
+    <h3 class="title is-4">Books by {{ author.name }}</h3>
     {% include 'snippets/book_tiles.html' with books=books %}
 </div>
 {% endblock %}
diff --git a/bookwyrm/templates/snippets/authors.html b/bookwyrm/templates/snippets/authors.html
index 165c4cda..e8106f5d 100644
--- a/bookwyrm/templates/snippets/authors.html
+++ b/bookwyrm/templates/snippets/authors.html
@@ -1 +1 @@
-<a href="/author/{{ book.authors.first.id }}" class="author">{{ book.authors.first.display_name }}</a>
+<a href="/author/{{ book.authors.first.id }}" class="author">{{ book.authors.first.name }}</a>
diff --git a/bookwyrm/templates/snippets/shelf.html b/bookwyrm/templates/snippets/shelf.html
index 2df8b024..4e41cd30 100644
--- a/bookwyrm/templates/snippets/shelf.html
+++ b/bookwyrm/templates/snippets/shelf.html
@@ -43,7 +43,7 @@
         <a href="/book/{{ book.id }}">{{ book.title }}</a>
     </td>
     <td>
-        {{ book.authors.first.display_name }}
+        {{ book.authors.first.name }}
     </td>
     <td>
         {% if book.first_published_date %}{{ book.first_published_date }}{% endif %}
diff --git a/bookwyrm/tests/activitypub/test_author.py b/bookwyrm/tests/activitypub/test_author.py
index dd2e93af..fd31f105 100644
--- a/bookwyrm/tests/activitypub/test_author.py
+++ b/bookwyrm/tests/activitypub/test_author.py
@@ -12,8 +12,6 @@ class Author(TestCase):
         )
         self.author = models.Author.objects.create(
             name='Author fullname',
-            first_name='Auth',
-            last_name='Or',
             aliases=['One', 'Two'],
             bio='bio bio bio',
         )

From 7c43fa1f7c316b72f788dc7758aa6b69493e827a Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 13:39:55 -0800
Subject: [PATCH 074/104] Adds deduplication fields

---
 bookwyrm/activitypub/base_activity.py         | 46 +++++++++---------
 bookwyrm/context_processors.py                |  2 +-
 bookwyrm/models/author.py                     |  6 +--
 bookwyrm/models/base_model.py                 | 47 +++++++++++++++++++
 bookwyrm/models/book.py                       | 24 ++++++----
 bookwyrm/models/fields.py                     |  6 ++-
 bookwyrm/models/user.py                       |  4 +-
 .../tests/activitypub/test_base_activity.py   | 30 +-----------
 bookwyrm/tests/models/test_base_model.py      | 40 +++++++++++++++-
 bookwyrm/tests/models/test_fields.py          | 42 +++++++++++++++--
 10 files changed, 174 insertions(+), 73 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index c784c3c1..585f2449 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -78,11 +78,11 @@ class ActivityObject:
 
         # check for an existing instance, if we're not updating a known obj
         if not instance:
-            instance = find_existing_by_remote_id(model, self.id) or model()
-	# TODO: deduplicate books by identifiers
+            instance = model.find_existing(self.serialize()) or model()
 
         many_to_many_fields = {}
         for field in model._meta.get_fields():
+            # check if it's an activitypub field
             if not hasattr(field, 'field_to_activity'):
                 continue
             # call the formatter associated with the model field class
@@ -167,34 +167,25 @@ def set_related_field(
     if isinstance(data, str):
         item = resolve_remote_id(model, data, save=False)
     else:
-        item = model.activity_serializer(**data)
-        item = item.to_model(model, save=False)
-    instance = find_existing_by_remote_id(origin_model, related_remote_id)
+        # look for a match based on all the available data
+        item = model.find_existing(data)
+        if not item:
+            # create a new model instance
+            item = model.activity_serializer(**data)
+            item = item.to_model(model, save=False)
+    # this must exist because it's the object that triggered this function
+    instance = origin_model.find_existing_by_remote_id(related_remote_id)
+    if not instance:
+        raise ValueError('Invalid related remote id: %s' % related_remote_id)
+
+    # edition.parent_work = instance, for example
     setattr(item, related_field_name, instance)
     item.save()
 
 
-def find_existing_by_remote_id(model, remote_id):
-    ''' check for an existing instance of this id in the db '''
-    objects = model.objects
-    if hasattr(model.objects, 'select_subclasses'):
-        objects = objects.select_subclasses()
-
-    # first, check for an existing copy in the database
-    result = objects.filter(
-        remote_id=remote_id
-    ).first()
-
-    if not result and hasattr(model, 'origin_id'):
-        result = objects.filter(
-            origin_id=remote_id
-        ).first()
-    return result
-
-
 def resolve_remote_id(model, remote_id, refresh=False, save=True):
-    ''' look up the remote_id in the database or load it remotely '''
-    result = find_existing_by_remote_id(model, remote_id)
+    ''' take a remote_id and return an instance, creating if necessary '''
+    result = model.find_existing_by_remote_id(remote_id)
     if result and not refresh:
         return result
 
@@ -206,6 +197,11 @@ def resolve_remote_id(model, remote_id, refresh=False, save=True):
             'Could not connect to host for remote_id in %s model: %s' % \
                 (model.__name__, remote_id))
 
+    # check for existing items with shared unique identifiers
+    item = model.find_existing(data)
+    if item:
+        return item
+
     item = model.activity_serializer(**data)
     # if we're refreshing, "result" will be set and we'll update it
     return item.to_model(model, instance=result, save=save)
diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py
index 72839dce..8422f3c3 100644
--- a/bookwyrm/context_processors.py
+++ b/bookwyrm/context_processors.py
@@ -4,5 +4,5 @@ from bookwyrm import models
 def site_settings(request):
     ''' include the custom info about the site '''
     return {
-        'site': models.SiteSettings.objects.get()
+        'site': models.SiteSettings.objects.first()
     }
diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py
index 6098447f..79973a37 100644
--- a/bookwyrm/models/author.py
+++ b/bookwyrm/models/author.py
@@ -12,11 +12,11 @@ from . import fields
 class Author(ActivitypubMixin, BookWyrmModel):
     ''' basic biographic info '''
     origin_id = models.CharField(max_length=255, null=True)
-    ''' copy of an author from OL '''
-    openlibrary_key = fields.CharField(max_length=255, blank=True, null=True)
+    openlibrary_key = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
     sync = models.BooleanField(default=True)
     last_sync_date = models.DateTimeField(default=timezone.now)
-    wikipedia_link = fields.CharField(max_length=255, blank=True, null=True)
+    wikipedia_link = fields.CharField(max_length=255, blank=True, null=True, deduplication_field=True)
     # idk probably other keys would be useful here?
     born = fields.DateTimeField(blank=True, null=True)
     died = fields.DateTimeField(blank=True, null=True)
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 1e437152..f44797ab 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -1,5 +1,7 @@
 ''' base model with default fields '''
 from base64 import b64encode
+from functools import reduce
+import operator
 from uuid import uuid4
 
 from Crypto.PublicKey import RSA
@@ -7,6 +9,7 @@ from Crypto.Signature import pkcs1_15
 from Crypto.Hash import SHA256
 from django.core.paginator import Paginator
 from django.db import models
+from django.db.models import Q
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
@@ -64,6 +67,50 @@ class ActivitypubMixin:
     activity_serializer = lambda: {}
     reverse_unfurl = False
 
+    @classmethod
+    def find_existing_by_remote_id(cls, remote_id):
+        ''' look up a remote id in the db '''
+        return cls.find_existing({'id': remote_id})
+
+    @classmethod
+    def find_existing(cls, data):
+        ''' compare data to fields that can be used for deduplation.
+        This always includes remote_id, but can also be unique identifiers
+        like an isbn for an edition '''
+        filters = []
+        for field in cls._meta.get_fields():
+            if not hasattr(field, 'deduplication_field') or \
+                    not field.deduplication_field:
+                continue
+
+            value = data.get(field.activitypub_field)
+            if not value:
+                continue
+            filters.append({field.name: value})
+
+        if hasattr(cls, 'origin_id') and 'id' in data:
+            # kinda janky, but this handles special case for books
+            filters.append({'origin_id': data['id']})
+
+        if not filters:
+            # if there are no deduplication fields, it will match the first
+            # item no matter what. this shouldn't happen but just in case.
+            return None
+
+        objects = cls.objects
+        if hasattr(objects, 'select_subclasses'):
+            objects = objects.select_subclasses()
+
+        # an OR operation on all the match fields
+        match = objects.filter(
+            reduce(
+                operator.or_, (Q(**f) for f in filters)
+            )
+        )
+        # there OUGHT to be only one match
+        return match.first()
+
+
     def to_activity(self):
         ''' convert from a model to an activity '''
         activity = {}
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index 47b8b99e..bcd4bc04 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -16,9 +16,12 @@ class Book(ActivitypubMixin, BookWyrmModel):
     ''' a generic book, which can mean either an edition or a work '''
     origin_id = models.CharField(max_length=255, null=True, blank=True)
     # these identifiers apply to both works and editions
-    openlibrary_key = fields.CharField(max_length=255, blank=True, null=True)
-    librarything_key = fields.CharField(max_length=255, blank=True, null=True)
-    goodreads_key = fields.CharField(max_length=255, blank=True, null=True)
+    openlibrary_key = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    librarything_key = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    goodreads_key = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
 
     # info about where the data comes from and where/if to sync
     sync = models.BooleanField(default=True)
@@ -83,7 +86,8 @@ class Book(ActivitypubMixin, BookWyrmModel):
 class Work(OrderedCollectionPageMixin, Book):
     ''' a work (an abstract concept of a book that manifests in an edition) '''
     # library of congress catalog control number
-    lccn = fields.CharField(max_length=255, blank=True, null=True)
+    lccn = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
     # this has to be nullable but should never be null
     default_edition = fields.ForeignKey(
         'Edition',
@@ -103,10 +107,14 @@ class Work(OrderedCollectionPageMixin, Book):
 class Edition(Book):
     ''' an edition of a book '''
     # these identifiers only apply to editions, not works
-    isbn_10 = fields.CharField(max_length=255, blank=True, null=True)
-    isbn_13 = fields.CharField(max_length=255, blank=True, null=True)
-    oclc_number = fields.CharField(max_length=255, blank=True, null=True)
-    asin = fields.CharField(max_length=255, blank=True, null=True)
+    isbn_10 = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    isbn_13 = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    oclc_number = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    asin = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
     pages = fields.IntegerField(blank=True, null=True)
     physical_format = fields.CharField(max_length=255, blank=True, null=True)
     publishers = fields.ArrayField(
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index fb94e1a6..e6878fb9 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -28,7 +28,9 @@ def validate_remote_id(value):
 class ActivitypubFieldMixin:
     ''' make a database field serializable '''
     def __init__(self, *args, \
-            activitypub_field=None, activitypub_wrapper=None, **kwargs):
+                 activitypub_field=None, activitypub_wrapper=None,
+                 deduplication_field=False, **kwargs):
+        self.deduplication_field = deduplication_field
         if activitypub_wrapper:
             self.activitypub_wrapper = activitypub_field
             self.activitypub_field = activitypub_wrapper
@@ -86,6 +88,8 @@ class RemoteIdField(ActivitypubFieldMixin, models.CharField):
             *args, max_length=max_length, validators=validators,
             **kwargs
         )
+        # for this field, the default is true. false everywhere else.
+        self.deduplication_field = kwargs.get('deduplication_field', True)
 
 
 class UsernameField(ActivitypubFieldMixin, models.CharField):
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index 531b0da2..0ef9a2e6 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -32,7 +32,9 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     inbox = fields.RemoteIdField(unique=True)
     shared_inbox = fields.RemoteIdField(
         activitypub_field='sharedInbox',
-        activitypub_wrapper='endpoints', null=True)
+        activitypub_wrapper='endpoints',
+        deduplication_field=False,
+        null=True)
     federated_server = models.ForeignKey(
         'FederatedServer',
         on_delete=models.PROTECT,
diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py
index 26d63b61..100ff6cf 100644
--- a/bookwyrm/tests/activitypub/test_base_activity.py
+++ b/bookwyrm/tests/activitypub/test_base_activity.py
@@ -11,7 +11,7 @@ import responses
 
 from bookwyrm import activitypub
 from bookwyrm.activitypub.base_activity import ActivityObject, \
-    find_existing_by_remote_id, resolve_remote_id, set_related_field
+    resolve_remote_id, set_related_field
 from bookwyrm.activitypub import ActivitySerializerError
 from bookwyrm import models
 
@@ -77,34 +77,6 @@ class BaseActivity(TestCase):
         self.assertEqual(serialized['id'], 'a')
         self.assertEqual(serialized['type'], 'b')
 
-    def test_find_existing_by_remote_id(self):
-        ''' attempt to match a remote id to an object in the db '''
-        # uses a different remote id scheme
-        # this isn't really part of this test directly but it's helpful to state
-        self.assertEqual(self.book.origin_id, 'http://book.com/book')
-        self.assertNotEqual(self.book.remote_id, 'http://book.com/book')
-
-        # uses subclasses
-        models.Comment.objects.create(
-            user=self.user, content='test status', book=self.book, \
-            remote_id='https://comment.net')
-
-        result = find_existing_by_remote_id(models.User, 'hi')
-        self.assertIsNone(result)
-
-        result = find_existing_by_remote_id(
-            models.User, 'http://example.com/a/b')
-        self.assertEqual(result, self.user)
-
-        # test using origin id
-        result = find_existing_by_remote_id(
-            models.Edition, 'http://book.com/book')
-        self.assertEqual(result, self.book)
-
-        # test subclass match
-        result = find_existing_by_remote_id(
-            models.Status, 'https://comment.net')
-
     @responses.activate
     def test_resolve_remote_id(self):
         ''' look up or load remote data '''
diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py
index a807c47f..966fe3e2 100644
--- a/bookwyrm/tests/models/test_base_model.py
+++ b/bookwyrm/tests/models/test_base_model.py
@@ -11,13 +11,16 @@ from bookwyrm.models.base_model import ActivitypubMixin
 from bookwyrm.settings import DOMAIN
 
 class BaseModel(TestCase):
+    ''' functionality shared across models '''
     def test_remote_id(self):
+        ''' these should be generated '''
         instance = base_model.BookWyrmModel()
         instance.id = 1
         expected = instance.get_remote_id()
         self.assertEqual(expected, 'https://%s/bookwyrmmodel/1' % DOMAIN)
 
     def test_remote_id_with_user(self):
+        ''' format of remote id when there's a user object '''
         user = models.User.objects.create_user(
             'mouse', 'mouse@mouse.com', 'mouseword', local=True)
         instance = base_model.BookWyrmModel()
@@ -46,6 +49,7 @@ class BaseModel(TestCase):
         self.assertIsNone(instance.remote_id)
 
     def test_to_create_activity(self):
+        ''' wrapper for ActivityPub "create" action '''
         user = models.User.objects.create_user(
             'mouse', 'mouse@mouse.com', 'mouseword', local=True)
 
@@ -75,6 +79,7 @@ class BaseModel(TestCase):
         )
 
     def test_to_delete_activity(self):
+        ''' wrapper for Delete activity '''
         user = models.User.objects.create_user(
             'mouse', 'mouse@mouse.com', 'mouseword', local=True)
 
@@ -98,6 +103,7 @@ class BaseModel(TestCase):
             ['https://www.w3.org/ns/activitystreams#Public'])
 
     def test_to_update_activity(self):
+        ''' ditto above but for Update '''
         user = models.User.objects.create_user(
             'mouse', 'mouse@mouse.com', 'mouseword', local=True)
 
@@ -121,6 +127,7 @@ class BaseModel(TestCase):
         self.assertEqual(activity['object'], {})
 
     def test_to_undo_activity(self):
+        ''' and again, for Undo '''
         user = models.User.objects.create_user(
             'mouse', 'mouse@mouse.com', 'mouseword', local=True)
 
@@ -140,12 +147,14 @@ class BaseModel(TestCase):
 
 
     def test_to_activity(self):
+        ''' model to ActivityPub json '''
         @dataclass(init=False)
         class TestActivity(ActivityObject):
+            ''' real simple mock '''
             type: str = 'Test'
 
         class TestModel(ActivitypubMixin, base_model.BookWyrmModel):
-            pass
+            ''' real simple mock model because BookWyrmModel is abstract '''
 
         instance = TestModel()
         instance.remote_id = 'https://www.example.com/test'
@@ -155,3 +164,32 @@ class BaseModel(TestCase):
         self.assertIsInstance(activity, dict)
         self.assertEqual(activity['id'], 'https://www.example.com/test')
         self.assertEqual(activity['type'], 'Test')
+
+
+    def test_find_existing_by_remote_id(self):
+        ''' attempt to match a remote id to an object in the db '''
+        # uses a different remote id scheme
+        # this isn't really part of this test directly but it's helpful to state
+        self.assertEqual(self.book.origin_id, 'http://book.com/book')
+        self.assertNotEqual(self.book.remote_id, 'http://book.com/book')
+
+        # uses subclasses
+        models.Comment.objects.create(
+            user=self.user, content='test status', book=self.book, \
+            remote_id='https://comment.net')
+
+        result = models.User.find_existing_by_remote_id('hi')
+        self.assertIsNone(result)
+
+        result = models.User.find_existing_by_remote_id(
+            'http://example.com/a/b')
+        self.assertEqual(result, self.user)
+
+        # test using origin id
+        result = models.Edition.find_existing_by_remote_id(
+            'http://book.com/book')
+        self.assertEqual(result, self.book)
+
+        # test subclass match
+        result = models.Status.find_existing_by_remote_id(
+            'https://comment.net')
diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index 58d9e08c..a0a95ba9 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -48,6 +48,7 @@ class ActivitypubFields(TestCase):
         instance = fields.ActivitypubFieldMixin()
         self.assertEqual(instance.field_to_activity('fish'), 'fish')
         self.assertEqual(instance.field_from_activity('fish'), 'fish')
+        self.assertFalse(instance.deduplication_field)
 
         instance = fields.ActivitypubFieldMixin(
             activitypub_wrapper='endpoints', activitypub_field='outbox'
@@ -70,6 +71,7 @@ class ActivitypubFields(TestCase):
         ''' just sets some defaults on charfield '''
         instance = fields.RemoteIdField()
         self.assertEqual(instance.max_length, 255)
+        self.assertTrue(instance.deduplication_field)
 
         with self.assertRaises(ValidationError):
             instance.run_validators('http://www.example.com/dlfjg 23/x')
@@ -97,7 +99,7 @@ class ActivitypubFields(TestCase):
         self.assertEqual(instance.field_to_activity(item), 'https://e.b/c')
 
     @responses.activate
-    def test_foreign_key_from_activity(self):
+    def test_foreign_key_from_activity_str(self):
         ''' this is the important stuff '''
         instance = fields.ForeignKey(User, on_delete=models.CASCADE)
 
@@ -117,15 +119,47 @@ class ActivitypubFields(TestCase):
         with patch('bookwyrm.models.user.set_remote_server.delay'):
             value = instance.field_from_activity(
                 'https://example.com/user/mouse')
+        self.assertIsInstance(value, User)
+        self.assertEqual(value.remote_id, 'https://example.com/user/mouse')
+        self.assertEqual(value.name, 'MOUSE?? MOUSE!!')
 
-        # test recieving activity json
-        value = instance.field_from_activity(userdata)
+
+    def test_foreign_key_from_activity_dict(self):
+        ''' test recieving activity json '''
+        instance = fields.ForeignKey(User, on_delete=models.CASCADE)
+
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json'
+        )
+        userdata = json.loads(datafile.read_bytes())
+        # don't try to load the user icon
+        del userdata['icon']
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            value = instance.field_from_activity(userdata)
         self.assertIsInstance(value, User)
         self.assertEqual(value.remote_id, 'https://example.com/user/mouse')
         self.assertEqual(value.name, 'MOUSE?? MOUSE!!')
         # et cetera but we're not testing serializing user json
 
-        # test receiving a remote id of an object in the db
+
+    def test_foreign_key_from_activity_dict_existing(self):
+        ''' test receiving a dict of an existing object in the db '''
+        instance = fields.ForeignKey(User, on_delete=models.CASCADE)
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json'
+        )
+        userdata = json.loads(datafile.read_bytes())
+        user = User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        user.remote_id = 'https://example.com/user/mouse'
+        user.save()
+        value = instance.field_from_activity(userdata)
+        self.assertEqual(value, user)
+
+
+    def test_foreign_key_from_activity_str_existing(self):
+        ''' test receiving a remote id of an existing object in the db '''
+        instance = fields.ForeignKey(User, on_delete=models.CASCADE)
         user = User.objects.create_user(
             'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
         value = instance.field_from_activity(user.remote_id)

From 804066c523de59c535aa22e09c7d4ac6465e1ca9 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 13:47:51 -0800
Subject: [PATCH 075/104] a couple more assertions for testing fk field

---
 bookwyrm/tests/models/test_fields.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index a0a95ba9..a1e4ff71 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -100,16 +100,18 @@ class ActivitypubFields(TestCase):
 
     @responses.activate
     def test_foreign_key_from_activity_str(self):
-        ''' this is the important stuff '''
+        ''' create a new object from a foreign key '''
         instance = fields.ForeignKey(User, on_delete=models.CASCADE)
-
         datafile = pathlib.Path(__file__).parent.joinpath(
-            '../data/ap_user.json'
-        )
+            '../data/ap_user.json')
         userdata = json.loads(datafile.read_bytes())
         # don't try to load the user icon
         del userdata['icon']
 
+        # it shouldn't match with this unrelated user:
+        unrelated_user = User.objects.create_user(
+            'rat', 'rat@rat.rat', 'ratword', local=True)
+
         # test receiving an unknown remote id and loading data
         responses.add(
             responses.GET,
@@ -120,6 +122,7 @@ class ActivitypubFields(TestCase):
             value = instance.field_from_activity(
                 'https://example.com/user/mouse')
         self.assertIsInstance(value, User)
+        self.assertNotEqual(value, unrelated_user)
         self.assertEqual(value.remote_id, 'https://example.com/user/mouse')
         self.assertEqual(value.name, 'MOUSE?? MOUSE!!')
 
@@ -127,16 +130,19 @@ class ActivitypubFields(TestCase):
     def test_foreign_key_from_activity_dict(self):
         ''' test recieving activity json '''
         instance = fields.ForeignKey(User, on_delete=models.CASCADE)
-
         datafile = pathlib.Path(__file__).parent.joinpath(
-            '../data/ap_user.json'
-        )
+            '../data/ap_user.json')
         userdata = json.loads(datafile.read_bytes())
         # don't try to load the user icon
         del userdata['icon']
+
+        # it shouldn't match with this unrelated user:
+        unrelated_user = User.objects.create_user(
+            'rat', 'rat@rat.rat', 'ratword', local=True)
         with patch('bookwyrm.models.user.set_remote_server.delay'):
             value = instance.field_from_activity(userdata)
         self.assertIsInstance(value, User)
+        self.assertNotEqual(value, unrelated_user)
         self.assertEqual(value.remote_id, 'https://example.com/user/mouse')
         self.assertEqual(value.name, 'MOUSE?? MOUSE!!')
         # et cetera but we're not testing serializing user json
@@ -153,6 +159,9 @@ class ActivitypubFields(TestCase):
             'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
         user.remote_id = 'https://example.com/user/mouse'
         user.save()
+        User.objects.create_user(
+            'rat', 'rat@rat.rat', 'ratword', local=True)
+
         value = instance.field_from_activity(userdata)
         self.assertEqual(value, user)
 
@@ -162,6 +171,9 @@ class ActivitypubFields(TestCase):
         instance = fields.ForeignKey(User, on_delete=models.CASCADE)
         user = User.objects.create_user(
             'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        User.objects.create_user(
+            'rat', 'rat@rat.rat', 'ratword', local=True)
+
         value = instance.field_from_activity(user.remote_id)
         self.assertEqual(value, user)
 

From 4ed713662eca97d9a5ab50b9386c283fc4d188ac Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 13:57:53 -0800
Subject: [PATCH 076/104] Fixes skipping refresh on matched object

---
 bookwyrm/activitypub/base_activity.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 585f2449..b77e9e77 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -198,9 +198,10 @@ def resolve_remote_id(model, remote_id, refresh=False, save=True):
                 (model.__name__, remote_id))
 
     # check for existing items with shared unique identifiers
-    item = model.find_existing(data)
-    if item:
-        return item
+    if not result:
+        result = model.find_existing(data)
+        if result and not refresh:
+            return result
 
     item = model.activity_serializer(**data)
     # if we're refreshing, "result" will be set and we'll update it

From e7f400533e46827ddfc53776a4e30ed26a60b232 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 14:15:10 -0800
Subject: [PATCH 077/104] Fixes missing book and user objects

---
 .../tests/activitypub/test_base_activity.py     |  7 +++----
 bookwyrm/tests/models/test_base_model.py        | 17 ++++++++++++-----
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py
index 100ff6cf..88997c44 100644
--- a/bookwyrm/tests/activitypub/test_base_activity.py
+++ b/bookwyrm/tests/activitypub/test_base_activity.py
@@ -31,9 +31,6 @@ class BaseActivity(TestCase):
         # don't try to load the user icon
         del self.userdata['icon']
 
-        self.book = models.Edition.objects.create(
-            title='Test Edition', remote_id='http://book.com/book')
-
         image_file = pathlib.Path(__file__).parent.joinpath(
             '../../static/images/default_avi.jpg')
         image = Image.open(image_file)
@@ -146,6 +143,8 @@ class BaseActivity(TestCase):
             content='test status',
             user=self.user,
         )
+        book = models.Edition.objects.create(
+            title='Test Edition', remote_id='http://book.com/book')
         update_data = activitypub.Note(**status.to_activity())
         update_data.tag = [
             {
@@ -161,7 +160,7 @@ class BaseActivity(TestCase):
         ]
         update_data.to_model(models.Status, instance=status)
         self.assertEqual(status.mention_users.first(), self.user)
-        self.assertEqual(status.mention_books.first(), self.book)
+        self.assertEqual(status.mention_books.first(), book)
 
 
     @responses.activate
diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py
index 966fe3e2..65cf892e 100644
--- a/bookwyrm/tests/models/test_base_model.py
+++ b/bookwyrm/tests/models/test_base_model.py
@@ -170,12 +170,19 @@ class BaseModel(TestCase):
         ''' attempt to match a remote id to an object in the db '''
         # uses a different remote id scheme
         # this isn't really part of this test directly but it's helpful to state
-        self.assertEqual(self.book.origin_id, 'http://book.com/book')
-        self.assertNotEqual(self.book.remote_id, 'http://book.com/book')
+        book = models.Edition.objects.create(
+            title='Test Edition', remote_id='http://book.com/book')
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        user.remote_id = 'http://example.com/a/b'
+        user.save()
+
+        self.assertEqual(book.origin_id, 'http://book.com/book')
+        self.assertNotEqual(book.remote_id, 'http://book.com/book')
 
         # uses subclasses
         models.Comment.objects.create(
-            user=self.user, content='test status', book=self.book, \
+            user=user, content='test status', book=book, \
             remote_id='https://comment.net')
 
         result = models.User.find_existing_by_remote_id('hi')
@@ -183,12 +190,12 @@ class BaseModel(TestCase):
 
         result = models.User.find_existing_by_remote_id(
             'http://example.com/a/b')
-        self.assertEqual(result, self.user)
+        self.assertEqual(result, user)
 
         # test using origin id
         result = models.Edition.find_existing_by_remote_id(
             'http://book.com/book')
-        self.assertEqual(result, self.book)
+        self.assertEqual(result, book)
 
         # test subclass match
         result = models.Status.find_existing_by_remote_id(

From eb28708230d375f527c15f47375806c9b44fc13a Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 14:33:51 -0800
Subject: [PATCH 078/104] Reverts site settings to correct state

this was just changed to debug tests
---
 bookwyrm/context_processors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py
index 8422f3c3..72839dce 100644
--- a/bookwyrm/context_processors.py
+++ b/bookwyrm/context_processors.py
@@ -4,5 +4,5 @@ from bookwyrm import models
 def site_settings(request):
     ''' include the custom info about the site '''
     return {
-        'site': models.SiteSettings.objects.first()
+        'site': models.SiteSettings.objects.get()
     }

From 49979fabef7390ad6344c3ce164496af5cecf303 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 15:00:20 -0800
Subject: [PATCH 079/104] More user serialization tests

---
 bookwyrm/models/user.py                   |  1 +
 bookwyrm/tests/activitypub/test_person.py | 13 ++++++++++++-
 bookwyrm/tests/models/test_user_model.py  |  8 ++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index 0ef9a2e6..63549d36 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -190,6 +190,7 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
 
 
 @receiver(models.signals.post_save, sender=User)
+#pylint: disable=unused-argument
 def execute_after_save(sender, instance, created, *args, **kwargs):
     ''' create shelves for new users '''
     if not created:
diff --git a/bookwyrm/tests/activitypub/test_person.py b/bookwyrm/tests/activitypub/test_person.py
index 3e0d74e0..c7a8221c 100644
--- a/bookwyrm/tests/activitypub/test_person.py
+++ b/bookwyrm/tests/activitypub/test_person.py
@@ -1,8 +1,10 @@
+# pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring
 import json
 import pathlib
+from unittest.mock import patch
 from django.test import TestCase
 
-from bookwyrm import activitypub
+from bookwyrm import activitypub, models
 
 
 class Person(TestCase):
@@ -18,3 +20,12 @@ class Person(TestCase):
         self.assertEqual(activity.id, 'https://example.com/user/mouse')
         self.assertEqual(activity.preferredUsername, 'mouse')
         self.assertEqual(activity.type, 'Person')
+
+
+    def test_user_to_model(self):
+        activity = activitypub.Person(**self.user_data)
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            user = activity.to_model(models.User)
+        self.assertEqual(user.username, 'mouse@example.com')
+        self.assertEqual(user.remote_id, 'https://example.com/user/mouse')
+        self.assertFalse(user.local)
diff --git a/bookwyrm/tests/models/test_user_model.py b/bookwyrm/tests/models/test_user_model.py
index dabf760c..0454fb40 100644
--- a/bookwyrm/tests/models/test_user_model.py
+++ b/bookwyrm/tests/models/test_user_model.py
@@ -1,4 +1,5 @@
 ''' testing models '''
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models
@@ -22,6 +23,13 @@ class User(TestCase):
         self.assertIsNotNone(self.user.key_pair.private_key)
         self.assertIsNotNone(self.user.key_pair.public_key)
 
+    def test_remote_user(self):
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            user = models.User.objects.create_user(
+                'rat', 'rat@rat.rat', 'ratword', local=False,
+                remote_id='https://example.com/dfjkg')
+        self.assertEqual(user.username, 'rat@example.com')
+
 
     def test_user_shelves(self):
         shelves = models.Shelf.objects.filter(user=self.user).all()

From cb28c19abc36636ff0ecadaeebff0cbaf5a3549c Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 15:19:57 -0800
Subject: [PATCH 080/104] Use get_data in resolving remote id

---
 bookwyrm/outgoing.py                          | 12 +---
 .../tests/outgoing/test_remote_webfinger.py   | 61 +++++++++++++++++++
 bookwyrm/utils/regex.py                       |  2 +-
 3 files changed, 65 insertions(+), 10 deletions(-)
 create mode 100644 bookwyrm/tests/outgoing/test_remote_webfinger.py

diff --git a/bookwyrm/outgoing.py b/bookwyrm/outgoing.py
index 545ac491..2f94d2c0 100644
--- a/bookwyrm/outgoing.py
+++ b/bookwyrm/outgoing.py
@@ -4,10 +4,10 @@ import re
 from django.db import IntegrityError, transaction
 from django.http import HttpResponseNotFound, JsonResponse
 from django.views.decorators.csrf import csrf_exempt
-import requests
 
 from bookwyrm import activitypub
 from bookwyrm import models
+from bookwyrm.connectors import get_data
 from bookwyrm.broadcast import broadcast
 from bookwyrm.status import create_notification
 from bookwyrm.status import create_generated_note
@@ -52,14 +52,8 @@ def handle_remote_webfinger(query):
     except models.User.DoesNotExist:
         url = 'https://%s/.well-known/webfinger?resource=acct:%s' % \
             (domain, query)
-        try:
-            response = requests.get(url)
-        except requests.exceptions.ConnectionError:
-            return None
-        if not response.ok:
-            return None
-        data = response.json()
-        for link in data['links']:
+        data = get_data(url)
+        for link in data.get('links'):
             if link.get('rel') == 'self':
                 try:
                     user = activitypub.resolve_remote_id(
diff --git a/bookwyrm/tests/outgoing/test_remote_webfinger.py b/bookwyrm/tests/outgoing/test_remote_webfinger.py
new file mode 100644
index 00000000..1bf884a6
--- /dev/null
+++ b/bookwyrm/tests/outgoing/test_remote_webfinger.py
@@ -0,0 +1,61 @@
+''' testing user lookup '''
+import json
+import pathlib
+from unittest.mock import patch
+
+from django.test import TestCase
+import responses
+
+from bookwyrm import models, outgoing
+from bookwyrm.settings import DOMAIN
+
+class TestOutgoingRemoteWebfinger(TestCase):
+    ''' overwrites standard model feilds to work with activitypub '''
+    def setUp(self):
+        ''' get user data ready '''
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json'
+        )
+        self.userdata = json.loads(datafile.read_bytes())
+        del self.userdata['icon']
+
+    def test_existing_user(self):
+        ''' simple database lookup by username '''
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+
+        result = outgoing.handle_remote_webfinger('@mouse@%s' % DOMAIN)
+        self.assertEqual(result, user)
+
+        result = outgoing.handle_remote_webfinger('mouse@%s' % DOMAIN)
+        self.assertEqual(result, user)
+
+
+    @responses.activate
+    def test_load_user(self):
+        username = 'mouse@example.com'
+        wellknown = {
+            "subject": "acct:mouse@example.com",
+            "links": [
+                {
+                    "rel": "self",
+                    "type": "application/activity+json",
+                    "href": "https://example.com/user/mouse"
+                }
+            ]
+        }
+        responses.add(
+            responses.GET,
+            'https://example.com/.well-known/webfinger?resource=acct:%s' \
+                    % username,
+            json=wellknown,
+            status=200)
+        responses.add(
+            responses.GET,
+            'https://example.com/user/mouse',
+            json=self.userdata,
+            status=200)
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            result = outgoing.handle_remote_webfinger('@mouse@example.com')
+            self.assertIsInstance(result, models.User)
+            self.assertEqual(result.username, 'mouse@example.com')
diff --git a/bookwyrm/utils/regex.py b/bookwyrm/utils/regex.py
index 36e211d9..70a43b84 100644
--- a/bookwyrm/utils/regex.py
+++ b/bookwyrm/utils/regex.py
@@ -2,4 +2,4 @@
 
 domain = r'[a-z-A-Z0-9_\-]+\.[a-z]+'
 username = r'@[a-zA-Z_\-\.0-9]+(@%s)?' % domain
-full_username = r'@[a-zA-Z_\-\.0-9]+@%s' % domain
+full_username = r'@?[a-zA-Z_\-\.0-9]+@%s' % domain

From 37aaaa97b20eff2cf97696b612d8a0d80cc7568b Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 15:33:07 -0800
Subject: [PATCH 081/104] Catch http erros for remote_id

---
 bookwyrm/outgoing.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/bookwyrm/outgoing.py b/bookwyrm/outgoing.py
index 2f94d2c0..38b48282 100644
--- a/bookwyrm/outgoing.py
+++ b/bookwyrm/outgoing.py
@@ -4,10 +4,11 @@ import re
 from django.db import IntegrityError, transaction
 from django.http import HttpResponseNotFound, JsonResponse
 from django.views.decorators.csrf import csrf_exempt
+from requests import HTTPError
 
 from bookwyrm import activitypub
 from bookwyrm import models
-from bookwyrm.connectors import get_data
+from bookwyrm.connectors import get_data, ConnectorException
 from bookwyrm.broadcast import broadcast
 from bookwyrm.status import create_notification
 from bookwyrm.status import create_generated_note
@@ -52,7 +53,11 @@ def handle_remote_webfinger(query):
     except models.User.DoesNotExist:
         url = 'https://%s/.well-known/webfinger?resource=acct:%s' % \
             (domain, query)
-        data = get_data(url)
+        try:
+            data = get_data(url)
+        except (ConnectorException, HTTPError):
+            return None
+
         for link in data.get('links'):
             if link.get('rel') == 'self':
                 try:

From e58ef83f20c17a62859a81228531e1564d5f4134 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 15:44:17 -0800
Subject: [PATCH 082/104] Fixes image fields breaking user import

---
 bookwyrm/activitypub/base_activity.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index b77e9e77..ed19af99 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -81,6 +81,7 @@ class ActivityObject:
             instance = model.find_existing(self.serialize()) or model()
 
         many_to_many_fields = {}
+        image_fields = {}
         for field in model._meta.get_fields():
             # check if it's an activitypub field
             if not hasattr(field, 'field_to_activity'):
@@ -99,11 +100,15 @@ class ActivityObject:
                 many_to_many_fields[field.name] = value
             elif isinstance(model_field, ImageFileDescriptor):
                 # image fields need custom handling
-                getattr(instance, field.name).save(*value, save=save)
+                image_fields[field.name] = value
             else:
                 # just a good old fashioned model.field = value
                 setattr(instance, field.name, value)
 
+        # if this isn't here, it messes up saving users. who even knows.
+        for (model_key, value) in image_fields.items():
+            getattr(instance, model_key).save(*value, save=save)
+
         if not save:
             # we can't set many to many and reverse fields on an unsaved object
             return instance

From 9b7f0366e786e79f70cdfd53165a21f80c53fa1c Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 16:01:43 -0800
Subject: [PATCH 083/104] Adds site settings to initdb

---
 bookwyrm/management/commands/initdb.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py
index f29ed102..9fd11787 100644
--- a/bookwyrm/management/commands/initdb.py
+++ b/bookwyrm/management/commands/initdb.py
@@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand, CommandError
 from django.contrib.auth.models import Group, Permission
 from django.contrib.contenttypes.models import ContentType
 
-from bookwyrm.models import Connector, User
+from bookwyrm.models import Connector, SiteSettings, User
 from bookwyrm.settings import DOMAIN
 
 def init_groups():
@@ -73,7 +73,7 @@ def init_connectors():
         identifier='bookwyrm.social',
         name='BookWyrm dot Social',
         connector_file='bookwyrm_connector',
-        base_url='https://bookwyrm.social' ,
+        base_url='https://bookwyrm.social',
         books_url='https://bookwyrm.social/book',
         covers_url='https://bookwyrm.social/images/covers',
         search_url='https://bookwyrm.social/search?q=',
@@ -91,10 +91,14 @@ def init_connectors():
         priority=3,
     )
 
+def init_settings():
+    SiteSettings.objects.create()
+
 class Command(BaseCommand):
     help = 'Initializes the database with starter data'
 
     def handle(self, *args, **options):
         init_groups()
         init_permissions()
-        init_connectors()
\ No newline at end of file
+        init_connectors()
+        init_settings()

From 1e01e76ac25d9bb064317f6cfce794cda6a95b86 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 18:06:48 -0800
Subject: [PATCH 084/104] removes unneeded imports

---
 bookwyrm/models/author.py       | 3 ++-
 bookwyrm/models/base_model.py   | 1 +
 bookwyrm/models/import_job.py   | 1 -
 bookwyrm/models/relationship.py | 2 +-
 bookwyrm/status.py              | 2 +-
 5 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py
index 79973a37..331d2dd6 100644
--- a/bookwyrm/models/author.py
+++ b/bookwyrm/models/author.py
@@ -16,7 +16,8 @@ class Author(ActivitypubMixin, BookWyrmModel):
         max_length=255, blank=True, null=True, deduplication_field=True)
     sync = models.BooleanField(default=True)
     last_sync_date = models.DateTimeField(default=timezone.now)
-    wikipedia_link = fields.CharField(max_length=255, blank=True, null=True, deduplication_field=True)
+    wikipedia_link = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
     # idk probably other keys would be useful here?
     born = fields.DateTimeField(blank=True, null=True)
     died = fields.DateTimeField(blank=True, null=True)
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index f44797ab..dd3065c9 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -44,6 +44,7 @@ class BookWyrmModel(models.Model):
 
 
 @receiver(models.signals.post_save)
+#pylint: disable=unused-argument
 def execute_after_save(sender, instance, created, *args, **kwargs):
     ''' set the remote_id after save (when the id is available) '''
     if not created or not hasattr(instance, 'get_remote_id'):
diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py
index fe39325f..8b09216f 100644
--- a/bookwyrm/models/import_job.py
+++ b/bookwyrm/models/import_job.py
@@ -6,7 +6,6 @@ from django.db import models
 from django.utils import timezone
 
 from bookwyrm import books_manager
-from bookwyrm.connectors import ConnectorException
 from bookwyrm.models import ReadThrough, User, Book
 from bookwyrm.utils.fields import JSONField
 from .base_model import PrivacyLevels
diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py
index 8913b9ab..debe2ace 100644
--- a/bookwyrm/models/relationship.py
+++ b/bookwyrm/models/relationship.py
@@ -37,7 +37,7 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel):
 
     activity_serializer = activitypub.Follow
 
-    def get_remote_id(self, status=None):
+    def get_remote_id(self, status=None):# pylint: disable=arguments-differ
         ''' use shelf identifier in remote_id '''
         status = status or 'follows'
         base_path = self.user_subject.remote_id
diff --git a/bookwyrm/status.py b/bookwyrm/status.py
index 83a106e5..648f2e7d 100644
--- a/bookwyrm/status.py
+++ b/bookwyrm/status.py
@@ -1,7 +1,7 @@
 ''' Handle user activity '''
 from django.utils import timezone
 
-from bookwyrm import activitypub, books_manager, models
+from bookwyrm import models
 from bookwyrm.sanitize_html import InputHtmlParser
 
 

From 2b3daa022711e6e7181cf2574cf95dc59b8cbb27 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 18:13:00 -0800
Subject: [PATCH 085/104] disable some warnings

---
 bookwyrm/context_processors.py | 2 +-
 bookwyrm/forms.py              | 3 ++-
 bookwyrm/goodreads_import.py   | 2 +-
 bookwyrm/incoming.py           | 3 ---
 4 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py
index 72839dce..a1471ac4 100644
--- a/bookwyrm/context_processors.py
+++ b/bookwyrm/context_processors.py
@@ -1,7 +1,7 @@
 ''' customize the info available in context for rendering templates '''
 from bookwyrm import models
 
-def site_settings(request):
+def site_settings(request):# pylint: disable=unused-argument
     ''' include the custom info about the site '''
     return {
         'site': models.SiteSettings.objects.get()
diff --git a/bookwyrm/forms.py b/bookwyrm/forms.py
index 784f1038..a2c3e24b 100644
--- a/bookwyrm/forms.py
+++ b/bookwyrm/forms.py
@@ -30,7 +30,7 @@ class CustomForm(ModelForm):
                 visible.field.widget.attrs['rows'] = None
             visible.field.widget.attrs['class'] = css_classes[input_type]
 
-
+# pylint: disable=missing-class-docstring
 class LoginForm(CustomForm):
     class Meta:
         model = models.User
@@ -131,6 +131,7 @@ class ImportForm(forms.Form):
 
 class ExpiryWidget(widgets.Select):
     def value_from_datadict(self, data, files, name):
+        ''' human-readable exiration time buckets '''
         selected_string = super().value_from_datadict(data, files, name)
 
         if selected_string == 'day':
diff --git a/bookwyrm/goodreads_import.py b/bookwyrm/goodreads_import.py
index 3fd330ab..93fc1c48 100644
--- a/bookwyrm/goodreads_import.py
+++ b/bookwyrm/goodreads_import.py
@@ -53,7 +53,7 @@ def import_data(job_id):
         for item in job.items.all():
             try:
                 item.resolve()
-            except Exception as e:
+            except Exception as e:# pylint: disable=broad-except
                 logger.exception(e)
                 item.fail_reason = 'Error loading book'
                 item.save()
diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index bbbebf0f..4964d393 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -17,9 +17,6 @@ from bookwyrm.signatures import Signature
 @csrf_exempt
 def inbox(request, username):
     ''' incoming activitypub events '''
-    # TODO: should do some kind of checking if the user accepts
-    # this action from the sender probably? idk
-    # but this will just throw a 404 if the user doesn't exist
     try:
         models.User.objects.get(localname=username)
     except models.User.DoesNotExist:

From 1e08eeb4c295b79e8cce1aff6e411f32352ea442 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 18:25:04 -0800
Subject: [PATCH 086/104] Renames custom template tags file

---
 bookwyrm/templates/author.html                            | 2 +-
 bookwyrm/templates/book.html                              | 2 +-
 bookwyrm/templates/editions.html                          | 2 +-
 bookwyrm/templates/feed.html                              | 2 +-
 bookwyrm/templates/followers.html                         | 2 +-
 bookwyrm/templates/following.html                         | 2 +-
 bookwyrm/templates/import_status.html                     | 2 +-
 bookwyrm/templates/layout.html                            | 2 +-
 bookwyrm/templates/notifications.html                     | 2 +-
 bookwyrm/templates/shelf.html                             | 2 +-
 bookwyrm/templates/snippets/avatar.html                   | 2 +-
 bookwyrm/templates/snippets/book_cover.html               | 2 +-
 bookwyrm/templates/snippets/book_preview.html             | 2 +-
 bookwyrm/templates/snippets/boost_button.html             | 2 +-
 bookwyrm/templates/snippets/cover_alt.html                | 2 +-
 bookwyrm/templates/snippets/create_status.html            | 2 +-
 bookwyrm/templates/snippets/fav_button.html               | 2 +-
 bookwyrm/templates/snippets/finish_reading_modal.html     | 2 +-
 bookwyrm/templates/snippets/follow_request_buttons.html   | 2 +-
 bookwyrm/templates/snippets/privacy_select.html           | 2 +-
 bookwyrm/templates/snippets/rate_action.html              | 2 +-
 bookwyrm/templates/snippets/reply_form.html               | 2 +-
 bookwyrm/templates/snippets/shelf.html                    | 2 +-
 bookwyrm/templates/snippets/shelve_button.html            | 2 +-
 bookwyrm/templates/snippets/status.html                   | 2 +-
 bookwyrm/templates/snippets/status_body.html              | 2 +-
 bookwyrm/templates/snippets/status_content.html           | 2 +-
 bookwyrm/templates/snippets/status_header.html            | 2 +-
 bookwyrm/templates/snippets/thread.html                   | 2 +-
 bookwyrm/templates/snippets/trimmed_text.html             | 2 +-
 bookwyrm/templates/snippets/user_header.html              | 2 +-
 bookwyrm/templates/snippets/username.html                 | 2 +-
 bookwyrm/templates/tag.html                               | 2 +-
 bookwyrm/templatetags/{fr_display.py => bookwyrm_tags.py} | 2 +-
 34 files changed, 34 insertions(+), 34 deletions(-)
 rename bookwyrm/templatetags/{fr_display.py => bookwyrm_tags.py} (98%)

diff --git a/bookwyrm/templates/author.html b/bookwyrm/templates/author.html
index 3e3e0018..9a7a20ab 100644
--- a/bookwyrm/templates/author.html
+++ b/bookwyrm/templates/author.html
@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">{{ author.name }}</h1>
diff --git a/bookwyrm/templates/book.html b/bookwyrm/templates/book.html
index 8b21b88c..10c2a27b 100644
--- a/bookwyrm/templates/book.html
+++ b/bookwyrm/templates/book.html
@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% load humanize %}
 {% block content %}
 
diff --git a/bookwyrm/templates/editions.html b/bookwyrm/templates/editions.html
index 273b2cd6..619ceafb 100644
--- a/bookwyrm/templates/editions.html
+++ b/bookwyrm/templates/editions.html
@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">Editions of <a href="/book/{{ work.id }}">"{{ work.title }}"</a></h1>
diff --git a/bookwyrm/templates/feed.html b/bookwyrm/templates/feed.html
index 6e49943a..07ad8d0f 100644
--- a/bookwyrm/templates/feed.html
+++ b/bookwyrm/templates/feed.html
@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 
 <div class="columns">
diff --git a/bookwyrm/templates/followers.html b/bookwyrm/templates/followers.html
index 645e46a1..00cb13ca 100644
--- a/bookwyrm/templates/followers.html
+++ b/bookwyrm/templates/followers.html
@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">
diff --git a/bookwyrm/templates/following.html b/bookwyrm/templates/following.html
index 2cca9127..478ca813 100644
--- a/bookwyrm/templates/following.html
+++ b/bookwyrm/templates/following.html
@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">
diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html
index f91e2cce..6bb903b0 100644
--- a/bookwyrm/templates/import_status.html
+++ b/bookwyrm/templates/import_status.html
@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% load humanize %}
 {% block content %}
 <div class="block">
diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html
index b37c9cda..bcbdca2a 100644
--- a/bookwyrm/templates/layout.html
+++ b/bookwyrm/templates/layout.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
diff --git a/bookwyrm/templates/notifications.html b/bookwyrm/templates/notifications.html
index f9494c31..f31df76d 100644
--- a/bookwyrm/templates/notifications.html
+++ b/bookwyrm/templates/notifications.html
@@ -1,6 +1,6 @@
 {% extends 'layout.html' %}
 {% load humanize %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">Notifications</h1>
diff --git a/bookwyrm/templates/shelf.html b/bookwyrm/templates/shelf.html
index d6842d13..390b9fc6 100644
--- a/bookwyrm/templates/shelf.html
+++ b/bookwyrm/templates/shelf.html
@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 
 <div class="columns">
diff --git a/bookwyrm/templates/snippets/avatar.html b/bookwyrm/templates/snippets/avatar.html
index cb0a12ea..da4b5fd2 100644
--- a/bookwyrm/templates/snippets/avatar.html
+++ b/bookwyrm/templates/snippets/avatar.html
@@ -1,3 +1,3 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <img class="avatar image {% if large %}is-96x96{% else %}is-32x32{% endif %}" src="{% if user.avatar %}/images/{{ user.avatar }}{% else %}/static/images/default_avi.jpg{% endif %}" alt="avatar for {{ user|username }}">
 
diff --git a/bookwyrm/templates/snippets/book_cover.html b/bookwyrm/templates/snippets/book_cover.html
index 3fd83616..ceeef426 100644
--- a/bookwyrm/templates/snippets/book_cover.html
+++ b/bookwyrm/templates/snippets/book_cover.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="cover-container is-{{ size }}">
 {% if book.cover %}
     <img class="book-cover" src="/images/{{ book.cover }}" alt="{% include 'snippets/cover_alt.html' with book=book %}">
diff --git a/bookwyrm/templates/snippets/book_preview.html b/bookwyrm/templates/snippets/book_preview.html
index c675c45f..e7eca455 100644
--- a/bookwyrm/templates/snippets/book_preview.html
+++ b/bookwyrm/templates/snippets/book_preview.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="columns">
     <div class="column is-narrow">
         <div>
diff --git a/bookwyrm/templates/snippets/boost_button.html b/bookwyrm/templates/snippets/boost_button.html
index 57765bed..bf06cef7 100644
--- a/bookwyrm/templates/snippets/boost_button.html
+++ b/bookwyrm/templates/snippets/boost_button.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% with status.id|uuid as uuid %}
 <form name="boost" action="/boost/{{ status.id }}" method="post" onsubmit="return interact(event)" class="boost-{{ status.id }}-{{ uuid }} {% if request.user|boosted:status  %}hidden{% endif %}" data-id="boost-{{ status.id }}-{{ uuid }}">
     {% csrf_token %}
diff --git a/bookwyrm/templates/snippets/cover_alt.html b/bookwyrm/templates/snippets/cover_alt.html
index 52bd52f2..0cccc2e1 100644
--- a/bookwyrm/templates/snippets/cover_alt.html
+++ b/bookwyrm/templates/snippets/cover_alt.html
@@ -1,2 +1,2 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 '{{ book.title }}' Cover ({{ book|edition_info }})
diff --git a/bookwyrm/templates/snippets/create_status.html b/bookwyrm/templates/snippets/create_status.html
index e36b1b19..ac8c0b75 100644
--- a/bookwyrm/templates/snippets/create_status.html
+++ b/bookwyrm/templates/snippets/create_status.html
@@ -1,5 +1,5 @@
 {% load humanize %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 
 <div class="tabs is-boxed">
     <ul role="tablist">
diff --git a/bookwyrm/templates/snippets/fav_button.html b/bookwyrm/templates/snippets/fav_button.html
index 58ece1e4..11e03cdb 100644
--- a/bookwyrm/templates/snippets/fav_button.html
+++ b/bookwyrm/templates/snippets/fav_button.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% with status.id|uuid as uuid %}
 <form name="favorite" action="/favorite/{{ status.id }}" method="POST" onsubmit="return interact(event)" class="fav-{{ status.id }}-{{ uuid }} {% if request.user|liked:status %}hidden{% endif %}" data-id="fav-{{ status.id }}-{{ uuid }}">
     {% csrf_token %}
diff --git a/bookwyrm/templates/snippets/finish_reading_modal.html b/bookwyrm/templates/snippets/finish_reading_modal.html
index 8f8aeeff..d04d508d 100644
--- a/bookwyrm/templates/snippets/finish_reading_modal.html
+++ b/bookwyrm/templates/snippets/finish_reading_modal.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div>
     <input class="toggle-control" type="checkbox" name="finish-reading-{{ uuid }}" id="finish-reading-{{ uuid }}">
     <div class="modal toggle-content hidden">
diff --git a/bookwyrm/templates/snippets/follow_request_buttons.html b/bookwyrm/templates/snippets/follow_request_buttons.html
index e1b80631..b5a64cc8 100644
--- a/bookwyrm/templates/snippets/follow_request_buttons.html
+++ b/bookwyrm/templates/snippets/follow_request_buttons.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% if request.user|follow_request_exists:user %}
 <form action="/accept-follow-request/" method="POST">
     {% csrf_token %}
diff --git a/bookwyrm/templates/snippets/privacy_select.html b/bookwyrm/templates/snippets/privacy_select.html
index c8d974c0..dbe11c43 100644
--- a/bookwyrm/templates/snippets/privacy_select.html
+++ b/bookwyrm/templates/snippets/privacy_select.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="select">
     {% with 0|uuid as uuid %}
     {% if not no_label %}
diff --git a/bookwyrm/templates/snippets/rate_action.html b/bookwyrm/templates/snippets/rate_action.html
index 39154017..b9c443ce 100644
--- a/bookwyrm/templates/snippets/rate_action.html
+++ b/bookwyrm/templates/snippets/rate_action.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <span class="is-sr-only">Leave a rating</span>
 <div class="field is-grouped stars rate-stars">
     {% for i in '12345'|make_list %}
diff --git a/bookwyrm/templates/snippets/reply_form.html b/bookwyrm/templates/snippets/reply_form.html
index 2d8abd23..40cf6de1 100644
--- a/bookwyrm/templates/snippets/reply_form.html
+++ b/bookwyrm/templates/snippets/reply_form.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% with activity.id|uuid as uuid %}
 <form class="is-flex-grow-1" name="reply" action="/reply" method="post" onsubmit="return reply(event)">
     <div class="columns">
diff --git a/bookwyrm/templates/snippets/shelf.html b/bookwyrm/templates/snippets/shelf.html
index 4e41cd30..0d05e661 100644
--- a/bookwyrm/templates/snippets/shelf.html
+++ b/bookwyrm/templates/snippets/shelf.html
@@ -1,5 +1,5 @@
 {% load humanize %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% if shelf.books.all|length > 0 %}
 <table class="table is-striped is-fullwidth">
 
diff --git a/bookwyrm/templates/snippets/shelve_button.html b/bookwyrm/templates/snippets/shelve_button.html
index 857f0f9c..d452169e 100644
--- a/bookwyrm/templates/snippets/shelve_button.html
+++ b/bookwyrm/templates/snippets/shelve_button.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% if request.user.is_authenticated %}
 
 {% with book.id|uuid as uuid %}
diff --git a/bookwyrm/templates/snippets/status.html b/bookwyrm/templates/snippets/status.html
index a7f6a392..13d5ee7c 100644
--- a/bookwyrm/templates/snippets/status.html
+++ b/bookwyrm/templates/snippets/status.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% if not status.deleted %}
     {% if status.status_type == 'Boost' %}
         {% include 'snippets/avatar.html' with user=status.user %}
diff --git a/bookwyrm/templates/snippets/status_body.html b/bookwyrm/templates/snippets/status_body.html
index 4ef763fb..e2022247 100644
--- a/bookwyrm/templates/snippets/status_body.html
+++ b/bookwyrm/templates/snippets/status_body.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% load humanize %}
 
 {% if not status.deleted %}
diff --git a/bookwyrm/templates/snippets/status_content.html b/bookwyrm/templates/snippets/status_content.html
index e71588ad..2ef06421 100644
--- a/bookwyrm/templates/snippets/status_content.html
+++ b/bookwyrm/templates/snippets/status_content.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="block">
     {% if status.status_type == 'Review' %}
     <h3>
diff --git a/bookwyrm/templates/snippets/status_header.html b/bookwyrm/templates/snippets/status_header.html
index 1e4179ad..890b4ab8 100644
--- a/bookwyrm/templates/snippets/status_header.html
+++ b/bookwyrm/templates/snippets/status_header.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% include 'snippets/avatar.html' with user=status.user %}
 {% include 'snippets/username.html' with user=status.user %}
 
diff --git a/bookwyrm/templates/snippets/thread.html b/bookwyrm/templates/snippets/thread.html
index ae42c51a..589702ce 100644
--- a/bookwyrm/templates/snippets/thread.html
+++ b/bookwyrm/templates/snippets/thread.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="block">
 
 {% with depth=depth|add:1 %}
diff --git a/bookwyrm/templates/snippets/trimmed_text.html b/bookwyrm/templates/snippets/trimmed_text.html
index 9ed8fa87..43fcb38e 100644
--- a/bookwyrm/templates/snippets/trimmed_text.html
+++ b/bookwyrm/templates/snippets/trimmed_text.html
@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% with 0|uuid as uuid %}
 {% if full %}
 
diff --git a/bookwyrm/templates/snippets/user_header.html b/bookwyrm/templates/snippets/user_header.html
index d9ca71b2..f4f9733f 100644
--- a/bookwyrm/templates/snippets/user_header.html
+++ b/bookwyrm/templates/snippets/user_header.html
@@ -1,5 +1,5 @@
 {% load humanize %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="block">
     <div class="columns">
         <div class="column is-narrow">
diff --git a/bookwyrm/templates/snippets/username.html b/bookwyrm/templates/snippets/username.html
index ec621f60..fbd6ac85 100644
--- a/bookwyrm/templates/snippets/username.html
+++ b/bookwyrm/templates/snippets/username.html
@@ -1,2 +1,2 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <a href="/user/{{ user | username }}" class="user">{% if user.name %}{{ user.name }}{% else %}{{ user | username }}{% endif %}</a>{% if possessive %}'s{% endif %}{% if show_full and user.name or show_full and user.localname %} ({{ user.username }}){% endif %}
diff --git a/bookwyrm/templates/tag.html b/bookwyrm/templates/tag.html
index c6641385..7eea0c21 100644
--- a/bookwyrm/templates/tag.html
+++ b/bookwyrm/templates/tag.html
@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 
 <div class="block">
diff --git a/bookwyrm/templatetags/fr_display.py b/bookwyrm/templatetags/bookwyrm_tags.py
similarity index 98%
rename from bookwyrm/templatetags/fr_display.py
rename to bookwyrm/templatetags/bookwyrm_tags.py
index 690b4c64..8c77921c 100644
--- a/bookwyrm/templatetags/fr_display.py
+++ b/bookwyrm/templatetags/bookwyrm_tags.py
@@ -116,6 +116,7 @@ def get_book_description(book):
     ''' use the work's text if the book doesn't have it '''
     return book.description or book.parent_work.description
 
+
 @register.filter(name='text_overflow')
 def text_overflow(text):
     ''' dont' let book descriptions run for ages '''
@@ -160,7 +161,6 @@ def time_since(date):
 @register.simple_tag(takes_context=True)
 def active_shelf(context, book):
     ''' check what shelf a user has a book on, if any '''
-    #TODO: books can be on multiple shelves, handle that better
     shelf = models.ShelfBook.objects.filter(
         shelf__user=context['request'].user,
         book=book

From b61a7cccff8ddd9a2b4450d051344f72500b98be Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 19:34:33 -0800
Subject: [PATCH 087/104] Use trimmedwords_html instead of a custom filter

---
 bookwyrm/templates/snippets/trimmed_text.html |   2 +-
 bookwyrm/templatetags/bookwyrm_tags.py        |  17 +-
 bookwyrm/tests/test_templatetags.py           | 209 ++++++++++++++++++
 3 files changed, 211 insertions(+), 17 deletions(-)
 create mode 100644 bookwyrm/tests/test_templatetags.py

diff --git a/bookwyrm/templates/snippets/trimmed_text.html b/bookwyrm/templates/snippets/trimmed_text.html
index 43fcb38e..b3207a92 100644
--- a/bookwyrm/templates/snippets/trimmed_text.html
+++ b/bookwyrm/templates/snippets/trimmed_text.html
@@ -2,7 +2,7 @@
 {% with 0|uuid as uuid %}
 {% if full %}
 
-{% with full|text_overflow as trimmed %}
+{% with full|truncatewords_html:60 as trimmed %}
 {% if trimmed != full %}
 <div>
     <input type="radio" name="show-hide-{{ uuid }}" id="show-{{ uuid }}" class="toggle-control" checked>
diff --git a/bookwyrm/templatetags/bookwyrm_tags.py b/bookwyrm/templatetags/bookwyrm_tags.py
index 8c77921c..deb90427 100644
--- a/bookwyrm/templatetags/bookwyrm_tags.py
+++ b/bookwyrm/templatetags/bookwyrm_tags.py
@@ -13,7 +13,7 @@ register = template.Library()
 
 @register.filter(name='dict_key')
 def dict_key(d, k):
-    '''Returns the given key from a dictionary.'''
+    ''' Returns the given key from a dictionary. '''
     return d.get(k) or 0
 
 
@@ -117,21 +117,6 @@ def get_book_description(book):
     return book.description or book.parent_work.description
 
 
-@register.filter(name='text_overflow')
-def text_overflow(text):
-    ''' dont' let book descriptions run for ages '''
-    if not text:
-        return ''
-    char_max = 400
-    if text and len(text) < char_max:
-        return text
-
-    trimmed = text[:char_max]
-    # go back to the last space
-    trimmed = ' '.join(trimmed.split(' ')[:-1])
-    return trimmed + '...'
-
-
 @register.filter(name='uuid')
 def get_uuid(identifier):
     ''' for avoiding clashing ids when there are many forms '''
diff --git a/bookwyrm/tests/test_templatetags.py b/bookwyrm/tests/test_templatetags.py
new file mode 100644
index 00000000..28f08424
--- /dev/null
+++ b/bookwyrm/tests/test_templatetags.py
@@ -0,0 +1,209 @@
+''' style fixes and lookups for templates '''
+import re
+from unittest.mock import patch
+
+from dateutil.parser import parse
+from django.test import TestCase
+from django.utils import timezone
+
+from bookwyrm import models
+from bookwyrm.templatetags import bookwyrm_tags
+
+
+class TemplateTags(TestCase):
+    ''' lotta different things here '''
+    def setUp(self):
+        ''' create some filler objects '''
+        self.user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            self.remote_user = models.User.objects.create_user(
+                'rat', 'rat@rat.rat', 'ratword',
+                remote_id='http://example.com/rat', local=False)
+        self.book = models.Edition.objects.create(title='Test Book')
+
+
+    def test_dict_key(self):
+        ''' just getting a value out of a dict '''
+        test_dict = {'a': 1, 'b': 3}
+        self.assertEqual(
+            bookwyrm_tags.dict_key(test_dict, 'a'), 1)
+        self.assertEqual(
+            bookwyrm_tags.dict_key(test_dict, 'c'), 0)
+
+
+    def test_get_rating(self):
+        ''' get a user's most recent rating of a book '''
+        models.Review.objects.create(
+            user=self.user, book=self.book, rating=3)
+        self.assertEqual(
+            bookwyrm_tags.get_rating(self.book, self.user), 3)
+
+
+    def test_get_rating_doesnt_exist(self):
+        ''' there is no rating available '''
+        self.assertEqual(
+            bookwyrm_tags.get_rating(self.book, self.user), 0)
+
+
+    def test_get_user_identifer_local(self):
+        ''' fall back to the simplest uid available '''
+        self.assertNotEqual(self.user.username, self.user.localname)
+        self.assertEqual(
+            bookwyrm_tags.get_user_identifier(self.user), 'mouse')
+
+    def test_get_user_identifer_remote(self):
+        ''' for a remote user, should be their full username '''
+        self.assertEqual(
+            bookwyrm_tags.get_user_identifier(self.remote_user),
+            'rat@example.com')
+
+    def test_get_notification_count(self):
+        ''' just countin' '''
+        self.assertEqual(bookwyrm_tags.get_notification_count(self.user), 0)
+
+        models.Notification.objects.create(
+            user=self.user, notification_type='FOLLOW')
+        models.Notification.objects.create(
+            user=self.user, notification_type='FOLLOW')
+
+        models.Notification.objects.create(
+            user=self.remote_user, notification_type='FOLLOW')
+
+        self.assertEqual(bookwyrm_tags.get_notification_count(self.user), 2)
+
+
+    def test_get_replies(self):
+        ''' direct replies to a status '''
+        parent = models.Review.objects.create(
+            user=self.user, book=self.book)
+        first_child = models.Status.objects.create(
+            reply_parent=parent, user=self.user)
+        second_child = models.Status.objects.create(
+            reply_parent=parent, user=self.user)
+        third_child = models.Status.objects.create(
+            reply_parent=parent, user=self.user, deleted=True)
+
+        replies = bookwyrm_tags.get_replies(parent)
+        self.assertEqual(len(replies), 2)
+        self.assertTrue(first_child in replies)
+        self.assertTrue(second_child in replies)
+        self.assertFalse(third_child in replies)
+
+
+    def test_get_parent(self):
+        ''' get the reply parent of a status '''
+        parent = models.Review.objects.create(
+            user=self.user, book=self.book)
+        child = models.Status.objects.create(
+            reply_parent=parent, user=self.user)
+
+        result = bookwyrm_tags.get_parent(child)
+        self.assertEqual(result, parent)
+        self.assertIsInstance(result, models.Review)
+
+
+    def test_get_user_liked(self):
+        ''' did a user like a status '''
+        status = models.Review.objects.create(
+            user=self.remote_user, book=self.book)
+
+        self.assertFalse(bookwyrm_tags.get_user_liked(self.user, status))
+        models.Favorite.objects.create(
+            user=self.user,
+            status=status
+        )
+        self.assertTrue(bookwyrm_tags.get_user_liked(self.user, status))
+
+
+    def test_get_user_boosted(self):
+        ''' did a user boost a status '''
+        status = models.Review.objects.create(
+            user=self.remote_user, book=self.book)
+
+        self.assertFalse(bookwyrm_tags.get_user_boosted(self.user, status))
+        models.Boost.objects.create(
+            user=self.user,
+            boosted_status=status
+        )
+        self.assertTrue(bookwyrm_tags.get_user_boosted(self.user, status))
+
+
+    def test_follow_request_exists(self):
+        ''' does a user want to follow '''
+        self.assertFalse(
+            bookwyrm_tags.follow_request_exists(self.user, self.remote_user))
+
+        models.UserFollowRequest.objects.create(
+            user_subject=self.user,
+            user_object=self.remote_user)
+
+        self.assertFalse(
+            bookwyrm_tags.follow_request_exists(self.user, self.remote_user))
+        self.assertTrue(
+            bookwyrm_tags.follow_request_exists(self.remote_user, self.user))
+
+
+    def test_get_boosted(self):
+        ''' load a boosted status '''
+        status = models.Review.objects.create(
+            user=self.remote_user, book=self.book)
+        boost = models.Boost.objects.create(
+            user=self.user,
+            boosted_status=status
+        )
+        boosted = bookwyrm_tags.get_boosted(boost)
+        self.assertIsInstance(boosted, models.Review)
+        self.assertEqual(boosted, status)
+
+
+    def test_get_edition_info(self):
+        ''' text slug about an edition '''
+        self.assertEqual(
+            bookwyrm_tags.get_edition_info(self.book), '')
+
+        self.book.physical_format = 'worm'
+        self.book.save()
+        self.assertEqual(
+            bookwyrm_tags.get_edition_info(self.book), 'worm')
+
+        self.book.languages = ['English']
+        self.book.save()
+        self.assertEqual(
+            bookwyrm_tags.get_edition_info(self.book), 'worm')
+
+        self.book.languages = ['Glorbish', 'English']
+        self.book.save()
+        self.assertEqual(
+            bookwyrm_tags.get_edition_info(self.book),
+            'worm, Glorbish language')
+
+        self.book.published_date = timezone.make_aware(parse('2020'))
+        self.book.save()
+        self.assertEqual(
+            bookwyrm_tags.get_edition_info(self.book),
+            'worm, Glorbish language, 2020')
+
+
+    def test_get_book_description(self):
+        ''' grab it from the edition or the parent '''
+        work = models.Work.objects.create(title='Test Work')
+        self.book.parent_work = work
+        self.book.save()
+
+        self.assertIsNone(bookwyrm_tags.get_book_description(self.book))
+
+        work.description = 'hi'
+        work.save()
+        self.assertEqual(bookwyrm_tags.get_book_description(self.book), 'hi')
+
+        self.book.description = 'hello'
+        self.book.save()
+        self.assertEqual(bookwyrm_tags.get_book_description(self.book), 'hello')
+
+
+    def test_get_uuid(self):
+        ''' uuid functionality '''
+        uuid = bookwyrm_tags.get_uuid('hi')
+        print(uuid)
+        self.assertTrue(re.match(r'hi[A-Za-z0-9\-]', uuid))

From 47981dc15e59a7f0a2f5442dcd985131c888b947 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 20:02:14 -0800
Subject: [PATCH 088/104] Show year in time_since when ambiguous

---
 bookwyrm/templatetags/bookwyrm_tags.py |  2 ++
 bookwyrm/tests/test_templatetags.py    | 35 +++++++++++++++++++++++++-
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/bookwyrm/templatetags/bookwyrm_tags.py b/bookwyrm/templatetags/bookwyrm_tags.py
index deb90427..849974bf 100644
--- a/bookwyrm/templatetags/bookwyrm_tags.py
+++ b/bookwyrm/templatetags/bookwyrm_tags.py
@@ -132,6 +132,8 @@ def time_since(date):
     delta = now - date
 
     if date < (now - relativedelta(weeks=1)):
+        if date.year != now.year:
+            return date.strftime('%b %-d %Y')
         return date.strftime('%b %-d')
     delta = relativedelta(now, date)
     if delta.days:
diff --git a/bookwyrm/tests/test_templatetags.py b/bookwyrm/tests/test_templatetags.py
index 28f08424..67d28848 100644
--- a/bookwyrm/tests/test_templatetags.py
+++ b/bookwyrm/tests/test_templatetags.py
@@ -3,6 +3,7 @@ import re
 from unittest.mock import patch
 
 from dateutil.parser import parse
+from dateutil.relativedelta import relativedelta
 from django.test import TestCase
 from django.utils import timezone
 
@@ -205,5 +206,37 @@ class TemplateTags(TestCase):
     def test_get_uuid(self):
         ''' uuid functionality '''
         uuid = bookwyrm_tags.get_uuid('hi')
-        print(uuid)
         self.assertTrue(re.match(r'hi[A-Za-z0-9\-]', uuid))
+
+
+    def test_time_since(self):
+        ''' ultraconcise timestamps '''
+        self.assertEqual(bookwyrm_tags.time_since('bleh'), '')
+
+        now = timezone.now()
+        self.assertEqual(bookwyrm_tags.time_since(now), '0s')
+
+        seconds_ago = now - relativedelta(seconds=4)
+        self.assertEqual(bookwyrm_tags.time_since(seconds_ago), '4s')
+
+        minutes_ago = now - relativedelta(minutes=8)
+        self.assertEqual(bookwyrm_tags.time_since(minutes_ago), '8m')
+
+        hours_ago = now - relativedelta(hours=9)
+        self.assertEqual(bookwyrm_tags.time_since(hours_ago), '9h')
+
+        days_ago = now - relativedelta(days=3)
+        self.assertEqual(bookwyrm_tags.time_since(days_ago), '3d')
+
+        # I am not going to figure out how to mock dates tonight.
+        months_ago = now - relativedelta(months=5)
+        self.assertTrue(re.match(
+            r'[A-Z][a-z]{2} \d?\d',
+            bookwyrm_tags.time_since(months_ago)
+        ))
+
+        years_ago = now - relativedelta(years=10)
+        self.assertTrue(re.match(
+            r'[A-Z][a-z]{2} \d?\d \d{4}',
+            bookwyrm_tags.time_since(years_ago)
+        ))

From ae435a31654cfc5e2447c7dca757c1e40a45bbb9 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 20:11:23 -0800
Subject: [PATCH 089/104] =?UTF-8?q?Removes=20sqlite=20support=20?=
 =?UTF-8?q?=F0=9F=98=A2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

RIP, things have gotten too complicated for this I think
---
 bookwyrm/models/import_job.py |  2 +-
 bookwyrm/settings.py          |  4 ---
 bookwyrm/utils/fields.py      | 67 -----------------------------------
 3 files changed, 1 insertion(+), 72 deletions(-)
 delete mode 100644 bookwyrm/utils/fields.py

diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py
index 8b09216f..bf5d5caf 100644
--- a/bookwyrm/models/import_job.py
+++ b/bookwyrm/models/import_job.py
@@ -2,12 +2,12 @@
 import re
 import dateutil.parser
 
+from django.contrib.postgres.fields import JSONField
 from django.db import models
 from django.utils import timezone
 
 from bookwyrm import books_manager
 from bookwyrm.models import ReadThrough, User, Book
-from bookwyrm.utils.fields import JSONField
 from .base_model import PrivacyLevels
 
 
diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py
index 3784158c..c42215b4 100644
--- a/bookwyrm/settings.py
+++ b/bookwyrm/settings.py
@@ -99,10 +99,6 @@ BOOKWYRM_DBS = {
         'HOST': env('POSTGRES_HOST', ''),
         'PORT': 5432
     },
-    'sqlite': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'fedireads.db')
-    }
 }
 
 DATABASES = {
diff --git a/bookwyrm/utils/fields.py b/bookwyrm/utils/fields.py
deleted file mode 100644
index 8978f4bc..00000000
--- a/bookwyrm/utils/fields.py
+++ /dev/null
@@ -1,67 +0,0 @@
-'''Quick and dirty shim for JSONField and ArrayField compatibility on sqlite.
-
-For more info and original code, see:
-- https://medium.com/@philamersune/using-postgresql-jsonfield-in-sqlite-95ad4ad2e5f1
-- https://gist.github.com/pvsune/2e5f9f9ae356d0bff633d896bc7d168b#file-django-sqlite-fields-py
-'''
-
-import json
-
-from django.conf import settings
-from django.contrib.postgres.fields import (
-    JSONField as DjangoJSONField,
-    ArrayField as DjangoArrayField,
-)
-from django.db.models import Field
-
-
-class JSONField(DjangoJSONField):
-    pass
-
-
-class ArrayField(DjangoArrayField):
-    pass
-
-
-if 'sqlite' in settings.DATABASES['default']['ENGINE']:
-    class JSONField(Field):
-        def db_type(self, connection):
-            return 'text'
-
-        def from_db_value(self, value, expression, connection):
-            if value is not None:
-                return self.to_python(value)
-            return value
-
-        def to_python(self, value):
-            if value is not None:
-                try:
-                    return json.loads(value)
-                except (TypeError, ValueError):
-                    return value
-            return value
-
-        def get_prep_value(self, value):
-            if value is not None:
-                return str(json.dumps(value))
-            return value
-
-        def value_to_string(self, obj):
-            return self.value_from_object(obj)
-
-
-    class ArrayField(JSONField):
-        def __init__(self, base_field, size=None, **kwargs):
-            """Care for DjangoArrayField's kwargs."""
-            self.base_field = base_field
-            self.size = size
-            super().__init__(**kwargs)
-
-        def deconstruct(self):
-            """Need to create migrations properly."""
-            name, path, args, kwargs = super().deconstruct()
-            kwargs.update({
-                'base_field': self.base_field.clone(),
-                'size': self.size,
-            })
-            return name, path, args, kwargs

From d3371991e1d3c5775b8c217db1697b376df4e393 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 20:18:03 -0800
Subject: [PATCH 090/104] Remove deleted util file from migrations

---
 bookwyrm/migrations/0001_initial.py              |  6 +++---
 ...221_1702_squashed_0064_merge_20201101_1913.py | 16 ++++++++--------
 bookwyrm/migrations/0016_auto_20201129_0304.py   |  7 +++----
 3 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/bookwyrm/migrations/0001_initial.py b/bookwyrm/migrations/0001_initial.py
index b1aba7df..347057e1 100644
--- a/bookwyrm/migrations/0001_initial.py
+++ b/bookwyrm/migrations/0001_initial.py
@@ -7,7 +7,7 @@ import django.core.validators
 from django.db import migrations, models
 import django.db.models.deletion
 import django.utils.timezone
-import bookwyrm.utils.fields
+from django.contrib.postgres.fields import JSONField
 
 
 class Migration(migrations.Migration):
@@ -62,7 +62,7 @@ class Migration(migrations.Migration):
                 ('content', models.TextField(blank=True, null=True)),
                 ('created_date', models.DateTimeField(auto_now_add=True)),
                 ('openlibrary_key', models.CharField(max_length=255)),
-                ('data', bookwyrm.utils.fields.JSONField()),
+                ('data', JSONField()),
             ],
             options={
                 'abstract': False,
@@ -75,7 +75,7 @@ class Migration(migrations.Migration):
                 ('content', models.TextField(blank=True, null=True)),
                 ('created_date', models.DateTimeField(auto_now_add=True)),
                 ('openlibrary_key', models.CharField(max_length=255, unique=True)),
-                ('data', bookwyrm.utils.fields.JSONField()),
+                ('data', JSONField()),
                 ('cover', models.ImageField(blank=True, null=True, upload_to='covers/')),
                 ('added_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
                 ('authors', models.ManyToManyField(to='bookwyrm.Author')),
diff --git a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py
index 13cb1406..6a149ab5 100644
--- a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py
+++ b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py
@@ -2,7 +2,6 @@
 
 import bookwyrm.models.connector
 import bookwyrm.models.site
-import bookwyrm.utils.fields
 from django.conf import settings
 import django.contrib.postgres.operations
 import django.core.validators
@@ -10,6 +9,7 @@ from django.db import migrations, models
 import django.db.models.deletion
 import django.db.models.expressions
 import django.utils.timezone
+from django.contrib.postgres.fields import JSONField, ArrayField
 import uuid
 
 
@@ -148,7 +148,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='misc_identifiers',
-            field=bookwyrm.utils.fields.JSONField(null=True),
+            field=JSONField(null=True),
         ),
         migrations.AddField(
             model_name='book',
@@ -226,7 +226,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='author',
             name='aliases',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='user',
@@ -394,17 +394,17 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='subject_places',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='book',
             name='subjects',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='edition',
             name='publishers',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AlterField(
             model_name='connector',
@@ -578,7 +578,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='languages',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='edition',
@@ -676,7 +676,7 @@ class Migration(migrations.Migration):
             name='ImportItem',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('data', bookwyrm.utils.fields.JSONField()),
+                ('data', JSONField()),
             ],
         ),
         migrations.CreateModel(
diff --git a/bookwyrm/migrations/0016_auto_20201129_0304.py b/bookwyrm/migrations/0016_auto_20201129_0304.py
index 2bf820e1..1e715969 100644
--- a/bookwyrm/migrations/0016_auto_20201129_0304.py
+++ b/bookwyrm/migrations/0016_auto_20201129_0304.py
@@ -1,10 +1,9 @@
 # Generated by Django 3.0.7 on 2020-11-29 03:04
 
-import bookwyrm.utils.fields
 from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-
+from django.contrib.postgres.fields import ArrayField
 
 class Migration(migrations.Migration):
 
@@ -16,12 +15,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='book',
             name='subject_places',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
         ),
         migrations.AlterField(
             model_name='book',
             name='subjects',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
         ),
         migrations.AlterField(
             model_name='edition',

From 007ccb22fd5f5fa76e00164209d55c34eedaf9d7 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 20:25:58 -0800
Subject: [PATCH 091/104] Update README.md

---
 README.md | 30 +-----------------------------
 1 file changed, 1 insertion(+), 29 deletions(-)

diff --git a/README.md b/README.md
index 51e39541..2dfedebb 100644
--- a/README.md
+++ b/README.md
@@ -60,8 +60,6 @@ cp .env.example .env
 
 For most testing, you'll want to use ngrok. Remember to set the DOMAIN in `.env` to your ngrok domain.
 
-
-#### With Docker
 You'll have to install the Docker and docker-compose. When you're ready, run:
 
 ```bash
@@ -70,33 +68,7 @@ docker-compose run --rm web python manage.py migrate
 docker-compose run --rm web python manage.py initdb
 ```
 
-### Without Docker
-You will need postgres installed and running on your computer.
-
-``` bash
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements.txt
-createdb bookwyrm
-```
-
-Create the psql user in `psql bookwyrm`:
-``` psql
-CREATE ROLE bookwyrm WITH LOGIN PASSWORD 'bookwyrm';
-GRANT ALL PRIVILEGES ON DATABASE bookwyrm TO bookwyrm;
-```
-
-Initialize the database (or, more specifically, delete the existing database, run migrations, and start fresh):
-``` bash
-./rebuilddb.sh
-```
-This creates two users, `mouse` with password `password123` and `rat` with password `ratword`.
-
-The application uses Celery and Redis for task management, which must also be installed and configured.
-
-And go to the app at `localhost:8000`
-
-
+Once the build is complete, you can access the instance at `localhost:1333`
 
 ## Installing in Production
 

From faa21cf2af05234836b2c0b2ae60c9dee345f45f Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 20:26:15 -0800
Subject: [PATCH 092/104] Removes outdated script

---
 rebuilddb.sh | 25 -------------------------
 1 file changed, 25 deletions(-)
 delete mode 100755 rebuilddb.sh

diff --git a/rebuilddb.sh b/rebuilddb.sh
deleted file mode 100755
index d32e5dab..00000000
--- a/rebuilddb.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-set -e
-
-if [ ! -f .env ]; then
-  echo "No .env found -- copying .example.env to .env!"
-  cp .env.example .env
-fi
-
-source .env
-
-if [ $BOOKWYRM_DATABASE_BACKEND = 'sqlite' ]; then
-  if [ -f fedireads.db ]; then
-    rm fedireads.db
-  fi
-else
-  # assume postgres
-  dropdb fedireads
-  createdb fedireads
-fi
-
-python manage.py makemigrations fedireads
-python manage.py migrate
-
-python manage.py initdb
-python manage.py runserver

From ffc4cc2018faf1637f20c2eaf9006e6d4aae604c Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 20:59:41 -0800
Subject: [PATCH 093/104] Fixes create status handler

---
 bookwyrm/incoming.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index 4964d393..de2c5bcc 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -185,12 +185,13 @@ def handle_follow_reject(activity):
 def handle_create(activity):
     ''' someone did something, good on them '''
     # deduplicate incoming activities
-    status_id = activity['object']['id']
+    activity = activity['object']
+    status_id = activity['id']
     if models.Status.objects.filter(remote_id=status_id).count():
         return
 
     serializer = activitypub.activity_objects[activity['type']]
-    status = serializer(**activity)
+    activity = serializer(**activity)
     try:
         model = models.activity_models[activity.type]
     except KeyError:
@@ -198,13 +199,14 @@ def handle_create(activity):
         return
 
     if activity.type == 'Note':
+        # discard notes that aren't replies to existing statuses
         reply = models.Status.objects.filter(
             remote_id=activity.inReplyTo
         ).first()
         if not reply:
             return
 
-    activity.to_model(model)
+    status = activity.to_model(model)
     # create a notification if this is a reply
     if status.reply_parent and status.reply_parent.user.local:
         status_builder.create_notification(

From d65657882e5fda65b977155a17c52aaa235916bb Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sat, 12 Dec 2020 21:11:51 -0800
Subject: [PATCH 094/104] Keep any status that mentions a local user

---
 bookwyrm/incoming.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index de2c5bcc..fe521772 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -199,12 +199,23 @@ def handle_create(activity):
         return
 
     if activity.type == 'Note':
-        # discard notes that aren't replies to existing statuses
+        # keep notes if they are replies to existing statuses
         reply = models.Status.objects.filter(
             remote_id=activity.inReplyTo
         ).first()
+
         if not reply:
-            return
+            discard = True
+            # keep notes if they mention local users
+            tags = [l['href'] for l in activity.tag if l['type'] == 'Mention']
+            for tag in tags:
+                if models.User.objects.filter(
+                        remote_id=tag, local=True).exists():
+                    # we found a mention of a known use boost
+                    discard = False
+                    break
+            if discard:
+                return
 
     status = activity.to_model(model)
     # create a notification if this is a reply

From 957f0889aaefba4fbd749469af074de2643c7a15 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 13 Dec 2020 11:15:42 -0800
Subject: [PATCH 095/104] Clean up models

removes unused function and sorts replies correctly
---
 bookwyrm/models/__init__.py | 5 -----
 bookwyrm/models/status.py   | 9 ++++++---
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py
index b9a2814e..86bdf219 100644
--- a/bookwyrm/models/__init__.py
+++ b/bookwyrm/models/__init__.py
@@ -25,8 +25,3 @@ from .site import SiteSettings, SiteInvite, PasswordReset
 cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass)
 activity_models = {c[1].activity_serializer.__name__: c[1] \
     for c in cls_members if hasattr(c[1], 'activity_serializer')}
-
-def to_activity(activity_json):
-    ''' link up models and activities '''
-    activity_type = activity_json.get('type')
-    return activity_models[activity_type].to_activity(activity_json)
diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index 55036f2c..43fc4511 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -24,7 +24,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         choices=PrivacyLevels.choices
     )
     sensitive = fields.BooleanField(default=False)
-    # the created date can't be this, because of receiving federated posts
+    # created date is different than publish date because of federated posts
     published_date = fields.DateTimeField(
         default=timezone.now, activitypub_field='published')
     deleted = models.BooleanField(default=False)
@@ -53,7 +53,9 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
     def replies(cls, status):
         ''' load all replies to a status. idk if there's a better way
             to write this so it's just a property '''
-        return cls.objects.filter(reply_parent=status).select_subclasses()
+        return cls.objects.filter(
+            reply_parent=status
+        ).select_subclasses().order_by('published_date')
 
     @property
     def status_type(self):
@@ -68,7 +70,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
             **kwargs
         )
 
-    def to_activity(self, pure=False):
+    def to_activity(self, pure=False):# pylint: disable=arguments-differ
         ''' return tombstone if the status is deleted '''
         if self.deleted:
             return activitypub.Tombstone(
@@ -190,6 +192,7 @@ class Review(Status):
     def pure_name(self):
         ''' clarify review names for mastodon serialization '''
         if self.rating:
+            #pylint: disable=bad-string-format-type
             return 'Review of "%s" (%d stars): %s' % (
                 self.book.title,
                 self.rating,

From b67aea22fc707c2b06ee59a49d69aadd3794dda3 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 13 Dec 2020 11:16:12 -0800
Subject: [PATCH 096/104] Aggregates (de)serializable model fields

---
 bookwyrm/models/base_model.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index dd3065c9..d9aafcce 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -10,6 +10,8 @@ from Crypto.Hash import SHA256
 from django.core.paginator import Paginator
 from django.db import models
 from django.db.models import Q
+from django.db.models.fields.files import ImageFileDescriptor
+from django.db.models.fields.related_descriptors import ManyToManyDescriptor
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
@@ -68,6 +70,30 @@ class ActivitypubMixin:
     activity_serializer = lambda: {}
     reverse_unfurl = False
 
+    def __init__(self, *args, **kwargs):
+        ''' collect some info on model fields '''
+        self.image_fields = []
+        self.many_to_many_fields = []
+        self.simple_fields = [] # "simple"
+        for field in self._meta.get_fields():
+            if not hasattr(field, 'field_to_activity'):
+                continue
+
+            if isinstance(field, ImageFileDescriptor):
+                self.image_fields.append(field)
+            elif isinstance(field, ManyToManyDescriptor):
+                self.many_to_many_fields.append(field)
+            else:
+                self.simple_fields.append(field)
+
+        self.deserialize_reverse_fields = self.deserialize_reverse_fields \
+                if hasattr(self, 'deserialize_reverse_fields') else []
+        self.serialize_reverse_fields = self.serialize_reverse_fields \
+                if hasattr(self, 'serialize_reverse_fields') else []
+
+        super().__init__(*args, **kwargs)
+
+
     @classmethod
     def find_existing_by_remote_id(cls, remote_id):
         ''' look up a remote id in the db '''

From c470aeb3cec449d1eec4e22c4c28cb4afb19e37c Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 13 Dec 2020 12:02:26 -0800
Subject: [PATCH 097/104] Create helper function on field for settings values

---
 bookwyrm/activitypub/base_activity.py | 49 ++++++---------------------
 bookwyrm/models/base_model.py         |  8 ++---
 bookwyrm/models/fields.py             | 30 ++++++++++++++++
 3 files changed, 44 insertions(+), 43 deletions(-)

diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index ed19af99..6401bb89 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -4,8 +4,6 @@ from json import JSONEncoder
 
 from django.apps import apps
 from django.db import transaction
-from django.db.models.fields.files import ImageFileDescriptor
-from django.db.models.fields.related_descriptors import ManyToManyDescriptor
 
 from bookwyrm.connectors import ConnectorException, get_data
 from bookwyrm.tasks import app
@@ -77,55 +75,30 @@ class ActivityObject:
             )
 
         # check for an existing instance, if we're not updating a known obj
-        if not instance:
-            instance = model.find_existing(self.serialize()) or model()
+        instance = instance or model.find_existing(self.serialize()) or model()
 
-        many_to_many_fields = {}
-        image_fields = {}
-        for field in model._meta.get_fields():
-            # check if it's an activitypub field
-            if not hasattr(field, 'field_to_activity'):
-                continue
-            # call the formatter associated with the model field class
-            value = field.field_from_activity(
-                getattr(self, field.get_activitypub_field())
-            )
-            if value is None or value is MISSING:
-                continue
+        for field in instance.simple_fields:
+            field.set_field_from_activity(instance, self)
 
-            model_field = getattr(model, field.name)
-
-            if isinstance(model_field, ManyToManyDescriptor):
-                # status mentions book/users for example, stash this for later
-                many_to_many_fields[field.name] = value
-            elif isinstance(model_field, ImageFileDescriptor):
-                # image fields need custom handling
-                image_fields[field.name] = value
-            else:
-                # just a good old fashioned model.field = value
-                setattr(instance, field.name, value)
-
-        # if this isn't here, it messes up saving users. who even knows.
-        for (model_key, value) in image_fields.items():
-            getattr(instance, model_key).save(*value, save=save)
+        # image fields have to be set after other fields because they can save
+        # too early and jank up users
+        for field in instance.image_fields:
+            field.set_field_from_activity(instance, self, save=save)
 
         if not save:
-            # we can't set many to many and reverse fields on an unsaved object
             return instance
 
+        # we can't set many to many and reverse fields on an unsaved object
         instance.save()
 
         # add many to many fields, which have to be set post-save
-        for (model_key, values) in many_to_many_fields.items():
+        for field in instance.many_to_many_fields:
             # mention books/users, for example
-            getattr(instance, model_key).set(values)
-
-        if not save or not hasattr(model, 'deserialize_reverse_fields'):
-            return instance
+            field.set_field_from_activity(instance, self)
 
         # reversed relationships in the models
         for (model_field_name, activity_field_name) in \
-                model.deserialize_reverse_fields:
+                instance.deserialize_reverse_fields:
             # attachments on Status, for example
             values = getattr(self, activity_field_name)
             if values is None or values is MISSING:
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index d9aafcce..e899b3c5 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -10,13 +10,11 @@ from Crypto.Hash import SHA256
 from django.core.paginator import Paginator
 from django.db import models
 from django.db.models import Q
-from django.db.models.fields.files import ImageFileDescriptor
-from django.db.models.fields.related_descriptors import ManyToManyDescriptor
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN, PAGE_LENGTH
-from .fields import RemoteIdField
+from .fields import ImageField, ManyToManyField, RemoteIdField
 
 
 PrivacyLevels = models.TextChoices('Privacy', [
@@ -79,9 +77,9 @@ class ActivitypubMixin:
             if not hasattr(field, 'field_to_activity'):
                 continue
 
-            if isinstance(field, ImageFileDescriptor):
+            if isinstance(field, ImageField):
                 self.image_fields.append(field)
-            elif isinstance(field, ManyToManyDescriptor):
+            elif isinstance(field, ManyToManyField):
                 self.many_to_many_fields.append(field)
             else:
                 self.simple_fields.append(field)
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index e6878fb9..21bdfadb 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -1,4 +1,5 @@
 ''' activitypub-aware django model fields '''
+from dataclasses import MISSING
 import re
 from uuid import uuid4
 
@@ -38,6 +39,16 @@ class ActivitypubFieldMixin:
             self.activitypub_field = activitypub_field
         super().__init__(*args, **kwargs)
 
+
+    def set_field_from_activity(self, instance, data):
+        ''' helper function for assinging a value to the field '''
+        value = getattr(data, self.get_activitypub_field())
+        formatted = self.field_from_activity(value)
+        if formatted is None or formatted is MISSING:
+            return
+        setattr(instance, self.name, formatted)
+
+
     def field_to_activity(self, value):
         ''' formatter to convert a model value into activitypub '''
         if hasattr(self, 'activitypub_wrapper'):
@@ -145,6 +156,14 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
         self.link_only = link_only
         super().__init__(*args, **kwargs)
 
+    def set_field_from_activity(self, instance, data):
+        ''' helper function for assinging a value to the field '''
+        value = getattr(data, self.get_activitypub_field())
+        formatted = self.field_from_activity(value)
+        if formatted is None or formatted is MISSING:
+            return
+        getattr(instance, self.name).set(formatted)
+
     def field_to_activity(self, value):
         if self.link_only:
             return '%s/%s' % (value.instance.remote_id, self.name)
@@ -210,9 +229,20 @@ def image_serializer(value):
 
 class ImageField(ActivitypubFieldMixin, models.ImageField):
     ''' activitypub-aware image field '''
+    # pylint: disable=arguments-differ
+    def set_field_from_activity(self, instance, data, save=True):
+        ''' helper function for assinging a value to the field '''
+        value = getattr(data, self.get_activitypub_field())
+        formatted = self.field_from_activity(value)
+        if formatted is None or formatted is MISSING:
+            return
+        getattr(instance, self.name).save(*formatted, save=save)
+
+
     def field_to_activity(self, value):
         return image_serializer(value)
 
+
     def field_from_activity(self, value):
         image_slug = value
         # when it's an inline image (User avatar/icon, Book cover), it's a json

From b6907f39e97529211f53ee175ed657f999261ecf Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 13 Dec 2020 13:03:17 -0800
Subject: [PATCH 098/104] Creates Privacy field that handles setting to/cc

---
 bookwyrm/models/base_model.py | 25 +++------------
 bookwyrm/models/fields.py     | 60 +++++++++++++++++++++++++++++++++++
 bookwyrm/models/import_job.py |  2 +-
 bookwyrm/models/shelf.py      |  4 +--
 bookwyrm/models/status.py     | 28 ++--------------
 5 files changed, 70 insertions(+), 49 deletions(-)

diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index e899b3c5..08cc6052 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -17,13 +17,6 @@ from bookwyrm.settings import DOMAIN, PAGE_LENGTH
 from .fields import ImageField, ManyToManyField, RemoteIdField
 
 
-PrivacyLevels = models.TextChoices('Privacy', [
-    'public',
-    'unlisted',
-    'followers',
-    'direct'
-])
-
 class BookWyrmModel(models.Model):
     ''' shared fields '''
     created_date = models.DateTimeField(auto_now_add=True)
@@ -84,6 +77,9 @@ class ActivitypubMixin:
             else:
                 self.simple_fields.append(field)
 
+        self.activity_fields = self.image_fields + \
+                self.many_to_many_fields + self.simple_fields
+
         self.deserialize_reverse_fields = self.deserialize_reverse_fields \
                 if hasattr(self, 'deserialize_reverse_fields') else []
         self.serialize_reverse_fields = self.serialize_reverse_fields \
@@ -139,19 +135,8 @@ class ActivitypubMixin:
     def to_activity(self):
         ''' convert from a model to an activity '''
         activity = {}
-        for field in self._meta.get_fields():
-            if not hasattr(field, 'field_to_activity'):
-                continue
-            value = field.field_to_activity(getattr(self, field.name))
-            if value is None:
-                continue
-
-            key = field.get_activitypub_field()
-            if key in activity and isinstance(activity[key], list):
-                # handles tags on status, which accumulate across fields
-                activity[key] += value
-            else:
-                activity[key] = value
+        for field in self.activity_fields:
+            field.set_activity_from_field(activity, self)
 
         if hasattr(self, 'serialize_reverse_fields'):
             # for example, editions of a work
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 21bdfadb..05453397 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -49,6 +49,20 @@ class ActivitypubFieldMixin:
         setattr(instance, self.name, formatted)
 
 
+    def set_activity_from_field(self, activity, instance):
+        ''' update the json object '''
+        value = getattr(instance, self.name)
+        formatted = self.field_to_activity(value)
+        if formatted is None:
+            return
+
+        key = self.get_activitypub_field()
+        if isinstance(activity.get(key), list):
+            activity[key] += formatted
+        else:
+            activity[key] = value
+
+
     def field_to_activity(self, value):
         ''' formatter to convert a model value into activitypub '''
         if hasattr(self, 'activitypub_wrapper'):
@@ -134,6 +148,52 @@ class UsernameField(ActivitypubFieldMixin, models.CharField):
         return value.split('@')[0]
 
 
+PrivacyLevels = models.TextChoices('Privacy', [
+    'public',
+    'unlisted',
+    'followers',
+    'direct'
+])
+
+class PrivacyField(ActivitypubFieldMixin, models.CharField):
+    ''' this maps to two differente activitypub fields '''
+    public = 'https://www.w3.org/ns/activitystreams#Public'
+    def __init__(self, *args, **kwargs):
+        super().__init__(
+            *args, max_length=255,
+            choices=PrivacyLevels.choices, default='public')
+
+    def set_field_from_activity(self, instance, data):
+        to = data.to
+        cc = data.cc
+        if to == [self.public]:
+            setattr(instance, self.name, 'public')
+        elif cc == []:
+            setattr(instance, self.name, 'direct')
+        elif self.public in cc:
+            setattr(instance, self.name, 'unlisted')
+        else:
+            setattr(instance, self.name, 'followers')
+
+    def set_activity_from_field(self, activity, instance):
+        mentions = [u.remote_id for u in instance.mention_users.all()]
+        # this is a link to the followers list
+        followers = instance.user.__class__._meta.get_field('followers')\
+                .field_to_activity(instance.user.followers)
+        if self.privacy == 'public':
+            activity['to'] = [self.public]
+            activity['cc'] = [followers] + mentions
+        elif self.privacy == 'unlisted':
+            activity['to'] = [followers]
+            activity['cc'] = [self.public] + mentions
+        elif self.privacy == 'followers':
+            activity['to'] = [followers]
+            activity['cc'] = mentions
+        if self.privacy == 'direct':
+            activity['to'] = mentions
+            activity['cc'] = []
+
+
 class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey):
     ''' activitypub-aware foreign key field '''
     def field_to_activity(self, value):
diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py
index bf5d5caf..835094cd 100644
--- a/bookwyrm/models/import_job.py
+++ b/bookwyrm/models/import_job.py
@@ -8,7 +8,7 @@ from django.utils import timezone
 
 from bookwyrm import books_manager
 from bookwyrm.models import ReadThrough, User, Book
-from .base_model import PrivacyLevels
+from .fields import PrivacyLevels
 
 
 # Mapping goodreads -> bookwyrm shelf titles.
diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py
index fc63d198..68f3614f 100644
--- a/bookwyrm/models/shelf.py
+++ b/bookwyrm/models/shelf.py
@@ -4,7 +4,7 @@ from django.db import models
 
 from bookwyrm import activitypub
 from .base_model import BookWyrmModel
-from .base_model import OrderedCollectionMixin, PrivacyLevels
+from .base_model import OrderedCollectionMixin
 from . import fields
 
 
@@ -18,7 +18,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
     privacy = fields.CharField(
         max_length=255,
         default='public',
-        choices=PrivacyLevels.choices
+        choices=fields.PrivacyLevels.choices
     )
     books = models.ManyToManyField(
         'Edition',
diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index 43fc4511..0bf897dc 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -6,7 +6,7 @@ from model_utils.managers import InheritanceManager
 
 from bookwyrm import activitypub
 from .base_model import ActivitypubMixin, OrderedCollectionPageMixin
-from .base_model import BookWyrmModel, PrivacyLevels
+from .base_model import BookWyrmModel
 from . import fields
 from .fields import image_serializer
 
@@ -18,11 +18,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
     mention_users = fields.TagField('User', related_name='mention_user')
     mention_books = fields.TagField('Edition', related_name='mention_book')
     local = models.BooleanField(default=True)
-    privacy = models.CharField(
-        max_length=255,
-        default='public',
-        choices=PrivacyLevels.choices
-    )
+    privacy = fields.PrivacyField(max_length=255)
     sensitive = fields.BooleanField(default=False)
     # created date is different than publish date because of federated posts
     published_date = fields.DateTimeField(
@@ -48,7 +44,6 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
     serialize_reverse_fields = [('attachments', 'attachment')]
     deserialize_reverse_fields = [('attachments', 'attachment')]
 
-    #----- replies collection activitypub ----#
     @classmethod
     def replies(cls, status):
         ''' load all replies to a status. idk if there's a better way
@@ -82,25 +77,6 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         activity = ActivitypubMixin.to_activity(self)
         activity['replies'] = self.to_replies()
 
-        # privacy controls
-        public = 'https://www.w3.org/ns/activitystreams#Public'
-        mentions = [u.remote_id for u in self.mention_users.all()]
-        # this is a link to the followers list:
-        followers = self.user.__class__._meta.get_field('followers')\
-                .field_to_activity(self.user.followers)
-        if self.privacy == 'public':
-            activity['to'] = [public]
-            activity['cc'] = [followers] + mentions
-        elif self.privacy == 'unlisted':
-            activity['to'] = [followers]
-            activity['cc'] = [public] + mentions
-        elif self.privacy == 'followers':
-            activity['to'] = [followers]
-            activity['cc'] = mentions
-        if self.privacy == 'direct':
-            activity['to'] = mentions
-            activity['cc'] = []
-
         # "pure" serialization for non-bookwyrm instances
         if pure:
             activity['content'] = self.pure_content

From 44cbf7c07fd9449dca81b56e798b518fe8be034c Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 13 Dec 2020 14:35:56 -0800
Subject: [PATCH 099/104] Fixes checking privacy when serializing status

---
 bookwyrm/models/fields.py                     |   8 +-
 .../tests/activitypub/test_base_activity.py   | 113 +++++++++++++-----
 2 files changed, 84 insertions(+), 37 deletions(-)

diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 05453397..960be161 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -180,16 +180,16 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
         # this is a link to the followers list
         followers = instance.user.__class__._meta.get_field('followers')\
                 .field_to_activity(instance.user.followers)
-        if self.privacy == 'public':
+        if instance.privacy == 'public':
             activity['to'] = [self.public]
             activity['cc'] = [followers] + mentions
-        elif self.privacy == 'unlisted':
+        elif instance.privacy == 'unlisted':
             activity['to'] = [followers]
             activity['cc'] = [self.public] + mentions
-        elif self.privacy == 'followers':
+        elif instance.privacy == 'followers':
             activity['to'] = [followers]
             activity['cc'] = mentions
-        if self.privacy == 'direct':
+        if instance.privacy == 'direct':
             activity['to'] = mentions
             activity['cc'] = []
 
diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py
index 88997c44..87420aa7 100644
--- a/bookwyrm/tests/activitypub/test_base_activity.py
+++ b/bookwyrm/tests/activitypub/test_base_activity.py
@@ -95,34 +95,67 @@ class BaseActivity(TestCase):
         self.assertEqual(result.remote_id, 'https://example.com/user/mouse')
         self.assertEqual(result.name, 'MOUSE?? MOUSE!!')
 
-    def test_to_model(self):
-        ''' the big boy of this module. it feels janky to test this with actual
-        models rather than a test model, but I don't know how to make a test
-        model so here we are. '''
+    def test_to_model_invalid_model(self):
+        ''' catch mismatch between activity type and model type '''
         instance = ActivityObject(id='a', type='b')
         with self.assertRaises(ActivitySerializerError):
             instance.to_model(models.User)
 
-        # test setting simple fields
+    def test_to_model_simple_fields(self):
+        ''' test setting simple fields '''
         self.assertEqual(self.user.name, '')
-        update_data = activitypub.Person(**self.user.to_activity())
-        update_data.name = 'New Name'
-        update_data.to_model(models.User, self.user)
+
+        activity = activitypub.Person(
+            id=self.user.remote_id,
+            name='New Name',
+            preferredUsername='mouse',
+            inbox='http://www.com/',
+            outbox='http://www.com/',
+            followers='',
+            summary='',
+            publicKey=None,
+            endpoints={},
+        )
+
+        activity.to_model(models.User, self.user)
 
         self.assertEqual(self.user.name, 'New Name')
 
     def test_to_model_foreign_key(self):
         ''' test setting one to one/foreign key '''
-        update_data = activitypub.Person(**self.user.to_activity())
-        update_data.publicKey['publicKeyPem'] = 'hi im secure'
-        update_data.to_model(models.User, self.user)
+        activity = activitypub.Person(
+            id=self.user.remote_id,
+            name='New Name',
+            preferredUsername='mouse',
+            inbox='http://www.com/',
+            outbox='http://www.com/',
+            followers='',
+            summary='',
+            publicKey=self.user.key_pair.to_activity(),
+            endpoints={},
+        )
+
+        activity.publicKey['publicKeyPem'] = 'hi im secure'
+
+        activity.to_model(models.User, self.user)
         self.assertEqual(self.user.key_pair.public_key, 'hi im secure')
 
     @responses.activate
     def test_to_model_image(self):
         ''' update an image field '''
-        update_data = activitypub.Person(**self.user.to_activity())
-        update_data.icon = {'url': 'http://www.example.com/image.jpg'}
+        activity = activitypub.Person(
+            id=self.user.remote_id,
+            name='New Name',
+            preferredUsername='mouse',
+            inbox='http://www.com/',
+            outbox='http://www.com/',
+            followers='',
+            summary='',
+            publicKey=None,
+            endpoints={},
+            icon={'url': 'http://www.example.com/image.jpg'}
+        )
+
         responses.add(
             responses.GET,
             'http://www.example.com/image.jpg',
@@ -133,7 +166,7 @@ class BaseActivity(TestCase):
         with self.assertRaises(ValueError):
             self.user.avatar.file #pylint: disable=pointless-statement
 
-        update_data.to_model(models.User, self.user)
+        activity.to_model(models.User, self.user)
         self.assertIsNotNone(self.user.avatar.name)
         self.assertIsNotNone(self.user.avatar.file)
 
@@ -145,19 +178,26 @@ class BaseActivity(TestCase):
         )
         book = models.Edition.objects.create(
             title='Test Edition', remote_id='http://book.com/book')
-        update_data = activitypub.Note(**status.to_activity())
-        update_data.tag = [
-            {
-                'type': 'Mention',
-                'name': 'gerald',
-                'href': 'http://example.com/a/b'
-            },
-            {
-                'type': 'Edition',
-                'name': 'gerald j. books',
-                'href': 'http://book.com/book'
-            },
-        ]
+        update_data = activitypub.Note(
+            id=status.remote_id,
+            content=status.content,
+            attributedTo=self.user.remote_id,
+            published='hi',
+            to=[],
+            cc=[],
+            tag=[
+                {
+                    'type': 'Mention',
+                    'name': 'gerald',
+                    'href': 'http://example.com/a/b'
+                },
+                {
+                    'type': 'Edition',
+                    'name': 'gerald j. books',
+                    'href': 'http://book.com/book'
+                },
+            ]
+        )
         update_data.to_model(models.Status, instance=status)
         self.assertEqual(status.mention_users.first(), self.user)
         self.assertEqual(status.mention_books.first(), book)
@@ -171,12 +211,19 @@ class BaseActivity(TestCase):
             content='test status',
             user=self.user,
         )
-        update_data = activitypub.Note(**status.to_activity())
-        update_data.attachment = [{
-            'url': 'http://www.example.com/image.jpg',
-            'name': 'alt text',
-            'type': 'Image',
-        }]
+        update_data = activitypub.Note(
+            id=status.remote_id,
+            content=status.content,
+            attributedTo=self.user.remote_id,
+            published='hi',
+            to=[],
+            cc=[],
+            attachment=[{
+                'url': 'http://www.example.com/image.jpg',
+                'name': 'alt text',
+                'type': 'Image',
+            }],
+        )
 
         responses.add(
             responses.GET,

From 5c7ac4611642fff72233e9db7ca92a847940e047 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 13 Dec 2020 14:53:25 -0800
Subject: [PATCH 100/104] Fixes foreign key field setting wrong value on
 activity

---
 bookwyrm/models/fields.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 960be161..34f90103 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -60,7 +60,7 @@ class ActivitypubFieldMixin:
         if isinstance(activity.get(key), list):
             activity[key] += formatted
         else:
-            activity[key] = value
+            activity[key] = formatted
 
 
     def field_to_activity(self, value):

From c75f5a159827170d082ccdb9623a3e0a244ea03d Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 13 Dec 2020 15:43:39 -0800
Subject: [PATCH 101/104] Unit tests for privacy model field

---
 bookwyrm/tests/models/test_fields.py | 98 +++++++++++++++++++++++++++-
 1 file changed, 97 insertions(+), 1 deletion(-)

diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index a1e4ff71..b0d67263 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -1,9 +1,11 @@
 ''' testing models '''
 from io import BytesIO
 from collections import namedtuple
+from dataclasses import dataclass
 import json
 import pathlib
 import re
+from typing import List
 from unittest.mock import patch
 
 from PIL import Image
@@ -15,7 +17,9 @@ from django.db import models
 from django.test import TestCase
 from django.utils import timezone
 
-from bookwyrm.models import fields, User
+from bookwyrm.activitypub.base_activity import ActivityObject
+from bookwyrm.models import fields, User, Status
+from bookwyrm.models.base_model import ActivitypubMixin, BookWyrmModel
 
 class ActivitypubFields(TestCase):
     ''' overwrites standard model feilds to work with activitypub '''
@@ -90,6 +94,97 @@ class ActivitypubFields(TestCase):
 
         self.assertEqual(instance.field_to_activity('test@example.com'), 'test')
 
+
+    def test_privacy_field_defaults(self):
+        ''' post privacy field's many default values '''
+        instance = fields.PrivacyField()
+        self.assertEqual(instance.max_length, 255)
+        self.assertEqual(
+            [c[0] for c in instance.choices],
+            ['public', 'unlisted', 'followers', 'direct'])
+        self.assertEqual(instance.default, 'public')
+        self.assertEqual(
+            instance.public, 'https://www.w3.org/ns/activitystreams#Public')
+
+    def test_privacy_field_set_field_from_activity(self):
+        ''' translate between to/cc fields and privacy '''
+        @dataclass(init=False)
+        class TestActivity(ActivityObject):
+            ''' real simple mock '''
+            to: List[str]
+            cc: List[str]
+            id: str = 'http://hi.com'
+            type: str = 'Test'
+
+        class TestModel(ActivitypubMixin, BookWyrmModel):
+            ''' real simple mock model because BookWyrmModel is abstract '''
+            privacy_field = fields.PrivacyField()
+            mention_users = fields.TagField(User)
+            user = fields.ForeignKey(User, on_delete=models.CASCADE)
+
+        public = 'https://www.w3.org/ns/activitystreams#Public'
+        data = TestActivity(
+            to=[public],
+            cc=['bleh'],
+        )
+        model_instance = TestModel(privacy_field='direct')
+        self.assertEqual(model_instance.privacy_field, 'direct')
+
+        instance = fields.PrivacyField()
+        instance.name = 'privacy_field'
+        instance.set_field_from_activity(model_instance, data)
+        self.assertEqual(model_instance.privacy_field, 'public')
+
+        data.to = ['bleh']
+        data.cc = []
+        instance.set_field_from_activity(model_instance, data)
+        self.assertEqual(model_instance.privacy_field, 'direct')
+
+        data.to = ['bleh']
+        data.cc = [public, 'waah']
+        instance.set_field_from_activity(model_instance, data)
+        self.assertEqual(model_instance.privacy_field, 'unlisted')
+
+
+    def test_privacy_field_set_activity_from_field(self):
+        ''' translate between to/cc fields and privacy '''
+        user = User.objects.create_user(
+            'rat', 'rat@rat.rat', 'ratword', local=True)
+        public = 'https://www.w3.org/ns/activitystreams#Public'
+        followers = '%s/followers' % user.remote_id
+
+        instance = fields.PrivacyField()
+        instance.name = 'privacy_field'
+
+        model_instance = Status.objects.create(user=user, content='hi')
+        activity = {}
+        instance.set_activity_from_field(activity, model_instance)
+        self.assertEqual(activity['to'], [public])
+        self.assertEqual(activity['cc'], [followers])
+
+        model_instance = Status.objects.create(user=user, privacy='unlisted')
+        activity = {}
+        instance.set_activity_from_field(activity, model_instance)
+        self.assertEqual(activity['to'], [followers])
+        self.assertEqual(activity['cc'], [public])
+
+        model_instance = Status.objects.create(user=user, privacy='followers')
+        activity = {}
+        instance.set_activity_from_field(activity, model_instance)
+        self.assertEqual(activity['to'], [followers])
+        self.assertEqual(activity['cc'], [])
+
+        model_instance = Status.objects.create(
+            user=user,
+            privacy='direct',
+        )
+        model_instance.mention_users.set([user])
+        activity = {}
+        instance.set_activity_from_field(activity, model_instance)
+        self.assertEqual(activity['to'], [user.remote_id])
+        self.assertEqual(activity['cc'], [])
+
+
     def test_foreign_key(self):
         ''' should be able to format a related model '''
         instance = fields.ForeignKey('User', on_delete=models.CASCADE)
@@ -98,6 +193,7 @@ class ActivitypubFields(TestCase):
         # returns the remote_id field of the related object
         self.assertEqual(instance.field_to_activity(item), 'https://e.b/c')
 
+
     @responses.activate
     def test_foreign_key_from_activity_str(self):
         ''' create a new object from a foreign key '''

From 4fcdbe5299a21c23bca82b9648b9a920d728db81 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 13 Dec 2020 15:56:30 -0800
Subject: [PATCH 102/104] Fixes clashing test model name

---
 bookwyrm/tests/models/test_fields.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py
index b0d67263..8c86b23c 100644
--- a/bookwyrm/tests/models/test_fields.py
+++ b/bookwyrm/tests/models/test_fields.py
@@ -116,7 +116,7 @@ class ActivitypubFields(TestCase):
             id: str = 'http://hi.com'
             type: str = 'Test'
 
-        class TestModel(ActivitypubMixin, BookWyrmModel):
+        class TestPrivacyModel(ActivitypubMixin, BookWyrmModel):
             ''' real simple mock model because BookWyrmModel is abstract '''
             privacy_field = fields.PrivacyField()
             mention_users = fields.TagField(User)
@@ -127,7 +127,7 @@ class ActivitypubFields(TestCase):
             to=[public],
             cc=['bleh'],
         )
-        model_instance = TestModel(privacy_field='direct')
+        model_instance = TestPrivacyModel(privacy_field='direct')
         self.assertEqual(model_instance.privacy_field, 'direct')
 
         instance = fields.PrivacyField()

From 943d97c0bc791efd0c62c0e08525aedae2be3734 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 13 Dec 2020 16:16:12 -0800
Subject: [PATCH 103/104] Adds direct messages UI

---
 bookwyrm/templates/direct_messages.html | 37 +++++++++++++++++++++++++
 bookwyrm/templates/layout.html          |  3 ++
 bookwyrm/urls.py                        |  3 +-
 bookwyrm/views.py                       | 37 ++++++++++++++++++++++++-
 4 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 bookwyrm/templates/direct_messages.html

diff --git a/bookwyrm/templates/direct_messages.html b/bookwyrm/templates/direct_messages.html
new file mode 100644
index 00000000..6a20b111
--- /dev/null
+++ b/bookwyrm/templates/direct_messages.html
@@ -0,0 +1,37 @@
+{% extends 'layout.html' %}
+{% block content %}
+
+<div class="block">
+    <h1 class="title">Direct Messages</h1>
+
+    {% if not activities %}
+    <p>You have no messages right now.</p>
+    {% endif %}
+    {% for activity in activities %}
+    <div class="block">
+    {% include 'snippets/status.html' with status=activity %}
+    </div>
+    {% endfor %}
+
+    <nav class="pagination" role="navigation" aria-label="pagination">
+        {% if prev %}
+        <p class="pagination-previous">
+            <a href="{{ prev }}">
+                <span class="icon icon-arrow-left"></span>
+                Previous
+            </a>
+        </p>
+        {% endif %}
+
+        {% if next %}
+        <p class="pagination-next">
+            <a href="{{ next }}">
+                Next
+                <span class="icon icon-arrow-right"></span>
+            </a>
+        </p>
+        {% endif %}
+    </nav>
+</div>
+
+{% endblock %}
diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html
index bcbdca2a..ab113ad0 100644
--- a/bookwyrm/templates/layout.html
+++ b/bookwyrm/templates/layout.html
@@ -68,6 +68,9 @@
                     {% include 'snippets/username.html' with user=request.user %}
                 </p></div>
                 <div class="navbar-dropdown">
+                    <a href="/direct-messages" class="navbar-item">
+                        Direct messages
+                    </a>
                     <a href="/user/{{request.user.localname}}" class="navbar-item">
                         Profile
                     </a>
diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py
index a9792038..3cbb7510 100644
--- a/bookwyrm/urls.py
+++ b/bookwyrm/urls.py
@@ -53,7 +53,8 @@ urlpatterns = [
 
     path('', views.home),
     re_path(r'^(?P<tab>home|local|federated)/?$', views.home_tab),
-    re_path(r'^notifications/?', views.notifications_page),
+    re_path(r'^notifications/?$', views.notifications_page),
+    re_path(r'^direct-messages/?$', views.direct_messages_page),
     re_path(r'^import/?$', views.import_page),
     re_path(r'^import-status/(\d+)/?$', views.import_status),
     re_path(r'^user-edit/?$', views.edit_profile_page),
diff --git a/bookwyrm/views.py b/bookwyrm/views.py
index 562f575e..f32638d4 100644
--- a/bookwyrm/views.py
+++ b/bookwyrm/views.py
@@ -113,11 +113,36 @@ def get_suggested_books(user, max_books=5):
     return suggested_books
 
 
+@login_required
+@require_GET
+def direct_messages_page(request, page=1):
+    ''' like a feed but for dms only '''
+    activities = get_activity_feed(request.user, 'direct')
+    paginated = Paginator(activities, PAGE_LENGTH)
+    activity_page = paginated.page(page)
+
+    prev_page = next_page = None
+    if activity_page.has_next():
+        next_page = '/direct-message/?page=%d#feed' % \
+                activity_page.next_page_number()
+    if activity_page.has_previous():
+        prev_page = '/direct-messages/?page=%d#feed' % \
+                activity_page.previous_page_number()
+    data = {
+        'title': 'Direct Messages',
+        'user': request.user,
+        'activities': activity_page.object_list,
+        'next': next_page,
+        'prev': prev_page,
+    }
+    return TemplateResponse(request, 'direct_messages.html', data)
+
+
 def get_activity_feed(user, filter_level, model=models.Status):
     ''' get a filtered queryset of statuses '''
-    # status updates for your follow network
     if user.is_anonymous:
         user = None
+
     if user:
         following = models.User.objects.filter(
             Q(followers=user) | Q(id=user.id)
@@ -135,6 +160,16 @@ def get_activity_feed(user, filter_level, model=models.Status):
         '-published_date'
     )
 
+    if filter_level == 'direct':
+        return activities.filter(
+            Q(user=user) | Q(mention_users=user),
+            privacy='direct'
+        )
+
+    # never show DMs in the regular feed
+    activities = activities.filter(~Q(privacy='direct'))
+
+
     if hasattr(activities, 'select_subclasses'):
         activities = activities.select_subclasses()
 

From 693dfc42ac2affeb7535a1300dd8925276987581 Mon Sep 17 00:00:00 2001
From: Mouse Reeve <mousereeve@riseup.net>
Date: Sun, 13 Dec 2020 16:20:59 -0800
Subject: [PATCH 104/104] Match privacy level in reply

---
 bookwyrm/templates/snippets/reply_form.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bookwyrm/templates/snippets/reply_form.html b/bookwyrm/templates/snippets/reply_form.html
index 40cf6de1..d0a0f6b9 100644
--- a/bookwyrm/templates/snippets/reply_form.html
+++ b/bookwyrm/templates/snippets/reply_form.html
@@ -13,7 +13,7 @@
 
         <div class="column is-narrow">
             <div class="field">
-                {% include 'snippets/privacy_select.html' %}
+                {% include 'snippets/privacy_select.html' with current=activity.privacy %}
             </div>
             <div class="field">
                 <button class="button is-primary" type="submit">