Merge branch 'main' into switch-edition

README.md

@@ -60,8 +60,6 @@ cp .env.example .env
 
 For most testing, you'll want to use ngrok. Remember to set the DOMAIN in `.env` to your ngrok domain.
 
-
-#### With Docker
 You'll have to install the Docker and docker-compose. When you're ready, run:
 
 ```bash
@@ -70,33 +68,7 @@ docker-compose run --rm web python manage.py migrate
 docker-compose run --rm web python manage.py initdb
 ```
 
-### Without Docker
-You will need postgres installed and running on your computer.
-
-``` bash
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements.txt
-createdb bookwyrm
-```
-
-Create the psql user in `psql bookwyrm`:
-``` psql
-CREATE ROLE bookwyrm WITH LOGIN PASSWORD 'bookwyrm';
-GRANT ALL PRIVILEGES ON DATABASE bookwyrm TO bookwyrm;
-```
-
-Initialize the database (or, more specifically, delete the existing database, run migrations, and start fresh):
-``` bash
-./rebuilddb.sh
-```
-This creates two users, `mouse` with password `password123` and `rat` with password `ratword`.
-
-The application uses Celery and Redis for task management, which must also be installed and configured.
-
-And go to the app at `localhost:8000`
-
-
+Once the build is complete, you can access the instance at `localhost:1333`
 
 ## Installing in Production
 

bookwyrm/activitypub/__init__.py

@@ -2,20 +2,19 @@
 import inspect
 import sys
 
-from .base_activity import ActivityEncoder, PublicKey, Signature
+from .base_activity import ActivityEncoder, Signature
 from .base_activity import Link, Mention
-from .base_activity import ActivitySerializerError
-from .base_activity import tag_formatter
+from .base_activity import ActivitySerializerError, resolve_remote_id
 from .image import Image
 from .note import Note, GeneratedNote, Article, Comment, Review, Quotation
 from .note import Tombstone
 from .interaction import Boost, Like
 from .ordered_collection import OrderedCollection, OrderedCollectionPage
-from .person import Person
+from .person import Person, PublicKey
 from .book import Edition, Work, Author
 from .verbs import Create, Delete, Undo, Update
 from .verbs import Follow, Accept, Reject
-from .verbs import Add, Remove
+from .verbs import Add, AddBook, Remove
 
 # this creates a list of all the Activity types that we can serialize,
 # so when an Activity comes in from outside, we can check if it's known

bookwyrm/activitypub/base_activity.py

@@ -1,18 +1,12 @@
 ''' basics for an activitypub serializer '''
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
-from uuid import uuid4
 
-from django.core.files.base import ContentFile
+from django.apps import apps
 from django.db import transaction
-from django.db.models.fields.related_descriptors \
-        import ForwardManyToOneDescriptor, ManyToManyDescriptor, \
-        ReverseManyToOneDescriptor
-from django.db.models.fields.files import ImageFileDescriptor
-import requests
-
-from bookwyrm import books_manager, models
 
+from bookwyrm.connectors import ConnectorException, get_data
+from bookwyrm.tasks import app
 
 class ActivitySerializerError(ValueError):
     ''' routine problems serializing activitypub json '''
@@ -25,26 +19,19 @@ class ActivityEncoder(JSONEncoder):
 
 
 @dataclass
-class Link():
+class Link:
     ''' for tagging a book in a status '''
     href: str
     name: str
     type: str = 'Link'
 
+
 @dataclass
 class Mention(Link):
     ''' a subtype of Link for mentioning an actor '''
     type: str = 'Mention'
 
 
-@dataclass
-class PublicKey:
-    ''' public key block '''
-    id: str
-    owner: str
-    publicKeyPem: str
-
-
 @dataclass
 class Signature:
     ''' public key block '''
@@ -76,88 +63,63 @@ class ActivityObject:
             setattr(self, field.name, value)
 
 
-    def to_model(self, model, instance=None):
+    @transaction.atomic
+    def to_model(self, model, instance=None, save=True):
         ''' convert from an activity to a model instance '''
         if not isinstance(self, model.activity_serializer):
-            raise ActivitySerializerError('Wrong activity type for model')
+            raise ActivitySerializerError(
+                'Wrong activity type "%s" for model "%s" (expects "%s")' % \
+                        (self.__class__,
+                         model.__name__,
+                         model.activity_serializer)
+            )
 
         # check for an existing instance, if we're not updating a known obj
-        if not instance:
-            try:
-                return model.objects.get(remote_id=self.id)
-            except model.DoesNotExist:
-                pass
+        instance = instance or model.find_existing(self.serialize()) or model()
 
-        model_fields = [m.name for m in model._meta.get_fields()]
-        mapped_fields = {}
-        many_to_many_fields = {}
-        one_to_many_fields = {}
-        image_fields = {}
+        for field in instance.simple_fields:
+            field.set_field_from_activity(instance, self)
 
-        for mapping in model.activity_mappings:
-            if mapping.model_key not in model_fields:
-                continue
-            # value is None if there's a default that isn't supplied
-            # in the activity but is supplied in the formatter
-            value = None
-            if mapping.activity_key:
-                value = getattr(self, mapping.activity_key)
-            model_field = getattr(model, mapping.model_key)
+        # image fields have to be set after other fields because they can save
+        # too early and jank up users
+        for field in instance.image_fields:
+            field.set_field_from_activity(instance, self, save=save)
 
-            formatted_value = mapping.model_formatter(value)
-            if isinstance(model_field, ForwardManyToOneDescriptor) and \
-                    formatted_value:
-                # foreign key remote id reolver (work on Edition, for example)
-                fk_model = model_field.field.related_model
-                reference = resolve_foreign_key(fk_model, formatted_value)
-                mapped_fields[mapping.model_key] = reference
-            elif isinstance(model_field, ManyToManyDescriptor):
-                # status mentions book/users
-                many_to_many_fields[mapping.model_key] = formatted_value
-            elif isinstance(model_field, ReverseManyToOneDescriptor):
-                # attachments on Status, for example
-                one_to_many_fields[mapping.model_key] = formatted_value
-            elif isinstance(model_field, ImageFileDescriptor):
-                # image fields need custom handling
-                image_fields[mapping.model_key] = formatted_value
-            else:
-                mapped_fields[mapping.model_key] = formatted_value
+        if not save:
+            return instance
 
-        with transaction.atomic():
-            if instance:
-                # updating an existing model isntance
-                for k, v in mapped_fields.items():
-                    setattr(instance, k, v)
+        # we can't set many to many and reverse fields on an unsaved object
         instance.save()
-            else:
-                # creating a new model instance
-                instance = model.objects.create(**mapped_fields)
 
-            # add images
-            for (model_key, value) in image_fields.items():
-                formatted_value = image_formatter(value)
-                if not formatted_value:
+        # add many to many fields, which have to be set post-save
+        for field in instance.many_to_many_fields:
+            # mention books/users, for example
+            field.set_field_from_activity(instance, self)
+
+        # reversed relationships in the models
+        for (model_field_name, activity_field_name) in \
+                instance.deserialize_reverse_fields:
+            # attachments on Status, for example
+            values = getattr(self, activity_field_name)
+            if values is None or values is MISSING:
                 continue
-                getattr(instance, model_key).save(*formatted_value, save=True)
+            try:
+                # this is for one to many
+                related_model = getattr(model, model_field_name).field.model
+            except AttributeError:
+                # it's a one to one or foreign key
+                related_model = getattr(model, model_field_name)\
+                        .related.related_model
+                values = [values]
 
-            for (model_key, values) in many_to_many_fields.items():
-                # mention books, mention users
-                getattr(instance, model_key).set(values)
-
-            # add one to many fields
-            for (model_key, values) in one_to_many_fields.items():
-                if values == MISSING:
-                    continue
-                model_field = getattr(instance, model_key)
-                model = model_field.model
             for item in values:
-                    item = model.activity_serializer(**item)
-                    field_name = instance.__class__.__name__.lower()
-                    with transaction.atomic():
-                        item = item.to_model(model)
-                        setattr(item, field_name, instance)
-                        item.save()
-
+                set_related_field.delay(
+                    related_model.__name__,
+                    instance.__class__.__name__,
+                    instance.__class__.__name__.lower(),
+                    instance.remote_id,
+                    item
+                )
         return instance
 
 
@@ -168,66 +130,57 @@ class ActivityObject:
         return data
 
 
-def resolve_foreign_key(model, remote_id):
-    ''' look up the remote_id on an activity json field '''
-    if model in [models.Edition, models.Work, models.Book]:
-        return books_manager.get_or_create_book(remote_id)
+@app.task
+@transaction.atomic
+def set_related_field(
+        model_name, origin_model_name,
+        related_field_name, related_remote_id, data):
+    ''' load reverse related fields (editions, attachments) without blocking '''
+    model = apps.get_model('bookwyrm.%s' % model_name, require_ready=True)
+    origin_model = apps.get_model(
+        'bookwyrm.%s' % origin_model_name,
+        require_ready=True
+    )
 
-    result = model.objects
-    if hasattr(model.objects, 'select_subclasses'):
-        result = result.select_subclasses()
+    if isinstance(data, str):
+        item = resolve_remote_id(model, data, save=False)
+    else:
+        # look for a match based on all the available data
+        item = model.find_existing(data)
+        if not item:
+            # create a new model instance
+            item = model.activity_serializer(**data)
+            item = item.to_model(model, save=False)
+    # this must exist because it's the object that triggered this function
+    instance = origin_model.find_existing_by_remote_id(related_remote_id)
+    if not instance:
+        raise ValueError('Invalid related remote id: %s' % related_remote_id)
 
-    result = result.filter(
-        remote_id=remote_id
-    ).first()
+    # edition.parent_work = instance, for example
+    setattr(item, related_field_name, instance)
+    item.save()
 
-    if not result:
-        raise ActivitySerializerError(
-            'Could not resolve remote_id in %s model: %s' % \
-                (model.__name__, remote_id))
+
+def resolve_remote_id(model, remote_id, refresh=False, save=True):
+    ''' take a remote_id and return an instance, creating if necessary '''
+    result = model.find_existing_by_remote_id(remote_id)
+    if result and not refresh:
         return result
 
-
-def tag_formatter(tags, tag_type):
-    ''' helper function to extract foreign keys from tag activity json '''
-    if not isinstance(tags, list):
-        return []
-    items = []
-    types = {
-        'Book': models.Book,
-        'Mention': models.User,
-    }
-    for tag in [t for t in tags if t.get('type') == tag_type]:
-        if not tag_type in types:
-            continue
-        remote_id = tag.get('href')
+    # load the data and create the object
     try:
-            item = resolve_foreign_key(types[tag_type], remote_id)
-        except ActivitySerializerError:
-            continue
-        items.append(item)
-    return items
+        data = get_data(remote_id)
+    except (ConnectorException, ConnectionError):
+        raise ActivitySerializerError(
+            'Could not connect to host for remote_id in %s model: %s' % \
+                (model.__name__, remote_id))
 
+    # check for existing items with shared unique identifiers
+    if not result:
+        result = model.find_existing(data)
+        if result and not refresh:
+            return result
 
-def image_formatter(image_slug):
-    ''' helper function to load images and format them for a model '''
-    # when it's an inline image (User avatar/icon, Book cover), it's a json
-    # blob, but when it's an attached image, it's just a url
-    if isinstance(image_slug, dict):
-        url = image_slug.get('url')
-    elif isinstance(image_slug, str):
-        url = image_slug
-    else:
-        return None
-    if not url:
-        return None
-    try:
-        response = requests.get(url)
-    except ConnectionError:
-        return None
-    if not response.ok:
-        return None
-
-    image_name = str(uuid4()) + '.' + url.split('.')[-1]
-    image_content = ContentFile(response.content)
-    return [image_name, image_content]
+    item = model.activity_serializer(**data)
+    # if we're refreshing, "result" will be set and we'll update it
+    return item.to_model(model, instance=result, save=save)

bookwyrm/activitypub/book.py

@@ -12,13 +12,13 @@ class Book(ActivityObject):
     sortTitle: str = ''
     subtitle: str = ''
     description: str = ''
-    languages: List[str]
+    languages: List[str] = field(default_factory=lambda: [])
     series: str = ''
     seriesNumber: str = ''
-    subjects: List[str]
-    subjectPlaces: List[str]
+    subjects: List[str] = field(default_factory=lambda: [])
+    subjectPlaces: List[str] = field(default_factory=lambda: [])
 
-    authors: List[str]
+    authors: List[str] = field(default_factory=lambda: [])
     firstPublishedDate: str = ''
     publishedDate: str = ''
 
@@ -33,22 +33,23 @@ class Book(ActivityObject):
 @dataclass(init=False)
 class Edition(Book):
     ''' Edition instance of a book object '''
-    isbn10: str
-    isbn13: str
-    oclcNumber: str
-    asin: str
-    pages: str
-    physicalFormat: str
-    publishers: List[str]
-
     work: str
+    isbn10: str = ''
+    isbn13: str = ''
+    oclcNumber: str = ''
+    asin: str = ''
+    pages: str = ''
+    physicalFormat: str = ''
+    publishers: List[str] = field(default_factory=lambda: [])
+
     type: str = 'Edition'
 
 
 @dataclass(init=False)
 class Work(Book):
     ''' work instance of a book object '''
-    lccn: str
+    lccn: str = ''
+    defaultEdition: str = ''
     editions: List[str]
     type: str = 'Work'
 

bookwyrm/activitypub/note.py

@@ -8,7 +8,6 @@ from .image import Image
 @dataclass(init=False)
 class Tombstone(ActivityObject):
     ''' the placeholder for a deleted status '''
-    url: str
     published: str
     deleted: str
     type: str = 'Tombstone'
@@ -17,14 +16,13 @@ class Tombstone(ActivityObject):
 @dataclass(init=False)
 class Note(ActivityObject):
     ''' Note activity '''
-    url: str
-    inReplyTo: str
     published: str
     attributedTo: str
-    to: List[str]
-    cc: List[str]
     content: str
-    replies: Dict
+    to: List[str] = field(default_factory=lambda: [])
+    cc: List[str] = field(default_factory=lambda: [])
+    replies: Dict = field(default_factory=lambda: {})
+    inReplyTo: str = ''
     tag: List[Link] = field(default_factory=lambda: [])
     attachment: List[Image] = field(default_factory=lambda: [])
     sensitive: bool = False

bookwyrm/activitypub/ordered_collection.py

@@ -12,6 +12,7 @@ class OrderedCollection(ActivityObject):
     first: str
     last: str = ''
     name: str = ''
+    owner: str = ''
     type: str = 'OrderedCollection'
 
 

bookwyrm/activitypub/person.py

@@ -2,9 +2,18 @@
 from dataclasses import dataclass, field
 from typing import Dict
 
-from .base_activity import ActivityObject, PublicKey
+from .base_activity import ActivityObject
 from .image import Image
 
+
+@dataclass(init=False)
+class PublicKey(ActivityObject):
+    ''' public key block '''
+    owner: str
+    publicKeyPem: str
+    type: str = 'PublicKey'
+
+
 @dataclass(init=False)
 class Person(ActivityObject):
     ''' actor activitypub json '''

bookwyrm/activitypub/verbs.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from typing import List
 
 from .base_activity import ActivityObject, Signature
+from .book import Book
 
 @dataclass(init=False)
 class Verb(ActivityObject):
@@ -69,6 +70,13 @@ class Add(Verb):
     type: str = 'Add'
 
 
+@dataclass(init=False)
+class AddBook(Verb):
+    '''Add activity that's aware of the book obj '''
+    target: Book
+    type: str = 'Add'
+
+
 @dataclass(init=False)
 class Remove(Verb):
     '''Remove activity '''

bookwyrm/books_manager.py

@@ -16,23 +16,6 @@ def get_edition(book_id):
     return book
 
 
-def get_or_create_book(remote_id):
-    ''' pull up a book record by whatever means possible '''
-    book = models.Book.objects.select_subclasses().filter(
-        remote_id=remote_id
-    ).first()
-    if book:
-        return book
-
-    connector = get_or_create_connector(remote_id)
-
-    # raises ConnectorException
-    book = connector.get_or_create_book(remote_id)
-    if book:
-        load_more_data.delay(book.id)
-    return book
-
-
 def get_or_create_connector(remote_id):
     ''' get the connector related to the author's server '''
     url = urlparse(remote_id)
@@ -102,12 +85,6 @@ def first_search_result(query, min_confidence=0.1):
     return None
 
 
-def update_book(book, data=None):
-    ''' re-sync with the original data source '''
-    connector = load_connector(book.connector)
-    connector.update_book(book, data=data)
-
-
 def get_connectors():
     ''' load all connectors '''
     for info in models.Connector.objects.order_by('priority').all():

bookwyrm/broadcast.py

@@ -65,7 +65,7 @@ def sign_and_send(sender, data, destination):
     ''' crpyto whatever and http junk '''
     now = http_date()
 
-    if not sender.private_key:
+    if not sender.key_pair.private_key:
         # this shouldn't happen. it would be bad if it happened.
         raise ValueError('No private key found for sender')
 

bookwyrm/connectors/__init__.py

@@ -1,3 +1,4 @@
 ''' bring connectors into the namespace '''
 from .settings import CONNECTORS
 from .abstract_connector import ConnectorException
+from .abstract_connector import get_data, get_image

bookwyrm/connectors/abstract_connector.py

@@ -8,6 +8,7 @@ from django.db import transaction
 from dateutil import parser
 import requests
 from requests import HTTPError
+from requests.exceptions import SSLError
 
 from bookwyrm import models
 
@@ -16,20 +17,13 @@ class ConnectorException(HTTPError):
     ''' when the connector can't do what was asked '''
 
 
-class AbstractConnector(ABC):
-    ''' generic book data connector '''
-
+class AbstractMinimalConnector(ABC):
+    ''' just the bare bones, for other bookwyrm instances '''
     def __init__(self, identifier):
         # load connector settings
         info = models.Connector.objects.get(identifier=identifier)
         self.connector = info
 
-        self.key_mappings = []
-
-        # fields we want to look for in book data to copy over
-        # title we handle separately.
-        self.book_mappings = []
-
         # the things in the connector model to copy over
         self_fields = [
             'base_url',
@@ -44,15 +38,6 @@ class AbstractConnector(ABC):
         for field in self_fields:
             setattr(self, field, getattr(info, field))
 
-
-    def is_available(self):
-        ''' check if you're allowed to use this connector '''
-        if self.max_query_count is not None:
-            if self.connector.query_count >= self.max_query_count:
-                return False
-        return True
-
-
     def search(self, query, min_confidence=None):
         ''' free text search '''
         resp = requests.get(
@@ -70,9 +55,40 @@ class AbstractConnector(ABC):
             results.append(self.format_search_result(doc))
         return results
 
-
+    @abstractmethod
     def get_or_create_book(self, remote_id):
         ''' pull up a book record by whatever means possible '''
+
+    @abstractmethod
+    def parse_search_data(self, data):
+        ''' turn the result json from a search into a list '''
+
+    @abstractmethod
+    def format_search_result(self, search_result):
+        ''' create a SearchResult obj from json '''
+
+
+class AbstractConnector(AbstractMinimalConnector):
+    ''' generic book data connector '''
+    def __init__(self, identifier):
+        super().__init__(identifier)
+
+        self.key_mappings = []
+
+        # fields we want to look for in book data to copy over
+        # title we handle separately.
+        self.book_mappings = []
+
+
+    def is_available(self):
+        ''' check if you're allowed to use this connector '''
+        if self.max_query_count is not None:
+            if self.connector.query_count >= self.max_query_count:
+                return False
+        return True
+
+
+    def get_or_create_book(self, remote_id):
         # try to load the book
         book = models.Book.objects.select_subclasses().filter(
             origin_id=remote_id
@@ -157,13 +173,12 @@ class AbstractConnector(ABC):
 
     def update_book_from_data(self, book, data, update_cover=True):
         ''' for creating a new book or syncing with data '''
-        book = self.update_from_mappings(book, data, self.book_mappings)
+        book = update_from_mappings(book, data, self.book_mappings)
 
         author_text = []
         for author in self.get_authors_from_data(data):
             book.authors.add(author)
-            if author.display_name:
-                author_text.append(author.display_name)
+            author_text.append(author.name)
         book.author_text = ', '.join(author_text)
         book.save()
 
@@ -246,23 +261,12 @@ class AbstractConnector(ABC):
     def get_cover_from_data(self, data):
         ''' load cover '''
 
-
-    @abstractmethod
-    def parse_search_data(self, data):
-        ''' turn the result json from a search into a list '''
-
-
-    @abstractmethod
-    def format_search_result(self, search_result):
-        ''' create a SearchResult obj from json '''
-
-
     @abstractmethod
     def expand_book_data(self, book):
         ''' get more info on a book '''
 
 
-    def update_from_mappings(self, obj, data, mappings):
+def update_from_mappings(obj, data, mappings):
     ''' assign data to model with mappings '''
     for mapping in mappings:
         # check if this field is present in the data
@@ -310,10 +314,25 @@ def get_data(url):
         raise ConnectorException()
     if not resp.ok:
         resp.raise_for_status()
+    try:
         data = resp.json()
+    except ValueError:
+        raise ConnectorException()
+
     return data
 
 
+def get_image(url):
+    ''' wrapper for requesting an image '''
+    try:
+        resp = requests.get(url)
+    except (RequestError, SSLError):
+        return None
+    if not resp.ok:
+        return None
+    return resp
+
+
 @dataclass
 class SearchResult:
     ''' standardized search result object '''

bookwyrm/connectors/bookwyrm_connector.py

@@ -1,83 +1,16 @@
 ''' using another bookwyrm instance as a source of book data '''
-from django.db import transaction
-
 from bookwyrm import activitypub, models
-from .abstract_connector import AbstractConnector, SearchResult
-from .abstract_connector import get_data
+from .abstract_connector import AbstractMinimalConnector, SearchResult
 
 
-class Connector(AbstractConnector):
-    ''' interact with other instances '''
-
-    def update_from_mappings(self, obj, data, mappings):
-        ''' serialize book data into a model '''
-        if self.is_work_data(data):
-            work_data = activitypub.Work(**data)
-            return work_data.to_model(models.Work, instance=obj)
-        edition_data = activitypub.Edition(**data)
-        return edition_data.to_model(models.Edition, instance=obj)
-
-
-    def get_remote_id_from_data(self, data):
-        return data.get('id')
-
-
-    def is_work_data(self, data):
-        return data.get('type') == 'Work'
-
-
-    def get_edition_from_work_data(self, data):
-        ''' we're served a list of edition urls '''
-        path = data['editions'][0]
-        return get_data(path)
-
-
-    def get_work_from_edition_date(self, data):
-        return get_data(data['work'])
-
-
-    def get_authors_from_data(self, data):
-        ''' load author data '''
-        for author_id in data.get('authors', []):
-            try:
-                yield models.Author.objects.get(origin_id=author_id)
-            except models.Author.DoesNotExist:
-                pass
-            data = get_data(author_id)
-            author_data = activitypub.Author(**data)
-            author = author_data.to_model(models.Author)
-            yield author
-
-
-    def get_cover_from_data(self, data):
-        pass
+class Connector(AbstractMinimalConnector):
+    ''' this is basically just for search '''
 
+    def get_or_create_book(self, remote_id):
+        return activitypub.resolve_remote_id(models.Edition, remote_id)
 
     def parse_search_data(self, data):
         return data
 
-
     def format_search_result(self, search_result):
         return SearchResult(**search_result)
-
-
-    def expand_book_data(self, book):
-        work = book
-        # go from the edition to the work, if necessary
-        if isinstance(book, models.Edition):
-            work = book.parent_work
-
-        # it may be that we actually want to request this url
-        editions_url = '%s/editions?page=true' % work.remote_id
-        edition_options = get_data(editions_url)
-        for edition_data in edition_options['orderedItems']:
-            with transaction.atomic():
-                edition = self.create_book(
-                    edition_data['id'],
-                    edition_data,
-                    models.Edition
-                )
-                edition.parent_work = work
-                edition.save()
-            if not edition.authors.exists() and work.authors.exists():
-                edition.authors.set(work.authors.all())

bookwyrm/connectors/openlibrary.py

@@ -7,7 +7,7 @@ from django.core.files.base import ContentFile
 from bookwyrm import models
 from .abstract_connector import AbstractConnector, SearchResult, Mapping
 from .abstract_connector import ConnectorException
-from .abstract_connector import get_date, get_data
+from .abstract_connector import get_date, get_data, update_from_mappings
 from .openlibrary_languages import languages
 
 
@@ -65,6 +65,7 @@ class Connector(AbstractConnector):
         ]
 
         self.author_mappings = [
+            Mapping('name'),
             Mapping('born', remote_field='birth_date', formatter=get_date),
             Mapping('died', remote_field='death_date', formatter=get_date),
             Mapping('bio', formatter=get_description),
@@ -184,12 +185,7 @@ class Connector(AbstractConnector):
         data = get_data(url)
 
         author = models.Author(openlibrary_key=olkey)
-        author = self.update_from_mappings(author, data, self.author_mappings)
-        name = data.get('name')
-        # TODO this is making some BOLD assumption
-        if name:
-            author.last_name = name.split(' ')[-1]
-            author.first_name = ' '.join(name.split(' ')[:-1])
+        author = update_from_mappings(author, data, self.author_mappings)
         author.save()
 
         return author

bookwyrm/context_processors.py

@@ -1,7 +1,7 @@
 ''' customize the info available in context for rendering templates '''
 from bookwyrm import models
 
-def site_settings(request):
+def site_settings(request):# pylint: disable=unused-argument
     ''' include the custom info about the site '''
     return {
         'site': models.SiteSettings.objects.get()

bookwyrm/forms.py

@@ -30,7 +30,7 @@ class CustomForm(ModelForm):
                 visible.field.widget.attrs['rows'] = None
             visible.field.widget.attrs['class'] = css_classes[input_type]
 
-
+# pylint: disable=missing-class-docstring
 class LoginForm(CustomForm):
     class Meta:
         model = models.User
@@ -131,6 +131,7 @@ class ImportForm(forms.Form):
 
 class ExpiryWidget(widgets.Select):
     def value_from_datadict(self, data, files, name):
+        ''' human-readable exiration time buckets '''
         selected_string = super().value_from_datadict(data, files, name)
 
         if selected_string == 'day':

bookwyrm/goodreads_import.py

@@ -53,7 +53,7 @@ def import_data(job_id):
         for item in job.items.all():
             try:
                 item.resolve()
-            except Exception as e:
+            except Exception as e:# pylint: disable=broad-except
                 logger.exception(e)
                 item.fail_reason = 'Error loading book'
                 item.save()

bookwyrm/incoming.py

@@ -1,6 +1,6 @@
 ''' handles all of the activity coming in to the server '''
 import json
-from urllib.parse import urldefrag, unquote_plus
+from urllib.parse import urldefrag
 
 import django.db.utils
 from django.http import HttpResponse
@@ -8,9 +8,8 @@ from django.http import HttpResponseBadRequest, HttpResponseNotFound
 from django.views.decorators.csrf import csrf_exempt
 import requests
 
-from bookwyrm import activitypub, books_manager, models, outgoing
+from bookwyrm import activitypub, models, outgoing
 from bookwyrm import status as status_builder
-from bookwyrm.remote_user import get_or_create_remote_user, refresh_remote_user
 from bookwyrm.tasks import app
 from bookwyrm.signatures import Signature
 
@@ -18,9 +17,6 @@ from bookwyrm.signatures import Signature
 @csrf_exempt
 def inbox(request, username):
     ''' incoming activitypub events '''
-    # TODO: should do some kind of checking if the user accepts
-    # this action from the sender probably? idk
-    # but this will just throw a 404 if the user doesn't exist
     try:
         models.User.objects.get(localname=username)
     except models.User.DoesNotExist:
@@ -60,9 +56,8 @@ def shared_inbox(request):
         'Like': handle_favorite,
         'Announce': handle_boost,
         'Add': {
-            'Tag': handle_tag,
-            'Edition': handle_shelve,
-            'Work': handle_shelve,
+            'Edition': handle_add,
+            'Work': handle_add,
         },
         'Undo': {
             'Follow': handle_unfollow,
@@ -97,16 +92,20 @@ def has_valid_signature(request, activity):
         if key_actor != activity.get('actor'):
             raise ValueError("Wrong actor created signature.")
 
-        remote_user = get_or_create_remote_user(key_actor)
+        remote_user = activitypub.resolve_remote_id(models.User, key_actor)
+        if not remote_user:
+            return False
 
         try:
-            signature.verify(remote_user.public_key, request)
+            signature.verify(remote_user.key_pair.public_key, request)
         except ValueError:
-            old_key = remote_user.public_key
-            refresh_remote_user(remote_user)
-            if remote_user.public_key == old_key:
+            old_key = remote_user.key_pair.public_key
+            remote_user = activitypub.resolve_remote_id(
+                models.User, remote_user.remote_id, refresh=True
+            )
+            if remote_user.key_pair.public_key == old_key:
                 raise # Key unchanged.
-            signature.verify(remote_user.public_key, request)
+            signature.verify(remote_user.key_pair.public_key, request)
     except (ValueError, requests.exceptions.HTTPError):
         return False
     return True
@@ -115,26 +114,10 @@ def has_valid_signature(request, activity):
 @app.task
 def handle_follow(activity):
     ''' someone wants to follow a local user '''
-    # figure out who they want to follow -- not using get_or_create because
-    # we only care if you want to follow local users
     try:
-        to_follow = models.User.objects.get(remote_id=activity['object'])
-    except models.User.DoesNotExist:
-        # some rando, who cares
-        return
-    if not to_follow.local:
-        # just ignore follow alerts about other servers. maybe they should be
-        # handled. maybe they shouldn't be sent at all.
-        return
-
-    # figure out who the actor is
-    actor = get_or_create_remote_user(activity['actor'])
-    try:
-        relationship = models.UserFollowRequest.objects.create(
-            user_subject=actor,
-            user_object=to_follow,
-            remote_id=activity['id']
-        )
+        relationship = activitypub.Follow(
+            **activity
+        ).to_model(models.UserFollowRequest)
     except django.db.utils.IntegrityError as err:
         if err.__cause__.diag.constraint_name != 'userfollowrequest_unique':
             raise
@@ -143,27 +126,22 @@ def handle_follow(activity):
         )
         # send the accept normally for a duplicate request
 
-    if not to_follow.manually_approves_followers:
+    manually_approves = relationship.user_object.manually_approves_followers
+
     status_builder.create_notification(
-            to_follow,
-            'FOLLOW',
-            related_user=actor
+        relationship.user_object,
+        'FOLLOW_REQUEST' if manually_approves else 'FOLLOW',
+        related_user=relationship.user_subject
     )
+    if not manually_approves:
         outgoing.handle_accept(relationship)
-    else:
-        # Accept will be triggered manually
-        status_builder.create_notification(
-            to_follow,
-            'FOLLOW_REQUEST',
-            related_user=actor
-        )
 
 
 @app.task
 def handle_unfollow(activity):
     ''' unfollow a local user '''
     obj = activity['object']
-    requester = get_or_create_remote_user(obj['actor'])
+    requester = activitypub.resolve_remote_id(models.user, obj['actor'])
     to_unfollow = models.User.objects.get(remote_id=obj['object'])
     # raises models.User.DoesNotExist
 
@@ -176,7 +154,7 @@ def handle_follow_accept(activity):
     # figure out who they want to follow
     requester = models.User.objects.get(remote_id=activity['object']['actor'])
     # figure out who they are
-    accepter = get_or_create_remote_user(activity['actor'])
+    accepter = activitypub.resolve_remote_id(models.User, activity['actor'])
 
     try:
         request = models.UserFollowRequest.objects.get(
@@ -193,7 +171,7 @@ def handle_follow_accept(activity):
 def handle_follow_reject(activity):
     ''' someone is rejecting a follow request '''
     requester = models.User.objects.get(remote_id=activity['object']['actor'])
-    rejecter = get_or_create_remote_user(activity['actor'])
+    rejecter = activitypub.resolve_remote_id(models.User, activity['actor'])
 
     request = models.UserFollowRequest.objects.get(
         user_subject=requester,
@@ -206,25 +184,40 @@ def handle_follow_reject(activity):
 @app.task
 def handle_create(activity):
     ''' someone did something, good on them '''
-    if activity['object'].get('type') not in \
-            ['Note', 'Comment', 'Quotation', 'Review', 'GeneratedNote']:
-        # if it's an article or unknown type, ignore it
-        return
-
-    user = get_or_create_remote_user(activity['actor'])
-    if user.local:
-        # we really oughtn't even be sending in this case
-        return
-
     # deduplicate incoming activities
-    status_id = activity['object']['id']
+    activity = activity['object']
+    status_id = activity['id']
     if models.Status.objects.filter(remote_id=status_id).count():
         return
 
-    status = status_builder.create_status(activity['object'])
-    if not status:
+    serializer = activitypub.activity_objects[activity['type']]
+    activity = serializer(**activity)
+    try:
+        model = models.activity_models[activity.type]
+    except KeyError:
+        # not a type of status we are prepared to deserialize
         return
 
+    if activity.type == 'Note':
+        # keep notes if they are replies to existing statuses
+        reply = models.Status.objects.filter(
+            remote_id=activity.inReplyTo
+        ).first()
+
+        if not reply:
+            discard = True
+            # keep notes if they mention local users
+            tags = [l['href'] for l in activity.tag if l['type'] == 'Mention']
+            for tag in tags:
+                if models.User.objects.filter(
+                        remote_id=tag, local=True).exists():
+                    # we found a mention of a known use boost
+                    discard = False
+                    break
+            if discard:
+                return
+
+    status = activity.to_model(model)
     # create a notification if this is a reply
     if status.reply_parent and status.reply_parent.user.local:
         status_builder.create_notification(
@@ -258,16 +251,14 @@ def handle_favorite(activity):
     ''' approval of your good good post '''
     fav = activitypub.Like(**activity)
 
-    liker = get_or_create_remote_user(activity['actor'])
-    if liker.local:
-        return
-
     fav = fav.to_model(models.Favorite)
+    if fav.user.local:
+        return
 
     status_builder.create_notification(
         fav.status.user,
         'FAVORITE',
-        related_user=liker,
+        related_user=fav.user,
         related_status=fav.status,
     )
 
@@ -312,35 +303,13 @@ def handle_unboost(activity):
 
 
 @app.task
-def handle_tag(activity):
-    ''' someone is tagging a book '''
-    user = get_or_create_remote_user(activity['actor'])
-    if not user.local:
-        # ordered collection weirndess so we can't just to_model
-        book = books_manager.get_or_create_book(activity['object']['id'])
-        name = activity['object']['target'].split('/')[-1]
-        name = unquote_plus(name)
-        models.Tag.objects.get_or_create(
-            user=user,
-            book=book,
-            name=name
-        )
-
-
-@app.task
-def handle_shelve(activity):
+def handle_add(activity):
     ''' putting a book on a shelf '''
-    user = get_or_create_remote_user(activity['actor'])
-    book = books_manager.get_or_create_book(activity['object'])
+    #this is janky as heck but I haven't thought of a better solution
     try:
-        shelf = models.Shelf.objects.get(remote_id=activity['target'])
-    except models.Shelf.DoesNotExist:
-        return
-    if shelf.user != user:
-        # this doesn't add up.
-        return
-    shelf.books.add(book)
-    shelf.save()
+        activitypub.AddBook(**activity).to_model(models.ShelfBook)
+    except activitypub.ActivitySerializerError:
+        activitypub.AddBook(**activity).to_model(models.Tag)
 
 
 @app.task
@@ -360,13 +329,4 @@ def handle_update_user(activity):
 @app.task
 def handle_update_book(activity):
     ''' a remote instance changed a book (Document) '''
-    document = activity['object']
-    # check if we have their copy and care about their updates
-    book = models.Book.objects.select_subclasses().filter(
-        remote_id=document['id'],
-        sync=True,
-    ).first()
-    if not book:
-        return
-
-    books_manager.update_book(book, data=document)
+    activitypub.Edition(**activity['object']).to_model(models.Edition)

bookwyrm/management/commands/initdb.py

@@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand, CommandError
 from django.contrib.auth.models import Group, Permission
 from django.contrib.contenttypes.models import ContentType
 
-from bookwyrm.models import Connector, User
+from bookwyrm.models import Connector, SiteSettings, User
 from bookwyrm.settings import DOMAIN
 
 def init_groups():
@@ -91,6 +91,9 @@ def init_connectors():
         priority=3,
     )
 
+def init_settings():
+    SiteSettings.objects.create()
+
 class Command(BaseCommand):
     help = 'Initializes the database with starter data'
 
@@ -98,3 +101,4 @@ class Command(BaseCommand):
         init_groups()
         init_permissions()
         init_connectors()
+        init_settings()

bookwyrm/migrations/0001_initial.py

@@ -7,7 +7,7 @@ import django.core.validators
 from django.db import migrations, models
 import django.db.models.deletion
 import django.utils.timezone
-import bookwyrm.utils.fields
+from django.contrib.postgres.fields import JSONField
 
 
 class Migration(migrations.Migration):
@@ -62,7 +62,7 @@ class Migration(migrations.Migration):
                 ('content', models.TextField(blank=True, null=True)),
                 ('created_date', models.DateTimeField(auto_now_add=True)),
                 ('openlibrary_key', models.CharField(max_length=255)),
-                ('data', bookwyrm.utils.fields.JSONField()),
+                ('data', JSONField()),
             ],
             options={
                 'abstract': False,
@@ -75,7 +75,7 @@ class Migration(migrations.Migration):
                 ('content', models.TextField(blank=True, null=True)),
                 ('created_date', models.DateTimeField(auto_now_add=True)),
                 ('openlibrary_key', models.CharField(max_length=255, unique=True)),
-                ('data', bookwyrm.utils.fields.JSONField()),
+                ('data', JSONField()),
                 ('cover', models.ImageField(blank=True, null=True, upload_to='covers/')),
                 ('added_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
                 ('authors', models.ManyToManyField(to='bookwyrm.Author')),

bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py

@@ -2,7 +2,6 @@
 
 import bookwyrm.models.connector
 import bookwyrm.models.site
-import bookwyrm.utils.fields
 from django.conf import settings
 import django.contrib.postgres.operations
 import django.core.validators
@@ -10,6 +9,7 @@ from django.db import migrations, models
 import django.db.models.deletion
 import django.db.models.expressions
 import django.utils.timezone
+from django.contrib.postgres.fields import JSONField, ArrayField
 import uuid
 
 
@@ -148,7 +148,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='misc_identifiers',
-            field=bookwyrm.utils.fields.JSONField(null=True),
+            field=JSONField(null=True),
         ),
         migrations.AddField(
             model_name='book',
@@ -226,7 +226,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='author',
             name='aliases',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='user',
@@ -394,17 +394,17 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='subject_places',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='book',
             name='subjects',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='edition',
             name='publishers',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AlterField(
             model_name='connector',
@@ -578,7 +578,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='languages',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='edition',
@@ -676,7 +676,7 @@ class Migration(migrations.Migration):
             name='ImportItem',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('data', bookwyrm.utils.fields.JSONField()),
+                ('data', JSONField()),
             ],
         ),
         migrations.CreateModel(

bookwyrm/migrations/0016_auto_20201129_0304.py

@@ -0,0 +1,62 @@
+# Generated by Django 3.0.7 on 2020-11-29 03:04
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+from django.contrib.postgres.fields import ArrayField
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0015_auto_20201128_0349'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='book',
+            name='subject_places',
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subjects',
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='parent_work',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=models.CharField(max_length=100, unique=True),
+        ),
+        migrations.AlterUniqueTogether(
+            name='tag',
+            unique_together=set(),
+        ),
+        migrations.RemoveField(
+            model_name='tag',
+            name='book',
+        ),
+        migrations.RemoveField(
+            model_name='tag',
+            name='user',
+        ),
+        migrations.CreateModel(
+            name='UserTag',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_date', models.DateTimeField(auto_now_add=True)),
+                ('updated_date', models.DateTimeField(auto_now=True)),
+                ('remote_id', models.CharField(max_length=255, null=True)),
+                ('book', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition')),
+                ('tag', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'unique_together': {('user', 'book', 'tag')},
+            },
+        ),
+    ]

bookwyrm/migrations/0017_auto_20201130_1819.py

@@ -0,0 +1,189 @@
+# Generated by Django 3.0.7 on 2020-11-30 18:19
+
+import bookwyrm.models.base_model
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+def copy_rsa_keys(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    users = app_registry.get_model('bookwyrm', 'User')
+    keypair = app_registry.get_model('bookwyrm', 'KeyPair')
+    for user in users.objects.using(db_alias):
+        if user.public_key or user.private_key:
+            user.key_pair = keypair.objects.create(
+                remote_id='%s/#main-key' % user.remote_id,
+                private_key=user.private_key,
+                public_key=user.public_key
+            )
+            user.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0016_auto_20201129_0304'),
+    ]
+    operations = [
+        migrations.CreateModel(
+            name='KeyPair',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_date', models.DateTimeField(auto_now_add=True)),
+                ('updated_date', models.DateTimeField(auto_now=True)),
+                ('remote_id', bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id])),
+                ('private_key', models.TextField(blank=True, null=True)),
+                ('public_key', bookwyrm.models.fields.TextField(blank=True, null=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(bookwyrm.models.base_model.ActivitypubMixin, models.Model),
+        ),
+        migrations.AddField(
+            model_name='user',
+            name='followers',
+            field=bookwyrm.models.fields.ManyToManyField(related_name='following', through='bookwyrm.UserFollows', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='connector',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='favorite',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='federatedserver',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='image',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='notification',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='readthrough',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='avatar',
+            field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='avatars/'),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='bookwyrm_user',
+            field=bookwyrm.models.fields.BooleanField(default=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='inbox',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='local',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='manually_approves_followers',
+            field=bookwyrm.models.fields.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='name',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='outbox',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='shared_inbox',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='summary',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='username',
+            field=bookwyrm.models.fields.UsernameField(),
+        ),
+        migrations.AlterField(
+            model_name='userblocks',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='userfollowrequest',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='userfollows',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AddField(
+            model_name='user',
+            name='key_pair',
+            field=bookwyrm.models.fields.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='owner', to='bookwyrm.KeyPair'),
+        ),
+        migrations.RunPython(copy_rsa_keys),
+    ]

bookwyrm/migrations/0018_auto_20201130_1832.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.0.7 on 2020-11-30 18:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0017_auto_20201130_1819'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='user',
+            name='following',
+        ),
+        migrations.RemoveField(
+            model_name='user',
+            name='private_key',
+        ),
+        migrations.RemoveField(
+            model_name='user',
+            name='public_key',
+        ),
+    ]

bookwyrm/migrations/0019_auto_20201130_1939.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.0.7 on 2020-11-30 19:39
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+def update_notnull(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    users = app_registry.get_model('bookwyrm', 'User')
+    for user in users.objects.using(db_alias):
+        if user.name and user.summary:
+            continue
+        if not user.summary:
+            user.summary = ''
+        if not user.name:
+            user.name = ''
+        user.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0018_auto_20201130_1832'),
+    ]
+
+    operations = [
+        migrations.RunPython(update_notnull),
+        migrations.AlterField(
+            model_name='user',
+            name='name',
+            field=bookwyrm.models.fields.CharField(default='', max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='summary',
+            field=bookwyrm.models.fields.TextField(default=''),
+        ),
+    ]

bookwyrm/migrations/0020_auto_20201208_0213.py

@@ -0,0 +1,353 @@
+# Generated by Django 3.0.7 on 2020-12-08 02:13
+
+import bookwyrm.models.fields
+from django.conf import settings
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0019_auto_20201130_1939'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='author',
+            name='aliases',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='bio',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='born',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='died',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='openlibrary_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='wikipedia_link',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='authors',
+            field=bookwyrm.models.fields.ManyToManyField(to='bookwyrm.Author'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='cover',
+            field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='covers/'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='description',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='first_published_date',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='goodreads_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='languages',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='librarything_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='openlibrary_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='published_date',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='series',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='series_number',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='sort_title',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subject_places',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subjects',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subtitle',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='title',
+            field=bookwyrm.models.fields.CharField(max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='boost',
+            name='boosted_status',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='boosters', to='bookwyrm.Status'),
+        ),
+        migrations.AlterField(
+            model_name='comment',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='asin',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='isbn_10',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='isbn_13',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='oclc_number',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='pages',
+            field=bookwyrm.models.fields.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='parent_work',
+            field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='physical_format',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='publishers',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+        ),
+        migrations.AlterField(
+            model_name='favorite',
+            name='status',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'),
+        ),
+        migrations.AlterField(
+            model_name='favorite',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='image',
+            name='caption',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='image',
+            name='image',
+            field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='status/'),
+        ),
+        migrations.AlterField(
+            model_name='quotation',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='quotation',
+            name='quote',
+            field=bookwyrm.models.fields.TextField(),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='rating',
+            field=bookwyrm.models.fields.IntegerField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(5)]),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='privacy',
+            field=bookwyrm.models.fields.CharField(choices=[('public', 'Public'), ('unlisted', 'Unlisted'), ('followers', 'Followers'), ('direct', 'Direct')], default='public', max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='added_by',
+            field=bookwyrm.models.fields.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='shelf',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Shelf'),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='content',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='mention_books',
+            field=bookwyrm.models.fields.TagField(related_name='mention_book', to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='mention_users',
+            field=bookwyrm.models.fields.TagField(related_name='mention_user', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='published_date',
+            field=bookwyrm.models.fields.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='reply_parent',
+            field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='sensitive',
+            field=bookwyrm.models.fields.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=100, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='userblocks',
+            name='user_object',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_object', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userblocks',
+            name='user_subject',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_subject', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollowrequest',
+            name='user_object',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_object', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollowrequest',
+            name='user_subject',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_subject', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollows',
+            name='user_object',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_object', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollows',
+            name='user_subject',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_subject', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='tag',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag'),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='work',
+            name='default_edition',
+            field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='work',
+            name='lccn',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

bookwyrm/migrations/0021_merge_20201212_1737.py

@@ -0,0 +1,14 @@
+# Generated by Django 3.0.7 on 2020-12-12 17:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0020_auto_20201208_0213'),
+        ('bookwyrm', '0016_auto_20201211_2026'),
+    ]
+
+    operations = [
+    ]

bookwyrm/migrations/0022_auto_20201212_1744.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.0.7 on 2020-12-12 17:44
+
+from django.db import migrations
+
+
+def set_author_name(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    authors = app_registry.get_model('bookwyrm', 'Author')
+    for author in authors.objects.using(db_alias):
+        if not author.name:
+            author.name = '%s %s' % (author.first_name, author.last_name)
+            author.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0021_merge_20201212_1737'),
+    ]
+
+    operations = [
+        migrations.RunPython(set_author_name),
+        migrations.RemoveField(
+            model_name='author',
+            name='first_name',
+        ),
+        migrations.RemoveField(
+            model_name='author',
+            name='last_name',
+        ),
+    ]

bookwyrm/models/__init__.py

@@ -12,9 +12,9 @@ from .status import Status, GeneratedNote, Review, Comment, Quotation
 from .status import Favorite, Boost, Notification, ReadThrough
 from .attachment import Image
 
-from .tag import Tag
+from .tag import Tag, UserTag
 
-from .user import User
+from .user import User, KeyPair
 from .relationship import UserFollows, UserFollowRequest, UserBlocks
 from .federated_server import FederatedServer
 

bookwyrm/models/attachment.py

@@ -3,7 +3,8 @@ from django.db import models
 
 from bookwyrm import activitypub
 from .base_model import ActivitypubMixin
-from .base_model import ActivityMapping, BookWyrmModel
+from .base_model import BookWyrmModel
+from . import fields
 
 
 class Attachment(ActivitypubMixin, BookWyrmModel):
@@ -14,19 +15,16 @@ class Attachment(ActivitypubMixin, BookWyrmModel):
         related_name='attachments',
         null=True
     )
+    reverse_unfurl = True
     class Meta:
         ''' one day we'll have other types of attachments besides images '''
         abstract = True
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('url', 'image'),
-        ActivityMapping('name', 'caption'),
-    ]
 
 class Image(Attachment):
     ''' an image attachment '''
-    image = models.ImageField(upload_to='status/', null=True, blank=True)
-    caption = models.TextField(null=True, blank=True)
+    image = fields.ImageField(
+        upload_to='status/', null=True, blank=True, activitypub_field='url')
+    caption = fields.TextField(null=True, blank=True, activitypub_field='name')
 
     activity_serializer = activitypub.Image

bookwyrm/models/author.py

@@ -3,48 +3,42 @@ from django.db import models
 from django.utils import timezone
 
 from bookwyrm import activitypub
-from bookwyrm.utils.fields import ArrayField
+from bookwyrm.settings import DOMAIN
 
-from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel
+from .base_model import ActivitypubMixin, BookWyrmModel
+from . import fields
 
 
 class Author(ActivitypubMixin, BookWyrmModel):
     ''' basic biographic info '''
     origin_id = models.CharField(max_length=255, null=True)
-    ''' copy of an author from OL '''
-    openlibrary_key = models.CharField(max_length=255, blank=True, null=True)
+    openlibrary_key = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
     sync = models.BooleanField(default=True)
     last_sync_date = models.DateTimeField(default=timezone.now)
-    wikipedia_link = models.CharField(max_length=255, blank=True, null=True)
+    wikipedia_link = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
     # idk probably other keys would be useful here?
-    born = models.DateTimeField(blank=True, null=True)
-    died = models.DateTimeField(blank=True, null=True)
-    name = models.CharField(max_length=255)
-    last_name = models.CharField(max_length=255, blank=True, null=True)
-    first_name = models.CharField(max_length=255, blank=True, null=True)
-    aliases = ArrayField(
+    born = fields.DateTimeField(blank=True, null=True)
+    died = fields.DateTimeField(blank=True, null=True)
+    name = fields.CharField(max_length=255)
+    aliases = fields.ArrayField(
         models.CharField(max_length=255), blank=True, default=list
     )
-    bio = models.TextField(null=True, blank=True)
+    bio = fields.TextField(null=True, blank=True)
 
-    @property
-    def display_name(self):
-        ''' Helper to return a displayable name'''
-        if self.name:
-            return self.name
-        # don't want to return a spurious space if all of these are None
-        if self.first_name and self.last_name:
-            return self.first_name + ' ' + self.last_name
-        return self.last_name or self.first_name
+    def save(self, *args, **kwargs):
+        ''' can't be abstract for query reasons, but you shouldn't USE it '''
+        if self.id and not self.remote_id:
+            self.remote_id = self.get_remote_id()
+
+        if not self.id:
+            self.origin_id = self.remote_id
+            self.remote_id = None
+        return super().save(*args, **kwargs)
+
+    def get_remote_id(self):
+        ''' editions and works both use "book" instead of model_name '''
+        return 'https://%s/author/%s' % (DOMAIN, self.id)
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('name', 'name'),
-        ActivityMapping('born', 'born'),
-        ActivityMapping('died', 'died'),
-        ActivityMapping('aliases', 'aliases'),
-        ActivityMapping('bio', 'bio'),
-        ActivityMapping('openlibraryKey', 'openlibrary_key'),
-        ActivityMapping('wikipediaLink', 'wikipedia_link'),
-    ]
     activity_serializer = activitypub.Author

bookwyrm/models/base_model.py

@@ -1,34 +1,27 @@
 ''' base model with default fields '''
-from datetime import datetime
 from base64 import b64encode
-from dataclasses import dataclass
-from typing import Callable
+from functools import reduce
+import operator
 from uuid import uuid4
-from urllib.parse import urlencode
 
 from Crypto.PublicKey import RSA
 from Crypto.Signature import pkcs1_15
 from Crypto.Hash import SHA256
+from django.core.paginator import Paginator
 from django.db import models
-from django.db.models.fields.files import ImageFieldFile
+from django.db.models import Q
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import DOMAIN, PAGE_LENGTH
+from .fields import ImageField, ManyToManyField, RemoteIdField
 
 
-PrivacyLevels = models.TextChoices('Privacy', [
-    'public',
-    'unlisted',
-    'followers',
-    'direct'
-])
-
 class BookWyrmModel(models.Model):
     ''' shared fields '''
     created_date = models.DateTimeField(auto_now_add=True)
     updated_date = models.DateTimeField(auto_now=True)
-    remote_id = models.CharField(max_length=255, null=True)
+    remote_id = RemoteIdField(null=True, activitypub_field='id')
 
     def get_remote_id(self):
         ''' generate a url that resolves to the local object '''
@@ -44,6 +37,7 @@ class BookWyrmModel(models.Model):
 
 
 @receiver(models.signals.post_save)
+#pylint: disable=unused-argument
 def execute_after_save(sender, instance, created, *args, **kwargs):
     ''' set the remote_id after save (when the id is available) '''
     if not created or not hasattr(instance, 'get_remote_id'):
@@ -53,58 +47,115 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
         instance.save()
 
 
+def unfurl_related_field(related_field):
+    ''' load reverse lookups (like public key owner or Status attachment '''
+    if hasattr(related_field, 'all'):
+        return [unfurl_related_field(i) for i in related_field.all()]
+    if related_field.reverse_unfurl:
+        return related_field.field_to_activity()
+    return related_field.remote_id
+
+
 class ActivitypubMixin:
     ''' add this mixin for models that are AP serializable '''
     activity_serializer = lambda: {}
+    reverse_unfurl = False
 
-    def to_activity(self, pure=False):
-        ''' convert from a model to an activity '''
-        if pure:
-            # works around bookwyrm-specific fields for vanilla AP services
-            mappings = self.pure_activity_mappings
-        else:
-            # may include custom fields that bookwyrm instances will understand
-            mappings = self.activity_mappings
-
-        fields = {}
-        for mapping in mappings:
-            if not hasattr(self, mapping.model_key) or not mapping.activity_key:
-                # this field on the model isn't serialized
+    def __init__(self, *args, **kwargs):
+        ''' collect some info on model fields '''
+        self.image_fields = []
+        self.many_to_many_fields = []
+        self.simple_fields = [] # "simple"
+        for field in self._meta.get_fields():
+            if not hasattr(field, 'field_to_activity'):
                 continue
-            value = getattr(self, mapping.model_key)
-            if hasattr(value, 'remote_id'):
-                # this is probably a foreign key field, which we want to
-                # serialize as just the remote_id url reference
-                value = value.remote_id
-            elif isinstance(value, datetime):
-                value = value.isoformat()
-            elif isinstance(value, ImageFieldFile):
-                value = image_formatter(value)
 
-            # run the custom formatter function set in the model
-            formatted_value = mapping.activity_formatter(value)
-            if mapping.activity_key in fields and \
-                    isinstance(fields[mapping.activity_key], list):
-                # there can be two database fields that map to the same AP list
-                # this happens in status tags, which combines user and book tags
-                fields[mapping.activity_key] += formatted_value
+            if isinstance(field, ImageField):
+                self.image_fields.append(field)
+            elif isinstance(field, ManyToManyField):
+                self.many_to_many_fields.append(field)
             else:
-                fields[mapping.activity_key] = formatted_value
+                self.simple_fields.append(field)
 
-        if pure:
-            return self.pure_activity_serializer(
-                **fields
-            ).serialize()
-        return self.activity_serializer(
-            **fields
-        ).serialize()
+        self.activity_fields = self.image_fields + \
+                self.many_to_many_fields + self.simple_fields
+
+        self.deserialize_reverse_fields = self.deserialize_reverse_fields \
+                if hasattr(self, 'deserialize_reverse_fields') else []
+        self.serialize_reverse_fields = self.serialize_reverse_fields \
+                if hasattr(self, 'serialize_reverse_fields') else []
+
+        super().__init__(*args, **kwargs)
 
 
-    def to_create_activity(self, user, pure=False):
+    @classmethod
+    def find_existing_by_remote_id(cls, remote_id):
+        ''' look up a remote id in the db '''
+        return cls.find_existing({'id': remote_id})
+
+    @classmethod
+    def find_existing(cls, data):
+        ''' compare data to fields that can be used for deduplation.
+        This always includes remote_id, but can also be unique identifiers
+        like an isbn for an edition '''
+        filters = []
+        for field in cls._meta.get_fields():
+            if not hasattr(field, 'deduplication_field') or \
+                    not field.deduplication_field:
+                continue
+
+            value = data.get(field.activitypub_field)
+            if not value:
+                continue
+            filters.append({field.name: value})
+
+        if hasattr(cls, 'origin_id') and 'id' in data:
+            # kinda janky, but this handles special case for books
+            filters.append({'origin_id': data['id']})
+
+        if not filters:
+            # if there are no deduplication fields, it will match the first
+            # item no matter what. this shouldn't happen but just in case.
+            return None
+
+        objects = cls.objects
+        if hasattr(objects, 'select_subclasses'):
+            objects = objects.select_subclasses()
+
+        # an OR operation on all the match fields
+        match = objects.filter(
+            reduce(
+                operator.or_, (Q(**f) for f in filters)
+            )
+        )
+        # there OUGHT to be only one match
+        return match.first()
+
+
+    def to_activity(self):
+        ''' convert from a model to an activity '''
+        activity = {}
+        for field in self.activity_fields:
+            field.set_activity_from_field(activity, self)
+
+        if hasattr(self, 'serialize_reverse_fields'):
+            # for example, editions of a work
+            for model_field_name, activity_field_name in \
+                    self.serialize_reverse_fields:
+                related_field = getattr(self, model_field_name)
+                activity[activity_field_name] = \
+                        unfurl_related_field(related_field)
+
+        if not activity.get('id'):
+            activity['id'] = self.get_remote_id()
+        return self.activity_serializer(**activity).serialize()
+
+
+    def to_create_activity(self, user):
         ''' returns the object wrapped in a Create activity '''
-        activity_object = self.to_activity(pure=pure)
+        activity_object = self.to_activity()
 
-        signer = pkcs1_15.new(RSA.import_key(user.private_key))
+        signer = pkcs1_15.new(RSA.import_key(user.key_pair.private_key))
         content = activity_object['content']
         signed_message = signer.sign(SHA256.new(content.encode('utf8')))
         create_id = self.remote_id + '/activity'
@@ -118,8 +169,8 @@ class ActivitypubMixin:
         return activitypub.Create(
             id=create_id,
             actor=user.remote_id,
-            to=['%s/followers' % user.remote_id],
-            cc=['https://www.w3.org/ns/activitystreams#Public'],
+            to=activity_object['to'],
+            cc=activity_object['cc'],
             object=activity_object,
             signature=signature,
         ).serialize()
@@ -127,21 +178,18 @@ class ActivitypubMixin:
 
     def to_delete_activity(self, user):
         ''' notice of deletion '''
-        # this should be a tombstone
-        activity_object = self.to_activity()
-
         return activitypub.Delete(
             id=self.remote_id + '/activity',
             actor=user.remote_id,
             to=['%s/followers' % user.remote_id],
             cc=['https://www.w3.org/ns/activitystreams#Public'],
-            object=activity_object,
+            object=self.to_activity(),
         ).serialize()
 
 
     def to_update_activity(self, user):
         ''' wrapper for Updates to an activity '''
-        activity_id = '%s#update/%s' % (user.remote_id, uuid4())
+        activity_id = '%s#update/%s' % (self.remote_id, uuid4())
         return activitypub.Update(
             id=activity_id,
             actor=user.remote_id,
@@ -153,10 +201,10 @@ class ActivitypubMixin:
     def to_undo_activity(self, user):
         ''' undo an action '''
         return activitypub.Undo(
-            id='%s#undo' % user.remote_id,
+            id='%s#undo' % self.remote_id,
             actor=user.remote_id,
             object=self.to_activity()
-        )
+        ).serialize()
 
 
 class OrderedCollectionPageMixin(ActivitypubMixin):
@@ -167,74 +215,50 @@ class OrderedCollectionPageMixin(ActivitypubMixin):
         ''' this can be overriden if there's a special remote id, ie outbox '''
         return self.remote_id
 
-    def page(self, min_id=None, max_id=None):
-        ''' helper function to create the pagination url '''
-        params = {'page': 'true'}
-        if min_id:
-            params['min_id'] = min_id
-        if max_id:
-            params['max_id'] = max_id
-        return '?%s' % urlencode(params)
-
-    def next_page(self, items):
-        ''' use the max id of the last item '''
-        if not items.count():
-            return ''
-        return self.page(max_id=items[items.count() - 1].id)
-
-    def prev_page(self, items):
-        ''' use the min id of the first item '''
-        if not items.count():
-            return ''
-        return self.page(min_id=items[0].id)
-
-    def to_ordered_collection_page(self, queryset, remote_id, \
-            id_only=False, min_id=None, max_id=None):
-        ''' serialize and pagiante a queryset '''
-        # TODO: weird place to define this
-        limit = 20
-        # filters for use in the django queryset min/max
-        filters = {}
-        if min_id is not None:
-            filters['id__gt'] = min_id
-        if max_id is not None:
-            filters['id__lte'] = max_id
-        page_id = self.page(min_id=min_id, max_id=max_id)
-
-        items = queryset.filter(
-            **filters
-        ).all()[:limit]
-
-        if id_only:
-            page = [s.remote_id for s in items]
-        else:
-            page = [s.to_activity() for s in items]
-        return activitypub.OrderedCollectionPage(
-            id='%s%s' % (remote_id, page_id),
-            partOf=remote_id,
-            orderedItems=page,
-            next='%s%s' % (remote_id, self.next_page(items)),
-            prev='%s%s' % (remote_id, self.prev_page(items))
-        ).serialize()
 
     def to_ordered_collection(self, queryset, \
             remote_id=None, page=False, **kwargs):
         ''' an ordered collection of whatevers '''
         remote_id = remote_id or self.remote_id
         if page:
-            return self.to_ordered_collection_page(
+            return to_ordered_collection_page(
                 queryset, remote_id, **kwargs)
-        name = ''
-        if hasattr(self, 'name'):
-            name = self.name
+        name = self.name if hasattr(self, 'name') else None
+        owner = self.user.remote_id if hasattr(self, 'user') else ''
 
-        size = queryset.count()
+        paginated = Paginator(queryset, PAGE_LENGTH)
         return activitypub.OrderedCollection(
             id=remote_id,
-            totalItems=size,
+            totalItems=paginated.count,
             name=name,
-            first='%s%s' % (remote_id, self.page()),
-            last='%s%s' % (remote_id, self.page(min_id=0))
+            owner=owner,
+            first='%s?page=1' % remote_id,
+            last='%s?page=%d' % (remote_id, paginated.num_pages)
+        ).serialize()
+
+
+def to_ordered_collection_page(queryset, remote_id, id_only=False, page=1):
+    ''' serialize and pagiante a queryset '''
+    paginated = Paginator(queryset, PAGE_LENGTH)
+
+    activity_page = paginated.page(page)
+    if id_only:
+        items = [s.remote_id for s in activity_page.object_list]
+    else:
+        items = [s.to_activity() for s in activity_page.object_list]
+
+    prev_page = next_page = None
+    if activity_page.has_next():
+        next_page = '%s?page=%d' % (remote_id, activity_page.next_page_number())
+    if activity_page.has_previous():
+        prev_page = '%s?page=%d' % \
+                (remote_id, activity_page.previous_page_number())
+    return activitypub.OrderedCollectionPage(
+        id='%s?page=%s' % (remote_id, page),
+        partOf=remote_id,
+        orderedItems=items,
+        next=next_page,
+        prev=prev_page
     ).serialize()
 
 
@@ -250,39 +274,3 @@ class OrderedCollectionMixin(OrderedCollectionPageMixin):
     def to_activity(self, **kwargs):
         ''' an ordered collection of the specified model queryset  '''
         return self.to_ordered_collection(self.collection_queryset, **kwargs)
-
-
-@dataclass(frozen=True)
-class ActivityMapping:
-    ''' translate between an activitypub json field and a model field '''
-    activity_key: str
-    model_key: str
-    activity_formatter: Callable = lambda x: x
-    model_formatter: Callable = lambda x: x
-
-
-def tag_formatter(items, name_field, activity_type):
-    ''' helper function to format lists of foreign keys into Tags '''
-    tags = []
-    for item in items.all():
-        tags.append(activitypub.Link(
-            href=item.remote_id,
-            name=getattr(item, name_field),
-            type=activity_type
-        ))
-    return tags
-
-
-def image_formatter(image):
-    ''' convert images into activitypub json '''
-    if image and hasattr(image, 'url'):
-        url = image.url
-    else:
-        return None
-    url = 'https://%s%s' % (DOMAIN, url)
-    return activitypub.Image(url=url)
-
-
-def image_attachments_formatter(images):
-    ''' create a list of image attachments '''
-    return [image_formatter(i) for i in images]

bookwyrm/models/book.py

@@ -2,24 +2,26 @@
 import re
 
 from django.db import models
-from django.db.models import Q
 from django.utils import timezone
 from model_utils.managers import InheritanceManager
 
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
-from bookwyrm.utils.fields import ArrayField
 
-from .base_model import ActivityMapping, BookWyrmModel
+from .base_model import BookWyrmModel
 from .base_model import ActivitypubMixin, OrderedCollectionPageMixin
+from . import fields
 
 class Book(ActivitypubMixin, BookWyrmModel):
     ''' a generic book, which can mean either an edition or a work '''
     origin_id = models.CharField(max_length=255, null=True, blank=True)
     # these identifiers apply to both works and editions
-    openlibrary_key = models.CharField(max_length=255, blank=True, null=True)
-    librarything_key = models.CharField(max_length=255, blank=True, null=True)
-    goodreads_key = models.CharField(max_length=255, blank=True, null=True)
+    openlibrary_key = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    librarything_key = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    goodreads_key = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
 
     # info about where the data comes from and where/if to sync
     sync = models.BooleanField(default=True)
@@ -31,78 +33,43 @@ class Book(ActivitypubMixin, BookWyrmModel):
     # TODO: edit history
 
     # book/work metadata
-    title = models.CharField(max_length=255)
-    sort_title = models.CharField(max_length=255, blank=True, null=True)
-    subtitle = models.CharField(max_length=255, blank=True, null=True)
-    description = models.TextField(blank=True, null=True)
-    languages = ArrayField(
+    title = fields.CharField(max_length=255)
+    sort_title = fields.CharField(max_length=255, blank=True, null=True)
+    subtitle = fields.CharField(max_length=255, blank=True, null=True)
+    description = fields.TextField(blank=True, null=True)
+    languages = fields.ArrayField(
         models.CharField(max_length=255), blank=True, default=list
     )
-    series = models.CharField(max_length=255, blank=True, null=True)
-    series_number = models.CharField(max_length=255, blank=True, null=True)
-    subjects = ArrayField(
-        models.CharField(max_length=255), blank=True, default=list
+    series = fields.CharField(max_length=255, blank=True, null=True)
+    series_number = fields.CharField(max_length=255, blank=True, null=True)
+    subjects = fields.ArrayField(
+        models.CharField(max_length=255), blank=True, null=True, default=list
     )
-    subject_places = ArrayField(
-        models.CharField(max_length=255), blank=True, default=list
+    subject_places = fields.ArrayField(
+        models.CharField(max_length=255), blank=True, null=True, default=list
     )
     # TODO: include an annotation about the type of authorship (ie, translator)
-    authors = models.ManyToManyField('Author')
+    authors = fields.ManyToManyField('Author')
     # preformatted authorship string for search and easier display
     author_text = models.CharField(max_length=255, blank=True, null=True)
-    cover = models.ImageField(upload_to='covers/', blank=True, null=True)
-    first_published_date = models.DateTimeField(blank=True, null=True)
-    published_date = models.DateTimeField(blank=True, null=True)
+    cover = fields.ImageField(upload_to='covers/', blank=True, null=True)
+    first_published_date = fields.DateTimeField(blank=True, null=True)
+    published_date = fields.DateTimeField(blank=True, null=True)
+
     objects = InheritanceManager()
 
-    @property
-    def ap_authors(self):
-        ''' the activitypub serialization should be a list of author ids '''
-        return [a.remote_id for a in self.authors.all()]
-
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-
-        ActivityMapping('authors', 'ap_authors'),
-        ActivityMapping('firstPublishedDate', 'firstpublished_date'),
-        ActivityMapping('publishedDate', 'published_date'),
-
-        ActivityMapping('title', 'title'),
-        ActivityMapping('sortTitle', 'sort_title'),
-        ActivityMapping('subtitle', 'subtitle'),
-        ActivityMapping('description', 'description'),
-        ActivityMapping('languages', 'languages'),
-        ActivityMapping('series', 'series'),
-        ActivityMapping('seriesNumber', 'series_number'),
-        ActivityMapping('subjects', 'subjects'),
-        ActivityMapping('subjectPlaces', 'subject_places'),
-
-        ActivityMapping('openlibraryKey', 'openlibrary_key'),
-        ActivityMapping('librarythingKey', 'librarything_key'),
-        ActivityMapping('goodreadsKey', 'goodreads_key'),
-
-        ActivityMapping('work', 'parent_work'),
-        ActivityMapping('isbn10', 'isbn_10'),
-        ActivityMapping('isbn13', 'isbn_13'),
-        ActivityMapping('oclcNumber', 'oclc_number'),
-        ActivityMapping('asin', 'asin'),
-        ActivityMapping('pages', 'pages'),
-        ActivityMapping('physicalFormat', 'physical_format'),
-        ActivityMapping('publishers', 'publishers'),
-
-        ActivityMapping('lccn', 'lccn'),
-        ActivityMapping('editions', 'editions_path'),
-        ActivityMapping('cover', 'cover'),
-    ]
-
     def save(self, *args, **kwargs):
         ''' can't be abstract for query reasons, but you shouldn't USE it '''
         if not isinstance(self, Edition) and not isinstance(self, Work):
             raise ValueError('Books should be added as Editions or Works')
+
         if self.id and not self.remote_id:
             self.remote_id = self.get_remote_id()
 
-        super().save(*args, **kwargs)
+        if not self.id:
+            self.origin_id = self.remote_id
+            self.remote_id = None
+        return super().save(*args, **kwargs)
 
     def get_remote_id(self):
         ''' editions and works both use "book" instead of model_name '''
@@ -119,47 +86,38 @@ class Book(ActivitypubMixin, BookWyrmModel):
 class Work(OrderedCollectionPageMixin, Book):
     ''' a work (an abstract concept of a book that manifests in an edition) '''
     # library of congress catalog control number
-    lccn = models.CharField(max_length=255, blank=True, null=True)
+    lccn = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
     # this has to be nullable but should never be null
-    default_edition = models.ForeignKey(
+    default_edition = fields.ForeignKey(
         'Edition',
         on_delete=models.PROTECT,
         null=True
     )
 
-    @property
-    def editions_path(self):
-        ''' it'd be nice to serialize the edition instead but, recursion '''
-        default = self.default_edition
-        ed_list = [
-            e.remote_id for e in self.edition_set.filter(~Q(id=default.id)).all()
-        ]
-        return [default.remote_id] + ed_list
-
-
-    def to_edition_list(self, **kwargs):
-        ''' activitypub serialization for this work's editions '''
-        remote_id = self.remote_id + '/editions'
-        return self.to_ordered_collection(
-            self.edition_set,
-            remote_id=remote_id,
-            **kwargs
-        )
-
+    def get_default_edition(self):
+        ''' in case the default edition is not set '''
+        return self.default_edition or self.editions.first()
 
     activity_serializer = activitypub.Work
+    serialize_reverse_fields = [('editions', 'editions')]
+    deserialize_reverse_fields = [('editions', 'editions')]
 
 
 class Edition(Book):
     ''' an edition of a book '''
     # these identifiers only apply to editions, not works
-    isbn_10 = models.CharField(max_length=255, blank=True, null=True)
-    isbn_13 = models.CharField(max_length=255, blank=True, null=True)
-    oclc_number = models.CharField(max_length=255, blank=True, null=True)
-    asin = models.CharField(max_length=255, blank=True, null=True)
-    pages = models.IntegerField(blank=True, null=True)
-    physical_format = models.CharField(max_length=255, blank=True, null=True)
-    publishers = ArrayField(
+    isbn_10 = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    isbn_13 = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    oclc_number = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    asin = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    pages = fields.IntegerField(blank=True, null=True)
+    physical_format = fields.CharField(max_length=255, blank=True, null=True)
+    publishers = fields.ArrayField(
         models.CharField(max_length=255), blank=True, default=list
     )
     shelves = models.ManyToManyField(
@@ -168,9 +126,12 @@ class Edition(Book):
         through='ShelfBook',
         through_fields=('book', 'shelf')
     )
-    parent_work = models.ForeignKey('Work', on_delete=models.PROTECT, null=True)
+    parent_work = fields.ForeignKey(
+        'Work', on_delete=models.PROTECT, null=True,
+        related_name='editions', activitypub_field='work')
 
     activity_serializer = activitypub.Edition
+    name_field = 'title'
 
     def save(self, *args, **kwargs):
         ''' calculate isbn 10/13 '''

bookwyrm/models/fields.py

@@ -0,0 +1,363 @@
+''' activitypub-aware django model fields '''
+from dataclasses import MISSING
+import re
+from uuid import uuid4
+
+import dateutil.parser
+from dateutil.parser import ParserError
+from django.contrib.auth.models import AbstractUser
+from django.contrib.postgres.fields import ArrayField as DjangoArrayField
+from django.core.exceptions import ValidationError
+from django.core.files.base import ContentFile
+from django.db import models
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+from bookwyrm import activitypub
+from bookwyrm.settings import DOMAIN
+from bookwyrm.connectors import get_image
+
+
+def validate_remote_id(value):
+    ''' make sure the remote_id looks like a url '''
+    if not value or not re.match(r'^http.?:\/\/[^\s]+$', value):
+        raise ValidationError(
+            _('%(value)s is not a valid remote_id'),
+            params={'value': value},
+        )
+
+
+class ActivitypubFieldMixin:
+    ''' make a database field serializable '''
+    def __init__(self, *args, \
+                 activitypub_field=None, activitypub_wrapper=None,
+                 deduplication_field=False, **kwargs):
+        self.deduplication_field = deduplication_field
+        if activitypub_wrapper:
+            self.activitypub_wrapper = activitypub_field
+            self.activitypub_field = activitypub_wrapper
+        else:
+            self.activitypub_field = activitypub_field
+        super().__init__(*args, **kwargs)
+
+
+    def set_field_from_activity(self, instance, data):
+        ''' helper function for assinging a value to the field '''
+        value = getattr(data, self.get_activitypub_field())
+        formatted = self.field_from_activity(value)
+        if formatted is None or formatted is MISSING:
+            return
+        setattr(instance, self.name, formatted)
+
+
+    def set_activity_from_field(self, activity, instance):
+        ''' update the json object '''
+        value = getattr(instance, self.name)
+        formatted = self.field_to_activity(value)
+        if formatted is None:
+            return
+
+        key = self.get_activitypub_field()
+        if isinstance(activity.get(key), list):
+            activity[key] += formatted
+        else:
+            activity[key] = formatted
+
+
+    def field_to_activity(self, value):
+        ''' formatter to convert a model value into activitypub '''
+        if hasattr(self, 'activitypub_wrapper'):
+            return {self.activitypub_wrapper: value}
+        return value
+
+    def field_from_activity(self, value):
+        ''' formatter to convert activitypub into a model value '''
+        if hasattr(self, 'activitypub_wrapper'):
+            value = value.get(self.activitypub_wrapper)
+        return value
+
+    def get_activitypub_field(self):
+        ''' model_field_name to activitypubFieldName '''
+        if self.activitypub_field:
+            return self.activitypub_field
+        name = self.name.split('.')[-1]
+        components = name.split('_')
+        return components[0] + ''.join(x.title() for x in components[1:])
+
+
+class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
+    ''' default (de)serialization for foreign key and one to one '''
+    def field_from_activity(self, value):
+        if not value:
+            return None
+
+        related_model = self.related_model
+        if isinstance(value, dict) and value.get('id'):
+            # this is an activitypub object, which we can deserialize
+            activity_serializer = related_model.activity_serializer
+            return activity_serializer(**value).to_model(related_model)
+        try:
+            # make sure the value looks like a remote id
+            validate_remote_id(value)
+        except ValidationError:
+            # we don't know what this is, ignore it
+            return None
+        # gets or creates the model field from the remote id
+        return activitypub.resolve_remote_id(related_model, value)
+
+
+class RemoteIdField(ActivitypubFieldMixin, models.CharField):
+    ''' a url that serves as a unique identifier '''
+    def __init__(self, *args, max_length=255, validators=None, **kwargs):
+        validators = validators or [validate_remote_id]
+        super().__init__(
+            *args, max_length=max_length, validators=validators,
+            **kwargs
+        )
+        # for this field, the default is true. false everywhere else.
+        self.deduplication_field = kwargs.get('deduplication_field', True)
+
+
+class UsernameField(ActivitypubFieldMixin, models.CharField):
+    ''' activitypub-aware username field '''
+    def __init__(self, activitypub_field='preferredUsername'):
+        self.activitypub_field = activitypub_field
+        # I don't totally know why pylint is mad at this, but it makes it work
+        super( #pylint: disable=bad-super-call
+            ActivitypubFieldMixin, self
+        ).__init__(
+            _('username'),
+            max_length=150,
+            unique=True,
+            validators=[AbstractUser.username_validator],
+            error_messages={
+                'unique': _('A user with that username already exists.'),
+            },
+        )
+
+    def deconstruct(self):
+        ''' implementation of models.Field deconstruct '''
+        name, path, args, kwargs = super().deconstruct()
+        del kwargs['verbose_name']
+        del kwargs['max_length']
+        del kwargs['unique']
+        del kwargs['validators']
+        del kwargs['error_messages']
+        return name, path, args, kwargs
+
+    def field_to_activity(self, value):
+        return value.split('@')[0]
+
+
+PrivacyLevels = models.TextChoices('Privacy', [
+    'public',
+    'unlisted',
+    'followers',
+    'direct'
+])
+
+class PrivacyField(ActivitypubFieldMixin, models.CharField):
+    ''' this maps to two differente activitypub fields '''
+    public = 'https://www.w3.org/ns/activitystreams#Public'
+    def __init__(self, *args, **kwargs):
+        super().__init__(
+            *args, max_length=255,
+            choices=PrivacyLevels.choices, default='public')
+
+    def set_field_from_activity(self, instance, data):
+        to = data.to
+        cc = data.cc
+        if to == [self.public]:
+            setattr(instance, self.name, 'public')
+        elif cc == []:
+            setattr(instance, self.name, 'direct')
+        elif self.public in cc:
+            setattr(instance, self.name, 'unlisted')
+        else:
+            setattr(instance, self.name, 'followers')
+
+    def set_activity_from_field(self, activity, instance):
+        mentions = [u.remote_id for u in instance.mention_users.all()]
+        # this is a link to the followers list
+        followers = instance.user.__class__._meta.get_field('followers')\
+                .field_to_activity(instance.user.followers)
+        if instance.privacy == 'public':
+            activity['to'] = [self.public]
+            activity['cc'] = [followers] + mentions
+        elif instance.privacy == 'unlisted':
+            activity['to'] = [followers]
+            activity['cc'] = [self.public] + mentions
+        elif instance.privacy == 'followers':
+            activity['to'] = [followers]
+            activity['cc'] = mentions
+        if instance.privacy == 'direct':
+            activity['to'] = mentions
+            activity['cc'] = []
+
+
+class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey):
+    ''' activitypub-aware foreign key field '''
+    def field_to_activity(self, value):
+        if not value:
+            return None
+        return value.remote_id
+
+
+class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField):
+    ''' activitypub-aware foreign key field '''
+    def field_to_activity(self, value):
+        if not value:
+            return None
+        return value.to_activity()
+
+
+class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
+    ''' activitypub-aware many to many field '''
+    def __init__(self, *args, link_only=False, **kwargs):
+        self.link_only = link_only
+        super().__init__(*args, **kwargs)
+
+    def set_field_from_activity(self, instance, data):
+        ''' helper function for assinging a value to the field '''
+        value = getattr(data, self.get_activitypub_field())
+        formatted = self.field_from_activity(value)
+        if formatted is None or formatted is MISSING:
+            return
+        getattr(instance, self.name).set(formatted)
+
+    def field_to_activity(self, value):
+        if self.link_only:
+            return '%s/%s' % (value.instance.remote_id, self.name)
+        return [i.remote_id for i in value.all()]
+
+    def field_from_activity(self, value):
+        items = []
+        for remote_id in value:
+            try:
+                validate_remote_id(remote_id)
+            except ValidationError:
+                continue
+            items.append(
+                activitypub.resolve_remote_id(self.related_model, remote_id)
+            )
+        return items
+
+
+class TagField(ManyToManyField):
+    ''' special case of many to many that uses Tags '''
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.activitypub_field = 'tag'
+
+    def field_to_activity(self, value):
+        tags = []
+        for item in value.all():
+            activity_type = item.__class__.__name__
+            if activity_type == 'User':
+                activity_type = 'Mention'
+            tags.append(activitypub.Link(
+                href=item.remote_id,
+                name=getattr(item, item.name_field),
+                type=activity_type
+            ))
+        return tags
+
+    def field_from_activity(self, value):
+        if not isinstance(value, list):
+            return None
+        items = []
+        for link_json in value:
+            link = activitypub.Link(**link_json)
+            tag_type = link.type if link.type != 'Mention' else 'Person'
+            if tag_type != self.related_model.activity_serializer.type:
+                # tags can contain multiple types
+                continue
+            items.append(
+                activitypub.resolve_remote_id(self.related_model, link.href)
+            )
+        return items
+
+
+def image_serializer(value):
+    ''' helper for serializing images '''
+    if value and hasattr(value, 'url'):
+        url = value.url
+    else:
+        return None
+    url = 'https://%s%s' % (DOMAIN, url)
+    return activitypub.Image(url=url)
+
+
+class ImageField(ActivitypubFieldMixin, models.ImageField):
+    ''' activitypub-aware image field '''
+    # pylint: disable=arguments-differ
+    def set_field_from_activity(self, instance, data, save=True):
+        ''' helper function for assinging a value to the field '''
+        value = getattr(data, self.get_activitypub_field())
+        formatted = self.field_from_activity(value)
+        if formatted is None or formatted is MISSING:
+            return
+        getattr(instance, self.name).save(*formatted, save=save)
+
+
+    def field_to_activity(self, value):
+        return image_serializer(value)
+
+
+    def field_from_activity(self, value):
+        image_slug = value
+        # when it's an inline image (User avatar/icon, Book cover), it's a json
+        # blob, but when it's an attached image, it's just a url
+        if isinstance(image_slug, dict):
+            url = image_slug.get('url')
+        elif isinstance(image_slug, str):
+            url = image_slug
+        else:
+            return None
+
+        try:
+            validate_remote_id(url)
+        except ValidationError:
+            return None
+
+        response = get_image(url)
+        if not response:
+            return None
+
+        image_name = str(uuid4()) + '.' + url.split('.')[-1]
+        image_content = ContentFile(response.content)
+        return [image_name, image_content]
+
+
+class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
+    ''' activitypub-aware datetime field '''
+    def field_to_activity(self, value):
+        if not value:
+            return None
+        return value.isoformat()
+
+    def field_from_activity(self, value):
+        try:
+            date_value = dateutil.parser.parse(value)
+            try:
+                return timezone.make_aware(date_value)
+            except ValueError:
+                return date_value
+        except (ParserError, TypeError):
+            return None
+
+class ArrayField(ActivitypubFieldMixin, DjangoArrayField):
+    ''' activitypub-aware array field '''
+    def field_to_activity(self, value):
+        return [str(i) for i in value]
+
+class CharField(ActivitypubFieldMixin, models.CharField):
+    ''' activitypub-aware char field '''
+
+class TextField(ActivitypubFieldMixin, models.TextField):
+    ''' activitypub-aware text field '''
+
+class BooleanField(ActivitypubFieldMixin, models.BooleanField):
+    ''' activitypub-aware boolean field '''
+
+class IntegerField(ActivitypubFieldMixin, models.IntegerField):
+    ''' activitypub-aware boolean field '''

bookwyrm/models/import_job.py

@@ -2,14 +2,13 @@
 import re
 import dateutil.parser
 
+from django.contrib.postgres.fields import JSONField
 from django.db import models
 from django.utils import timezone
 
 from bookwyrm import books_manager
-from bookwyrm.connectors import ConnectorException
 from bookwyrm.models import ReadThrough, User, Book
-from bookwyrm.utils.fields import JSONField
-from .base_model import PrivacyLevels
+from .fields import PrivacyLevels
 
 
 # Mapping goodreads -> bookwyrm shelf titles.

bookwyrm/models/relationship.py

@@ -2,20 +2,23 @@
 from django.db import models
 
 from bookwyrm import activitypub
-from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel
+from .base_model import ActivitypubMixin, BookWyrmModel
+from . import fields
 
 
 class UserRelationship(ActivitypubMixin, BookWyrmModel):
     ''' many-to-many through table for followers '''
-    user_subject = models.ForeignKey(
+    user_subject = fields.ForeignKey(
         'User',
         on_delete=models.PROTECT,
-        related_name='%(class)s_user_subject'
+        related_name='%(class)s_user_subject',
+        activitypub_field='actor',
     )
-    user_object = models.ForeignKey(
+    user_object = fields.ForeignKey(
         'User',
         on_delete=models.PROTECT,
-        related_name='%(class)s_user_object'
+        related_name='%(class)s_user_object',
+        activitypub_field='object',
     )
 
     class Meta:
@@ -32,14 +35,9 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel):
             )
         ]
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('actor', 'user_subject'),
-        ActivityMapping('object', 'user_object'),
-    ]
     activity_serializer = activitypub.Follow
 
-    def get_remote_id(self, status=None):
+    def get_remote_id(self, status=None):# pylint: disable=arguments-differ
         ''' use shelf identifier in remote_id '''
         status = status or 'follows'
         base_path = self.user_subject.remote_id

bookwyrm/models/shelf.py

@@ -3,19 +3,22 @@ import re
 from django.db import models
 
 from bookwyrm import activitypub
-from .base_model import BookWyrmModel, OrderedCollectionMixin, PrivacyLevels
+from .base_model import BookWyrmModel
+from .base_model import OrderedCollectionMixin
+from . import fields
 
 
 class Shelf(OrderedCollectionMixin, BookWyrmModel):
     ''' a list of books owned by a user '''
-    name = models.CharField(max_length=100)
+    name = fields.CharField(max_length=100)
     identifier = models.CharField(max_length=100)
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='owner')
     editable = models.BooleanField(default=True)
-    privacy = models.CharField(
+    privacy = fields.CharField(
         max_length=255,
         default='public',
-        choices=PrivacyLevels.choices
+        choices=fields.PrivacyLevels.choices
     )
     books = models.ManyToManyField(
         'Edition',
@@ -50,15 +53,20 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
 
 class ShelfBook(BookWyrmModel):
     ''' many to many join table for books and shelves '''
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
-    shelf = models.ForeignKey('Shelf', on_delete=models.PROTECT)
-    added_by = models.ForeignKey(
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='object')
+    shelf = fields.ForeignKey(
+        'Shelf', on_delete=models.PROTECT, activitypub_field='target')
+    added_by = fields.ForeignKey(
         'User',
         blank=True,
         null=True,
-        on_delete=models.PROTECT
+        on_delete=models.PROTECT,
+        activitypub_field='actor'
     )
 
+    activity_serializer = activitypub.AddBook
+
     def to_add_activity(self, user):
         ''' AP for shelving a book'''
         return activitypub.Add(

bookwyrm/models/status.py

@@ -6,26 +6,23 @@ from model_utils.managers import InheritanceManager
 
 from bookwyrm import activitypub
 from .base_model import ActivitypubMixin, OrderedCollectionPageMixin
-from .base_model import ActivityMapping, BookWyrmModel, PrivacyLevels
-from .base_model import tag_formatter, image_attachments_formatter
-
+from .base_model import BookWyrmModel
+from . import fields
+from .fields import image_serializer
 
 class Status(OrderedCollectionPageMixin, BookWyrmModel):
     ''' any post, like a reply to a review, etc '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    content = models.TextField(blank=True, null=True)
-    mention_users = models.ManyToManyField('User', related_name='mention_user')
-    mention_books = models.ManyToManyField(
-        'Edition', related_name='mention_book')
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='attributedTo')
+    content = fields.TextField(blank=True, null=True)
+    mention_users = fields.TagField('User', related_name='mention_user')
+    mention_books = fields.TagField('Edition', related_name='mention_book')
     local = models.BooleanField(default=True)
-    privacy = models.CharField(
-        max_length=255,
-        default='public',
-        choices=PrivacyLevels.choices
-    )
-    sensitive = models.BooleanField(default=False)
-    # the created date can't be this, because of receiving federated posts
-    published_date = models.DateTimeField(default=timezone.now)
+    privacy = fields.PrivacyField(max_length=255)
+    sensitive = fields.BooleanField(default=False)
+    # created date is different than publish date because of federated posts
+    published_date = fields.DateTimeField(
+        default=timezone.now, activitypub_field='published')
     deleted = models.BooleanField(default=False)
     deleted_date = models.DateTimeField(blank=True, null=True)
     favorites = models.ManyToManyField(
@@ -35,88 +32,25 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         through_fields=('status', 'user'),
         related_name='user_favorites'
     )
-    reply_parent = models.ForeignKey(
+    reply_parent = fields.ForeignKey(
         'self',
         null=True,
-        on_delete=models.PROTECT
+        on_delete=models.PROTECT,
+        activitypub_field='inReplyTo',
     )
     objects = InheritanceManager()
 
-    # ---- activitypub serialization settings for this model ----- #
-    @property
-    def ap_to(self):
-        ''' should be related to post privacy I think '''
-        return ['https://www.w3.org/ns/activitystreams#Public']
-
-    @property
-    def ap_cc(self):
-        ''' should be related to post privacy I think '''
-        return [self.user.ap_followers]
-
-    @property
-    def ap_replies(self):
-        ''' structured replies block '''
-        return self.to_replies()
-
-    @property
-    def ap_status_image(self):
-        ''' attach a book cover, if relevent '''
-        if hasattr(self, 'book'):
-            return self.book.ap_cover
-        if self.mention_books.first():
-            return self.mention_books.first().ap_cover
-        return None
-
-
-    shared_mappings = [
-        ActivityMapping('url', 'remote_id', lambda x: None),
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('inReplyTo', 'reply_parent'),
-        ActivityMapping('published', 'published_date'),
-        ActivityMapping('attributedTo', 'user'),
-        ActivityMapping('to', 'ap_to'),
-        ActivityMapping('cc', 'ap_cc'),
-        ActivityMapping('replies', 'ap_replies'),
-        ActivityMapping(
-            'tag', 'mention_books',
-            lambda x: tag_formatter(x, 'title', 'Book'),
-            lambda x: activitypub.tag_formatter(x, 'Book')
-        ),
-        ActivityMapping(
-            'tag', 'mention_users',
-            lambda x: tag_formatter(x, 'username', 'Mention'),
-            lambda x: activitypub.tag_formatter(x, 'Mention')
-        ),
-        ActivityMapping(
-            'attachment', 'attachments',
-            lambda x: image_attachments_formatter(x.all()),
-        )
-    ]
-
-    # serializing to bookwyrm expanded activitypub
-    activity_mappings = shared_mappings + [
-        ActivityMapping('name', 'name'),
-        ActivityMapping('inReplyToBook', 'book'),
-        ActivityMapping('rating', 'rating'),
-        ActivityMapping('quote', 'quote'),
-        ActivityMapping('content', 'content'),
-    ]
-
-    # for serializing to standard activitypub without extended types
-    pure_activity_mappings = shared_mappings + [
-        ActivityMapping('name', 'ap_pure_name'),
-        ActivityMapping('content', 'ap_pure_content'),
-        ActivityMapping('attachment', 'ap_status_image'),
-    ]
-
     activity_serializer = activitypub.Note
+    serialize_reverse_fields = [('attachments', 'attachment')]
+    deserialize_reverse_fields = [('attachments', 'attachment')]
 
-    #----- replies collection activitypub ----#
     @classmethod
     def replies(cls, status):
         ''' load all replies to a status. idk if there's a better way
             to write this so it's just a property '''
-        return cls.objects.filter(reply_parent=status).select_subclasses()
+        return cls.objects.filter(
+            reply_parent=status
+        ).select_subclasses().order_by('published_date')
 
     @property
     def status_type(self):
@@ -131,7 +65,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
             **kwargs
         )
 
-    def to_activity(self, pure=False):
+    def to_activity(self, pure=False):# pylint: disable=arguments-differ
         ''' return tombstone if the status is deleted '''
         if self.deleted:
             return activitypub.Tombstone(
@@ -140,7 +74,24 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
                 deleted=self.deleted_date.isoformat(),
                 published=self.deleted_date.isoformat()
             ).serialize()
-        return ActivitypubMixin.to_activity(self, pure=pure)
+        activity = ActivitypubMixin.to_activity(self)
+        activity['replies'] = self.to_replies()
+
+        # "pure" serialization for non-bookwyrm instances
+        if pure:
+            activity['content'] = self.pure_content
+            if 'name' in activity:
+                activity['name'] = self.pure_name
+            activity['type'] = self.pure_type
+            activity['attachment'] = [
+                image_serializer(b.cover) for b in self.mention_books.all() \
+                        if b.cover]
+            if hasattr(self, 'book'):
+                activity['attachment'].append(
+                    image_serializer(self.book.cover)
+                )
+        return activity
+
 
     def save(self, *args, **kwargs):
         ''' update user active time '''
@@ -153,40 +104,42 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
 class GeneratedNote(Status):
     ''' these are app-generated messages about user activity '''
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
         message = self.content
         books = ', '.join(
-            '<a href="%s">"%s"</a>' % (self.book.remote_id, self.book.title) \
+            '<a href="%s">"%s"</a>' % (book.remote_id, book.title) \
             for book in self.mention_books.all()
         )
-        return '%s %s' % (message, books)
+        return '%s %s %s' % (self.user.display_name, message, books)
 
     activity_serializer = activitypub.GeneratedNote
-    pure_activity_serializer = activitypub.Note
+    pure_type = 'Note'
 
 
 class Comment(Status):
     ''' like a review but without a rating and transient '''
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook')
 
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
         return self.content + '<br><br>(comment on <a href="%s">"%s"</a>)' % \
                 (self.book.remote_id, self.book.title)
 
     activity_serializer = activitypub.Comment
-    pure_activity_serializer = activitypub.Note
+    pure_type = 'Note'
 
 
 class Quotation(Status):
     ''' like a review but without a rating and transient '''
-    quote = models.TextField()
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
+    quote = fields.TextField()
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook')
 
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
         return '"%s"<br>-- <a href="%s">"%s"</a><br><br>%s' % (
             self.quote,
@@ -196,14 +149,15 @@ class Quotation(Status):
         )
 
     activity_serializer = activitypub.Quotation
-    pure_activity_serializer = activitypub.Note
+    pure_type = 'Note'
 
 
 class Review(Status):
     ''' a book review '''
-    name = models.CharField(max_length=255, null=True)
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
-    rating = models.IntegerField(
+    name = fields.CharField(max_length=255, null=True)
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook')
+    rating = fields.IntegerField(
         default=None,
         null=True,
         blank=True,
@@ -211,9 +165,10 @@ class Review(Status):
     )
 
     @property
-    def ap_pure_name(self):
+    def pure_name(self):
         ''' clarify review names for mastodon serialization '''
         if self.rating:
+            #pylint: disable=bad-string-format-type
             return 'Review of "%s" (%d stars): %s' % (
                 self.book.title,
                 self.rating,
@@ -225,26 +180,21 @@ class Review(Status):
         )
 
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
         return self.content + '<br><br>(<a href="%s">"%s"</a>)' % \
                 (self.book.remote_id, self.book.title)
 
     activity_serializer = activitypub.Review
-    pure_activity_serializer = activitypub.Article
+    pure_type = 'Article'
 
 
 class Favorite(ActivitypubMixin, BookWyrmModel):
     ''' fav'ing a post '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    status = models.ForeignKey('Status', on_delete=models.PROTECT)
-
-    # ---- activitypub serialization settings for this model ----- #
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('actor', 'user'),
-        ActivityMapping('object', 'status'),
-    ]
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='actor')
+    status = fields.ForeignKey(
+        'Status', on_delete=models.PROTECT, activitypub_field='object')
 
     activity_serializer = activitypub.Like
 
@@ -254,7 +204,6 @@ class Favorite(ActivitypubMixin, BookWyrmModel):
         self.user.save()
         super().save(*args, **kwargs)
 
-
     class Meta:
         ''' can't fav things twice '''
         unique_together = ('user', 'status')
@@ -262,16 +211,12 @@ class Favorite(ActivitypubMixin, BookWyrmModel):
 
 class Boost(Status):
     ''' boost'ing a post '''
-    boosted_status = models.ForeignKey(
+    boosted_status = fields.ForeignKey(
         'Status',
         on_delete=models.PROTECT,
-        related_name="boosters")
-
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('actor', 'user'),
-        ActivityMapping('object', 'boosted_status'),
-    ]
+        related_name='boosters',
+        activitypub_field='object',
+    )
 
     activity_serializer = activitypub.Boost
 

bookwyrm/models/tag.py

@@ -6,13 +6,12 @@ from django.db import models
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
 from .base_model import OrderedCollectionMixin, BookWyrmModel
+from . import fields
 
 
 class Tag(OrderedCollectionMixin, BookWyrmModel):
     ''' freeform tags for books '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
-    name = models.CharField(max_length=100)
+    name = fields.CharField(max_length=100, unique=True)
     identifier = models.CharField(max_length=100)
 
     @classmethod
@@ -30,6 +29,26 @@ class Tag(OrderedCollectionMixin, BookWyrmModel):
         base_path = 'https://%s' % DOMAIN
         return '%s/tag/%s' % (base_path, self.identifier)
 
+
+    def save(self, *args, **kwargs):
+        ''' create a url-safe lookup key for the tag '''
+        if not self.id:
+            # add identifiers to new tags
+            self.identifier = urllib.parse.quote_plus(self.name)
+        super().save(*args, **kwargs)
+
+
+class UserTag(BookWyrmModel):
+    ''' an instance of a tag on a book by a user '''
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='actor')
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='object')
+    tag = fields.ForeignKey(
+        'Tag', on_delete=models.PROTECT, activitypub_field='target')
+
+    activity_serializer = activitypub.AddBook
+
     def to_add_activity(self, user):
         ''' AP for shelving a book'''
         return activitypub.Add(
@@ -48,13 +67,7 @@ class Tag(OrderedCollectionMixin, BookWyrmModel):
             target=self.to_activity(),
         ).serialize()
 
-    def save(self, *args, **kwargs):
-        ''' create a url-safe lookup key for the tag '''
-        if not self.id:
-            # add identifiers to new tags
-            self.identifier = urllib.parse.quote_plus(self.name)
-        super().save(*args, **kwargs)
 
     class Meta:
         ''' unqiueness constraint '''
-        unique_together = ('user', 'book', 'name')
+        unique_together = ('user', 'book', 'tag')

bookwyrm/models/user.py

@@ -6,44 +6,61 @@ from django.db import models
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
+from bookwyrm.connectors import get_data
 from bookwyrm.models.shelf import Shelf
-from bookwyrm.models.status import Status
+from bookwyrm.models.status import Status, Review
 from bookwyrm.settings import DOMAIN
 from bookwyrm.signatures import create_key_pair
-from .base_model import ActivityMapping, OrderedCollectionPageMixin
-from .base_model import image_formatter
+from bookwyrm.tasks import app
+from .base_model import OrderedCollectionPageMixin
+from .base_model import ActivitypubMixin, BookWyrmModel
+from .federated_server import FederatedServer
+from . import fields
 
 
 class User(OrderedCollectionPageMixin, AbstractUser):
     ''' a user who wants to read books '''
-    private_key = models.TextField(blank=True, null=True)
-    public_key = models.TextField(blank=True, null=True)
-    inbox = models.CharField(max_length=255, unique=True)
-    shared_inbox = models.CharField(max_length=255, blank=True, null=True)
+    username = fields.UsernameField()
+
+    key_pair = fields.OneToOneField(
+        'KeyPair',
+        on_delete=models.CASCADE,
+        blank=True, null=True,
+        activitypub_field='publicKey',
+        related_name='owner'
+    )
+    inbox = fields.RemoteIdField(unique=True)
+    shared_inbox = fields.RemoteIdField(
+        activitypub_field='sharedInbox',
+        activitypub_wrapper='endpoints',
+        deduplication_field=False,
+        null=True)
     federated_server = models.ForeignKey(
         'FederatedServer',
         on_delete=models.PROTECT,
         null=True,
         blank=True,
     )
-    outbox = models.CharField(max_length=255, unique=True)
-    summary = models.TextField(blank=True, null=True)
-    local = models.BooleanField(default=True)
-    bookwyrm_user = models.BooleanField(default=True)
+    outbox = fields.RemoteIdField(unique=True)
+    summary = fields.TextField(default='')
+    local = models.BooleanField(default=False)
+    bookwyrm_user = fields.BooleanField(default=True)
     localname = models.CharField(
         max_length=255,
         null=True,
         unique=True
     )
     # name is your display name, which you can change at will
-    name = models.CharField(max_length=100, blank=True, null=True)
-    avatar = models.ImageField(upload_to='avatars/', blank=True, null=True)
-    following = models.ManyToManyField(
+    name = fields.CharField(max_length=100, default='')
+    avatar = fields.ImageField(
+        upload_to='avatars/', blank=True, null=True, activitypub_field='icon')
+    followers = fields.ManyToManyField(
         'self',
+        link_only=True,
         symmetrical=False,
         through='UserFollows',
-        through_fields=('user_subject', 'user_object'),
-        related_name='followers'
+        through_fields=('user_object', 'user_subject'),
+        related_name='following'
     )
     follow_requests = models.ManyToManyField(
         'self',
@@ -66,60 +83,20 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         through_fields=('user', 'status'),
         related_name='favorite_statuses'
     )
-    remote_id = models.CharField(max_length=255, null=True, unique=True)
+    remote_id = fields.RemoteIdField(
+        null=True, unique=True, activitypub_field='id')
     created_date = models.DateTimeField(auto_now_add=True)
     updated_date = models.DateTimeField(auto_now=True)
     last_active_date = models.DateTimeField(auto_now=True)
-    manually_approves_followers = models.BooleanField(default=False)
-
-    # ---- activitypub serialization settings for this model ----- #
-    @property
-    def ap_followers(self):
-        ''' generates url for activitypub followers page '''
-        return '%s/followers' % self.remote_id
+    manually_approves_followers = fields.BooleanField(default=False)
 
     @property
-    def ap_public_key(self):
-        ''' format the public key block for activitypub '''
-        return activitypub.PublicKey(**{
-            'id': '%s/#main-key' % self.remote_id,
-            'owner': self.remote_id,
-            'publicKeyPem': self.public_key,
-        })
+    def display_name(self):
+        ''' show the cleanest version of the user's name possible '''
+        if self.name != '':
+            return self.name
+        return self.localname or self.username
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping(
-            'preferredUsername',
-            'username',
-            activity_formatter=lambda x: x.split('@')[0]
-        ),
-        ActivityMapping('name', 'name'),
-        ActivityMapping('bookwyrmUser', 'bookwyrm_user'),
-        ActivityMapping('inbox', 'inbox'),
-        ActivityMapping('outbox', 'outbox'),
-        ActivityMapping('followers', 'ap_followers'),
-        ActivityMapping('summary', 'summary'),
-        ActivityMapping(
-            'publicKey',
-            'public_key',
-            model_formatter=lambda x: x.get('publicKeyPem')
-        ),
-        ActivityMapping('publicKey', 'ap_public_key'),
-        ActivityMapping(
-            'endpoints',
-            'shared_inbox',
-            activity_formatter=lambda x: {'sharedInbox': x},
-            model_formatter=lambda x: x.get('sharedInbox')
-        ),
-        ActivityMapping('icon', 'avatar'),
-        ActivityMapping(
-            'manuallyApprovesFollowers',
-            'manually_approves_followers'
-        ),
-        # this field isn't in the activity but should always be false
-        ActivityMapping(None, 'local', model_formatter=lambda x: False),
-    ]
     activity_serializer = activitypub.Person
 
     def to_outbox(self, **kwargs):
@@ -127,23 +104,23 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         queryset = Status.objects.filter(
             user=self,
             deleted=False,
-        ).select_subclasses()
+        ).select_subclasses().order_by('-published_date')
         return self.to_ordered_collection(queryset, \
                 remote_id=self.outbox, **kwargs)
 
     def to_following_activity(self, **kwargs):
         ''' activitypub following list '''
         remote_id = '%s/following' % self.remote_id
-        return self.to_ordered_collection(self.following, \
+        return self.to_ordered_collection(self.following.all(), \
                 remote_id=remote_id, id_only=True, **kwargs)
 
     def to_followers_activity(self, **kwargs):
         ''' activitypub followers list '''
         remote_id = '%s/followers' % self.remote_id
-        return self.to_ordered_collection(self.followers, \
+        return self.to_ordered_collection(self.followers.all(), \
                 remote_id=remote_id, id_only=True, **kwargs)
 
-    def to_activity(self, pure=False):
+    def to_activity(self):
         ''' override default AP serializer to add context object
             idk if this is the best way to go about this '''
         activity_object = super().to_activity()
@@ -180,18 +157,53 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         self.inbox = '%s/inbox' % self.remote_id
         self.shared_inbox = 'https://%s/inbox' % DOMAIN
         self.outbox = '%s/outbox' % self.remote_id
-        if not self.private_key:
-            self.private_key, self.public_key = create_key_pair()
 
         return super().save(*args, **kwargs)
 
 
+class KeyPair(ActivitypubMixin, BookWyrmModel):
+    ''' public and private keys for a user '''
+    private_key = models.TextField(blank=True, null=True)
+    public_key = fields.TextField(
+        blank=True, null=True, activitypub_field='publicKeyPem')
+
+    activity_serializer = activitypub.PublicKey
+    serialize_reverse_fields = [('owner', 'owner')]
+
+    def get_remote_id(self):
+        # self.owner is set by the OneToOneField on User
+        return '%s/#main-key' % self.owner.remote_id
+
+    def save(self, *args, **kwargs):
+        ''' create a key pair '''
+        if not self.public_key:
+            self.private_key, self.public_key = create_key_pair()
+        return super().save(*args, **kwargs)
+
+    def to_activity(self):
+        ''' override default AP serializer to add context object
+            idk if this is the best way to go about this '''
+        activity_object = super().to_activity()
+        del activity_object['@context']
+        del activity_object['type']
+        return activity_object
+
+
 @receiver(models.signals.post_save, sender=User)
+#pylint: disable=unused-argument
 def execute_after_save(sender, instance, created, *args, **kwargs):
     ''' create shelves for new users '''
-    if not instance.local or not created:
+    if not created:
         return
 
+    if not instance.local:
+        set_remote_server.delay(instance.id)
+        return
+
+    instance.key_pair = KeyPair.objects.create(
+        remote_id='%s/#main-key' % instance.remote_id)
+    instance.save()
+
     shelves = [{
         'name': 'To Read',
         'identifier': 'to-read',
@@ -210,3 +222,54 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
             user=instance,
             editable=False
         ).save()
+
+
+@app.task
+def set_remote_server(user_id):
+    ''' figure out the user's remote server in the background '''
+    user = User.objects.get(id=user_id)
+    actor_parts = urlparse(user.remote_id)
+    user.federated_server = \
+        get_or_create_remote_server(actor_parts.netloc)
+    user.save()
+    if user.bookwyrm_user:
+        get_remote_reviews.delay(user.outbox)
+
+
+def get_or_create_remote_server(domain):
+    ''' get info on a remote server '''
+    try:
+        return FederatedServer.objects.get(
+            server_name=domain
+        )
+    except FederatedServer.DoesNotExist:
+        pass
+
+    data = get_data('https://%s/.well-known/nodeinfo' % domain)
+
+    try:
+        nodeinfo_url = data.get('links')[0].get('href')
+    except (TypeError, KeyError):
+        return None
+
+    data = get_data(nodeinfo_url)
+
+    server = FederatedServer.objects.create(
+        server_name=domain,
+        application_type=data['software']['name'],
+        application_version=data['software']['version'],
+    )
+    return server
+
+
+@app.task
+def get_remote_reviews(outbox):
+    ''' ingest reviews by a new remote bookwyrm user '''
+    outbox_page = outbox + '?page=true'
+    data = get_data(outbox_page)
+
+    # TODO: pagination?
+    for activity in data['orderedItems']:
+        if not activity['type'] == 'Review':
+            continue
+        activitypub.Review(**activity).to_model(Review)

bookwyrm/outgoing.py

@@ -4,15 +4,15 @@ import re
 from django.db import IntegrityError, transaction
 from django.http import HttpResponseNotFound, JsonResponse
 from django.views.decorators.csrf import csrf_exempt
-import requests
+from requests import HTTPError
 
 from bookwyrm import activitypub
 from bookwyrm import models
+from bookwyrm.connectors import get_data, ConnectorException
 from bookwyrm.broadcast import broadcast
 from bookwyrm.status import create_notification
 from bookwyrm.status import create_generated_note
 from bookwyrm.status import delete_status
-from bookwyrm.remote_user import get_or_create_remote_user
 from bookwyrm.settings import DOMAIN
 from bookwyrm.utils import regex
 
@@ -54,16 +54,16 @@ def handle_remote_webfinger(query):
         url = 'https://%s/.well-known/webfinger?resource=acct:%s' % \
             (domain, query)
         try:
-            response = requests.get(url)
-        except requests.exceptions.ConnectionError:
+            data = get_data(url)
+        except (ConnectorException, HTTPError):
             return None
-        if not response.ok:
-            return None
-        data = response.json()
-        for link in data['links']:
-            if link['rel'] == 'self':
+
+        for link in data.get('links'):
+            if link.get('rel') == 'self':
                 try:
-                    user = get_or_create_remote_user(link['href'])
+                    user = activitypub.resolve_remote_id(
+                        models.User, link['href']
+                    )
                 except KeyError:
                     return None
     return user

bookwyrm/remote_user.py

@@ -1,111 +0,0 @@
-''' manage remote users '''
-from urllib.parse import urlparse
-import requests
-
-from django.db import transaction
-
-from bookwyrm import activitypub, models
-from bookwyrm import status as status_builder
-from bookwyrm.tasks import app
-
-
-def get_or_create_remote_user(actor):
-    ''' look up a remote user or add them '''
-    try:
-        return models.User.objects.get(remote_id=actor)
-    except models.User.DoesNotExist:
-        pass
-
-    data = fetch_user_data(actor)
-
-    actor_parts = urlparse(actor)
-    with transaction.atomic():
-        user = activitypub.Person(**data).to_model(models.User)
-        user.federated_server = get_or_create_remote_server(actor_parts.netloc)
-        user.save()
-    if user.bookwyrm_user:
-        get_remote_reviews.delay(user.id)
-    return user
-
-
-def fetch_user_data(actor):
-    ''' load the user's info from the actor url '''
-    try:
-        response = requests.get(
-            actor,
-            headers={'Accept': 'application/activity+json'}
-        )
-    except ConnectionError:
-        return None
-
-    if not response.ok:
-        response.raise_for_status()
-    data = response.json()
-
-    # make sure our actor is who they say they are
-    if actor != data['id']:
-        raise ValueError("Remote actor id must match url.")
-    return data
-
-
-def refresh_remote_user(user):
-    ''' get updated user data from its home instance '''
-    data = fetch_user_data(user.remote_id)
-
-    activity = activitypub.Person(**data)
-    activity.to_model(models.User, instance=user)
-
-
-@app.task
-def get_remote_reviews(user_id):
-    ''' ingest reviews by a new remote bookwyrm user '''
-    try:
-        user = models.User.objects.get(id=user_id)
-    except models.User.DoesNotExist:
-        return
-    outbox_page = user.outbox + '?page=true'
-    response = requests.get(
-        outbox_page,
-        headers={'Accept': 'application/activity+json'}
-    )
-    data = response.json()
-    # TODO: pagination?
-    for activity in data['orderedItems']:
-        status_builder.create_status(activity)
-
-
-def get_or_create_remote_server(domain):
-    ''' get info on a remote server '''
-    try:
-        return models.FederatedServer.objects.get(
-            server_name=domain
-        )
-    except models.FederatedServer.DoesNotExist:
-        pass
-
-    response = requests.get(
-        'https://%s/.well-known/nodeinfo' % domain,
-        headers={'Accept': 'application/activity+json'}
-    )
-
-    if response.status_code != 200:
-        return None
-
-    data = response.json()
-    try:
-        nodeinfo_url = data.get('links')[0].get('href')
-    except (TypeError, KeyError):
-        return None
-
-    response = requests.get(
-        nodeinfo_url,
-        headers={'Accept': 'application/activity+json'}
-    )
-    data = response.json()
-
-    server = models.FederatedServer.objects.create(
-        server_name=domain,
-        application_type=data['software']['name'],
-        application_version=data['software']['version'],
-    )
-    return server

bookwyrm/routine_book_tasks.py

@@ -1,16 +0,0 @@
-''' Routine tasks for keeping your library tidy '''
-from datetime import timedelta
-from django.utils import timezone
-from bookwyrm import books_manager
-from bookwyrm import models
-
-def sync_book_data():
-    ''' update books with any changes to their canonical source '''
-    expiry = timezone.now() - timedelta(days=1)
-    books = models.Edition.objects.filter(
-        sync=True,
-        last_sync_date__lte=expiry
-    ).all()
-    for book in books:
-        # TODO: create background tasks
-        books_manager.update_book(book)

bookwyrm/settings.py

@@ -99,10 +99,6 @@ BOOKWYRM_DBS = {
         'HOST': env('POSTGRES_HOST', ''),
         'PORT': 5432
     },
-    'sqlite': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'fedireads.db')
-    }
 }
 
 DATABASES = {

bookwyrm/signatures.py

@@ -31,7 +31,7 @@ def make_signature(sender, destination, date, digest):
         'digest: %s' % digest,
     ]
     message_to_sign = '\n'.join(signature_headers)
-    signer = pkcs1_15.new(RSA.import_key(sender.private_key))
+    signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key))
     signed_message = signer.sign(SHA256.new(message_to_sign.encode('utf8')))
     signature = {
         'keyId': '%s#main-key' % sender.remote_id,

bookwyrm/status.py

@@ -1,7 +1,7 @@
 ''' Handle user activity '''
 from django.utils import timezone
 
-from bookwyrm import activitypub, books_manager, models
+from bookwyrm import models
 from bookwyrm.sanitize_html import InputHtmlParser
 
 
@@ -12,37 +12,6 @@ def delete_status(status):
     status.save()
 
 
-def create_status(activity):
-    ''' unfortunately, it's not QUITE as simple as deserializing it '''
-    # render the json into an activity object
-    serializer = activitypub.activity_objects[activity['type']]
-    activity = serializer(**activity)
-    try:
-        model = models.activity_models[activity.type]
-    except KeyError:
-        # not a type of status we are prepared to deserialize
-        return None
-
-    # ignore notes that aren't replies to known statuses
-    if activity.type == 'Note':
-        reply = models.Status.objects.filter(
-            remote_id=activity.inReplyTo
-        ).first()
-        if not reply:
-            return None
-
-    # look up books
-    book_urls = []
-    if hasattr(activity, 'inReplyToBook'):
-        book_urls.append(activity.inReplyToBook)
-    if hasattr(activity, 'tag'):
-        book_urls += [t['href'] for t in activity.tag if t['type'] == 'Book']
-    for remote_id in book_urls:
-        books_manager.get_or_create_book(remote_id)
-
-    return activity.to_model(model)
-
-
 def create_generated_note(user, content, mention_books=None, privacy='public'):
     ''' a note created by the app about user activity '''
     # sanitize input html

bookwyrm/templates/author.html

@@ -1,8 +1,8 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
-    <h1 class="title">{{ author.display_name }}</h1>
+    <h1 class="title">{{ author.name }}</h1>
 
     {% if author.bio %}
     <p>
@@ -12,7 +12,7 @@
 </div>
 
 <div class="block">
-    <h3 class="title is-4">Books by {{ author.display_name }}</h3>
+    <h3 class="title is-4">Books by {{ author.name }}</h3>
     {% include 'snippets/book_tiles.html' with books=books %}
 </div>
 {% endblock %}

bookwyrm/templates/book.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% load humanize %}
 {% block content %}
 
@@ -86,8 +86,8 @@
                 {% endif %}
 
 
-                {% if book.parent_work.edition_set.count > 1 %}
-                <p><a href="/book/{{ book.parent_work.id }}/editions">{{ book.parent_work.edition_set.count }} editions</a></p>
+                {% if book.parent_work.editions.count > 1 %}
+                <p><a href="/book/{{ book.parent_work.id }}/editions">{{ book.parent_work.editions.count }} editions</a></p>
                 {% endif %}
             </div>
 

bookwyrm/templates/direct_messages.html

@@ -0,0 +1,37 @@
+{% extends 'layout.html' %}
+{% block content %}
+
+<div class="block">
+    <h1 class="title">Direct Messages</h1>
+
+    {% if not activities %}
+    <p>You have no messages right now.</p>
+    {% endif %}
+    {% for activity in activities %}
+    <div class="block">
+    {% include 'snippets/status.html' with status=activity %}
+    </div>
+    {% endfor %}
+
+    <nav class="pagination" role="navigation" aria-label="pagination">
+        {% if prev %}
+        <p class="pagination-previous">
+            <a href="{{ prev }}">
+                <span class="icon icon-arrow-left"></span>
+                Previous
+            </a>
+        </p>
+        {% endif %}
+
+        {% if next %}
+        <p class="pagination-next">
+            <a href="{{ next }}">
+                Next
+                <span class="icon icon-arrow-right"></span>
+            </a>
+        </p>
+        {% endif %}
+    </nav>
+</div>
+
+{% endblock %}

bookwyrm/templates/editions.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">Editions of <a href="/book/{{ work.id }}">"{{ work.title }}"</a></h1>

bookwyrm/templates/feed.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 
 <div class="columns">

bookwyrm/templates/followers.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">

bookwyrm/templates/following.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">

bookwyrm/templates/import_status.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% load humanize %}
 {% block content %}
 <div class="block">

bookwyrm/templates/layout.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
@@ -68,6 +68,9 @@
                     {% include 'snippets/username.html' with user=request.user %}
                 </p></div>
                 <div class="navbar-dropdown">
+                    <a href="/direct-messages" class="navbar-item">
+                        Direct messages
+                    </a>
                     <a href="/user/{{request.user.localname}}" class="navbar-item">
                         Profile
                     </a>

bookwyrm/templates/notifications.html

@@ -1,6 +1,6 @@
 {% extends 'layout.html' %}
 {% load humanize %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">Notifications</h1>

bookwyrm/templates/shelf.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 
 <div class="columns">

bookwyrm/templates/snippets/authors.html

@@ -1 +1 @@
-<a href="/author/{{ book.authors.first.id }}" class="author">{{ book.authors.first.display_name }}</a>
+<a href="/author/{{ book.authors.first.id }}" class="author">{{ book.authors.first.name }}</a>

bookwyrm/templates/snippets/avatar.html

@@ -1,3 +1,3 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <img class="avatar image {% if large %}is-96x96{% else %}is-32x32{% endif %}" src="{% if user.avatar %}/images/{{ user.avatar }}{% else %}/static/images/default_avi.jpg{% endif %}" alt="avatar for {{ user|username }}">
 

bookwyrm/templates/snippets/book_cover.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="cover-container is-{{ size }}">
 {% if book.cover %}
     <img class="book-cover" src="/images/{{ book.cover }}" alt="{% include 'snippets/cover_alt.html' with book=book %}">

bookwyrm/templates/snippets/book_preview.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="columns">
     <div class="column is-narrow">
         <div>

bookwyrm/templates/snippets/boost_button.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% with status.id|uuid as uuid %}
 <form name="boost" action="/boost/{{ status.id }}" method="post" onsubmit="return interact(event)" class="boost-{{ status.id }}-{{ uuid }} {% if request.user|boosted:status  %}hidden{% endif %}" data-id="boost-{{ status.id }}-{{ uuid }}">
     {% csrf_token %}

bookwyrm/templates/snippets/cover_alt.html

@@ -1,2 +1,2 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 '{{ book.title }}' Cover ({{ book|edition_info }})

bookwyrm/templates/snippets/create_status.html

@@ -1,5 +1,5 @@
 {% load humanize %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 
 <div class="tabs is-boxed">
     <ul role="tablist">

bookwyrm/templates/snippets/fav_button.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% with status.id|uuid as uuid %}
 <form name="favorite" action="/favorite/{{ status.id }}" method="POST" onsubmit="return interact(event)" class="fav-{{ status.id }}-{{ uuid }} {% if request.user|liked:status %}hidden{% endif %}" data-id="fav-{{ status.id }}-{{ uuid }}">
     {% csrf_token %}

bookwyrm/templates/snippets/finish_reading_modal.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div>
     <input class="toggle-control" type="checkbox" name="finish-reading-{{ uuid }}" id="finish-reading-{{ uuid }}">
     <div class="modal toggle-content hidden">

bookwyrm/templates/snippets/follow_request_buttons.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% if request.user|follow_request_exists:user %}
 <form action="/accept-follow-request/" method="POST">
     {% csrf_token %}

bookwyrm/templates/snippets/privacy_select.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="select">
     {% with 0|uuid as uuid %}
     {% if not no_label %}

bookwyrm/templates/snippets/rate_action.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <span class="is-sr-only">Leave a rating</span>
 <div class="field is-grouped stars rate-stars">
     {% for i in '12345'|make_list %}

bookwyrm/templates/snippets/reply_form.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% with activity.id|uuid as uuid %}
 <form class="is-flex-grow-1" name="reply" action="/reply" method="post" onsubmit="return reply(event)">
     <div class="columns">
@@ -13,7 +13,7 @@
 
         <div class="column is-narrow">
             <div class="field">
-                {% include 'snippets/privacy_select.html' %}
+                {% include 'snippets/privacy_select.html' with current=activity.privacy %}
             </div>
             <div class="field">
                 <button class="button is-primary" type="submit">

bookwyrm/templates/snippets/shelf.html

@@ -1,5 +1,5 @@
 {% load humanize %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% if shelf.books.all|length > 0 %}
 <table class="table is-striped is-fullwidth">
 
@@ -43,7 +43,7 @@
         <a href="/book/{{ book.id }}">{{ book.title }}</a>
     </td>
     <td>
-        {{ book.authors.first.display_name }}
+        {{ book.authors.first.name }}
     </td>
     <td>
         {% if book.first_published_date %}{{ book.first_published_date }}{% endif %}

bookwyrm/templates/snippets/shelve_button.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% if request.user.is_authenticated %}
 
 {% with book.id|uuid as uuid %}

bookwyrm/templates/snippets/status.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% if not status.deleted %}
     {% if status.status_type == 'Boost' %}
         {% include 'snippets/avatar.html' with user=status.user %}

bookwyrm/templates/snippets/status_body.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% load humanize %}
 
 {% if not status.deleted %}

bookwyrm/templates/snippets/status_content.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="block">
     {% if status.status_type == 'Review' %}
     <h3>

bookwyrm/templates/snippets/status_header.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% include 'snippets/avatar.html' with user=status.user %}
 {% include 'snippets/username.html' with user=status.user %}
 

bookwyrm/templates/snippets/tag.html

@@ -1,14 +1,14 @@
 <div class="control">
-    <form name="tag" action="/{% if tag.identifier in user_tags %}untag{% else %}tag{% endif %}/" method="post">
+    <form name="tag" action="/{% if tag.tag.identifier in user_tags %}untag{% else %}tag{% endif %}/" method="post">
         {% csrf_token %}
         <input type="hidden" name="book" value="{{ book.id }}">
-        <input type="hidden" name="name" value="{{ tag.name }}">
+        <input type="hidden" name="name" value="{{ tag.tag.name }}">
 
         <div class="tags has-addons">
-            <a class="tag" href="/tag/{{ tag.identifier|urlencode }}">
-                {{ tag.name }}
+            <a class="tag" href="/tag/{{ tag.tag.identifier|urlencode }}">
+                {{ tag.tag.name }}
             </a>
-            {% if tag.identifier in user_tags %}
+            {% if tag.tag.identifier in user_tags %}
             <button class="tag is-delete" type="submit">
                 <span class="is-sr-only">remove tag</span>
             </button>

bookwyrm/templates/snippets/thread.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="block">
 
 {% with depth=depth|add:1 %}

bookwyrm/templates/snippets/trimmed_text.html

@@ -1,8 +1,8 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% with 0|uuid as uuid %}
 {% if full %}
 
-{% with full|text_overflow as trimmed %}
+{% with full|truncatewords_html:60 as trimmed %}
 {% if trimmed != full %}
 <div>
     <input type="radio" name="show-hide-{{ uuid }}" id="show-{{ uuid }}" class="toggle-control" checked>

bookwyrm/templates/snippets/user_header.html

@@ -1,5 +1,5 @@
 {% load humanize %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="block">
     <div class="columns">
         <div class="column is-narrow">

bookwyrm/templates/snippets/username.html

@@ -1,2 +1,2 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <a href="/user/{{ user | username }}" class="user">{% if user.name %}{{ user.name }}{% else %}{{ user | username }}{% endif %}</a>{% if possessive %}'s{% endif %}{% if show_full and user.name or show_full and user.localname %} ({{ user.username }}){% endif %}

bookwyrm/templates/tag.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 
 <div class="block">

bookwyrm/templatetags/bookwyrm_tags.py

@@ -116,20 +116,6 @@ def get_book_description(book):
     ''' use the work's text if the book doesn't have it '''
     return book.description or book.parent_work.description
 
-@register.filter(name='text_overflow')
-def text_overflow(text):
-    ''' dont' let book descriptions run for ages '''
-    if not text:
-        return ''
-    char_max = 400
-    if text and len(text) < char_max:
-        return text
-
-    trimmed = text[:char_max]
-    # go back to the last space
-    trimmed = ' '.join(trimmed.split(' ')[:-1])
-    return trimmed + '...'
-
 
 @register.filter(name='uuid')
 def get_uuid(identifier):
@@ -146,6 +132,8 @@ def time_since(date):
     delta = now - date
 
     if date < (now - relativedelta(weeks=1)):
+        if date.year != now.year:
+            return date.strftime('%b %-d %Y')
         return date.strftime('%b %-d')
     delta = relativedelta(now, date)
     if delta.days:
@@ -160,7 +148,6 @@ def time_since(date):
 @register.simple_tag(takes_context=True)
 def active_shelf(context, book):
     ''' check what shelf a user has a book on, if any '''
-    #TODO: books can be on multiple shelves, handle that better
     shelf = models.ShelfBook.objects.filter(
         shelf__user=context['request'].user,
         book=book

bookwyrm/tests/activitypub/test_author.py

@@ -12,8 +12,6 @@ class Author(TestCase):
         )
         self.author = models.Author.objects.create(
             name='Author fullname',
-            first_name='Auth',
-            last_name='Or',
             aliases=['One', 'Two'],
             bio='bio bio bio',
         )

bookwyrm/tests/activitypub/test_base_activity.py

@@ -0,0 +1,262 @@
+''' tests the base functionality for activitypub dataclasses '''
+from io import BytesIO
+import json
+import pathlib
+from unittest.mock import patch
+
+from dataclasses import dataclass
+from django.test import TestCase
+from PIL import Image
+import responses
+
+from bookwyrm import activitypub
+from bookwyrm.activitypub.base_activity import ActivityObject, \
+    resolve_remote_id, set_related_field
+from bookwyrm.activitypub import ActivitySerializerError
+from bookwyrm import models
+
+class BaseActivity(TestCase):
+    ''' the super class for model-linked activitypub dataclasses '''
+    def setUp(self):
+        ''' we're probably going to re-use this so why copy/paste '''
+        self.user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        self.user.remote_id = 'http://example.com/a/b'
+        self.user.save()
+
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json'
+        )
+        self.userdata = json.loads(datafile.read_bytes())
+        # don't try to load the user icon
+        del self.userdata['icon']
+
+        image_file = pathlib.Path(__file__).parent.joinpath(
+            '../../static/images/default_avi.jpg')
+        image = Image.open(image_file)
+        output = BytesIO()
+        image.save(output, format=image.format)
+        self.image_data = output.getvalue()
+
+    def test_init(self):
+        ''' simple successfuly init '''
+        instance = ActivityObject(id='a', type='b')
+        self.assertTrue(hasattr(instance, 'id'))
+        self.assertTrue(hasattr(instance, 'type'))
+
+    def test_init_missing(self):
+        ''' init with missing required params '''
+        with self.assertRaises(ActivitySerializerError):
+            ActivityObject()
+
+    def test_init_extra_fields(self):
+        ''' init ignoring additional fields '''
+        instance = ActivityObject(id='a', type='b', fish='c')
+        self.assertTrue(hasattr(instance, 'id'))
+        self.assertTrue(hasattr(instance, 'type'))
+
+    def test_init_default_field(self):
+        ''' replace an existing required field with a default field '''
+        @dataclass(init=False)
+        class TestClass(ActivityObject):
+            ''' test class with default field '''
+            type: str = 'TestObject'
+
+        instance = TestClass(id='a')
+        self.assertEqual(instance.id, 'a')
+        self.assertEqual(instance.type, 'TestObject')
+
+    def test_serialize(self):
+        ''' simple function for converting dataclass to dict '''
+        instance = ActivityObject(id='a', type='b')
+        serialized = instance.serialize()
+        self.assertIsInstance(serialized, dict)
+        self.assertEqual(serialized['id'], 'a')
+        self.assertEqual(serialized['type'], 'b')
+
+    @responses.activate
+    def test_resolve_remote_id(self):
+        ''' look up or load remote data '''
+        # existing item
+        result = resolve_remote_id(models.User, 'http://example.com/a/b')
+        self.assertEqual(result, self.user)
+
+        # remote item
+        responses.add(
+            responses.GET,
+            'https://example.com/user/mouse',
+            json=self.userdata,
+            status=200)
+
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            result = resolve_remote_id(
+                models.User, 'https://example.com/user/mouse')
+        self.assertIsInstance(result, models.User)
+        self.assertEqual(result.remote_id, 'https://example.com/user/mouse')
+        self.assertEqual(result.name, 'MOUSE?? MOUSE!!')
+
+    def test_to_model_invalid_model(self):
+        ''' catch mismatch between activity type and model type '''
+        instance = ActivityObject(id='a', type='b')
+        with self.assertRaises(ActivitySerializerError):
+            instance.to_model(models.User)
+
+    def test_to_model_simple_fields(self):
+        ''' test setting simple fields '''
+        self.assertEqual(self.user.name, '')
+
+        activity = activitypub.Person(
+            id=self.user.remote_id,
+            name='New Name',
+            preferredUsername='mouse',
+            inbox='http://www.com/',
+            outbox='http://www.com/',
+            followers='',
+            summary='',
+            publicKey=None,
+            endpoints={},
+        )
+
+        activity.to_model(models.User, self.user)
+
+        self.assertEqual(self.user.name, 'New Name')
+
+    def test_to_model_foreign_key(self):
+        ''' test setting one to one/foreign key '''
+        activity = activitypub.Person(
+            id=self.user.remote_id,
+            name='New Name',
+            preferredUsername='mouse',
+            inbox='http://www.com/',
+            outbox='http://www.com/',
+            followers='',
+            summary='',
+            publicKey=self.user.key_pair.to_activity(),
+            endpoints={},
+        )
+
+        activity.publicKey['publicKeyPem'] = 'hi im secure'
+
+        activity.to_model(models.User, self.user)
+        self.assertEqual(self.user.key_pair.public_key, 'hi im secure')
+
+    @responses.activate
+    def test_to_model_image(self):
+        ''' update an image field '''
+        activity = activitypub.Person(
+            id=self.user.remote_id,
+            name='New Name',
+            preferredUsername='mouse',
+            inbox='http://www.com/',
+            outbox='http://www.com/',
+            followers='',
+            summary='',
+            publicKey=None,
+            endpoints={},
+            icon={'url': 'http://www.example.com/image.jpg'}
+        )
+
+        responses.add(
+            responses.GET,
+            'http://www.example.com/image.jpg',
+            body=self.image_data,
+            status=200)
+
+        self.assertIsNone(self.user.avatar.name)
+        with self.assertRaises(ValueError):
+            self.user.avatar.file #pylint: disable=pointless-statement
+
+        activity.to_model(models.User, self.user)
+        self.assertIsNotNone(self.user.avatar.name)
+        self.assertIsNotNone(self.user.avatar.file)
+
+    def test_to_model_many_to_many(self):
+        ''' annoying that these all need special handling '''
+        status = models.Status.objects.create(
+            content='test status',
+            user=self.user,
+        )
+        book = models.Edition.objects.create(
+            title='Test Edition', remote_id='http://book.com/book')
+        update_data = activitypub.Note(
+            id=status.remote_id,
+            content=status.content,
+            attributedTo=self.user.remote_id,
+            published='hi',
+            to=[],
+            cc=[],
+            tag=[
+                {
+                    'type': 'Mention',
+                    'name': 'gerald',
+                    'href': 'http://example.com/a/b'
+                },
+                {
+                    'type': 'Edition',
+                    'name': 'gerald j. books',
+                    'href': 'http://book.com/book'
+                },
+            ]
+        )
+        update_data.to_model(models.Status, instance=status)
+        self.assertEqual(status.mention_users.first(), self.user)
+        self.assertEqual(status.mention_books.first(), book)
+
+
+    @responses.activate
+    def test_to_model_one_to_many(self):
+        ''' these are reversed relationships, where the secondary object
+        keys the primary object but not vice versa '''
+        status = models.Status.objects.create(
+            content='test status',
+            user=self.user,
+        )
+        update_data = activitypub.Note(
+            id=status.remote_id,
+            content=status.content,
+            attributedTo=self.user.remote_id,
+            published='hi',
+            to=[],
+            cc=[],
+            attachment=[{
+                'url': 'http://www.example.com/image.jpg',
+                'name': 'alt text',
+                'type': 'Image',
+            }],
+        )
+
+        responses.add(
+            responses.GET,
+            'http://www.example.com/image.jpg',
+            body=self.image_data,
+            status=200)
+
+        # sets the celery task call to the function call
+        with patch(
+                'bookwyrm.activitypub.base_activity.set_related_field.delay'):
+            update_data.to_model(models.Status, instance=status)
+        self.assertIsNone(status.attachments.first())
+
+
+    @responses.activate
+    def test_set_related_field(self):
+        ''' celery task to add back-references to created objects '''
+        status = models.Status.objects.create(
+            content='test status',
+            user=self.user,
+        )
+        data = {
+            'url': 'http://www.example.com/image.jpg',
+            'name': 'alt text',
+            'type': 'Image',
+        }
+        responses.add(
+            responses.GET,
+            'http://www.example.com/image.jpg',
+            body=self.image_data,
+            status=200)
+        set_related_field(
+            'Image', 'Status', 'status', status.remote_id, data)
+
+        self.assertIsInstance(status.attachments.first(), models.Image)
+        self.assertIsNotNone(status.attachments.first().image)

bookwyrm/tests/activitypub/test_person.py

@@ -1,5 +1,7 @@
+# pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring
 import json
 import pathlib
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import activitypub, models
@@ -7,9 +9,6 @@ from bookwyrm import activitypub, models
 
 class Person(TestCase):
     def setUp(self):
-        self.user = models.User.objects.create_user(
-            'rat', 'rat@rat.com', 'ratword',
-        )
         datafile = pathlib.Path(__file__).parent.joinpath(
             '../data/ap_user.json'
         )
@@ -21,3 +20,12 @@ class Person(TestCase):
         self.assertEqual(activity.id, 'https://example.com/user/mouse')
         self.assertEqual(activity.preferredUsername, 'mouse')
         self.assertEqual(activity.type, 'Person')
+
+
+    def test_user_to_model(self):
+        activity = activitypub.Person(**self.user_data)
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            user = activity.to_model(models.User)
+        self.assertEqual(user.username, 'mouse@example.com')
+        self.assertEqual(user.remote_id, 'https://example.com/user/mouse')
+        self.assertFalse(user.local)

bookwyrm/tests/activitypub/test_quotation.py

@@ -1,5 +1,7 @@
+''' quotation activty object serializer class '''
 import json
 import pathlib
+from unittest.mock import patch
 
 from django.test import TestCase
 from bookwyrm import activitypub, models
@@ -8,6 +10,8 @@ from bookwyrm import activitypub, models
 class Quotation(TestCase):
     ''' we have hecka ways to create statuses '''
     def setUp(self):
+        ''' model objects we'll need '''
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
             self.user = models.User.objects.create_user(
                 'mouse', 'mouse@mouse.mouse', 'mouseword',
                 local=False,
@@ -26,6 +30,7 @@ class Quotation(TestCase):
 
 
     def test_quotation_activity(self):
+        ''' create a Quoteation ap object from json '''
         quotation = activitypub.Quotation(**self.status_data)
 
         self.assertEqual(quotation.type, 'Quotation')
@@ -39,6 +44,7 @@ class Quotation(TestCase):
 
 
     def test_activity_to_model(self):
+        ''' create a model instance from an activity object '''
         activity = activitypub.Quotation(**self.status_data)
         quotation = activity.to_model(models.Quotation)
 

bookwyrm/tests/connectors/test_abstract_connector.py

@@ -3,7 +3,7 @@ from django.test import TestCase
 
 from bookwyrm import models
 from bookwyrm.connectors.abstract_connector import Mapping
-from bookwyrm.connectors.bookwyrm_connector import Connector
+from bookwyrm.connectors.openlibrary import Connector
 
 
 class AbstractConnector(TestCase):
@@ -12,7 +12,7 @@ class AbstractConnector(TestCase):
 
         models.Connector.objects.create(
             identifier='example.com',
-            connector_file='bookwyrm_connector',
+            connector_file='openlibrary',
             base_url='https://example.com',
             books_url='https:/example.com',
             covers_url='https://example.com',

bookwyrm/tests/connectors/test_bookwyrm_connector.py

@@ -1,16 +1,17 @@
 ''' testing book data connectors '''
-from dateutil import parser
-from django.test import TestCase
 import json
 import pathlib
+from django.test import TestCase
 
 from bookwyrm import models
 from bookwyrm.connectors.bookwyrm_connector import Connector
-from bookwyrm.connectors.abstract_connector import SearchResult, get_date
+from bookwyrm.connectors.abstract_connector import SearchResult
 
 
 class BookWyrmConnector(TestCase):
+    ''' this connector doesn't do much, just search '''
     def setUp(self):
+        ''' create the connector '''
         models.Connector.objects.create(
             identifier='example.com',
             connector_file='bookwyrm_connector',
@@ -29,13 +30,9 @@ class BookWyrmConnector(TestCase):
         self.edition_data = json.loads(edition_file.read_bytes())
 
 
-    def test_is_work_data(self):
-        self.assertEqual(self.connector.is_work_data(self.work_data), True)
-        self.assertEqual(self.connector.is_work_data(self.edition_data), False)
-
-
     def test_format_search_result(self):
-        datafile = pathlib.Path(__file__).parent.joinpath('../data/fr_search.json')
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/fr_search.json')
         search_data = json.loads(datafile.read_bytes())
         results = self.connector.parse_search_data(search_data)
         self.assertIsInstance(results, list)
@@ -46,9 +43,3 @@ class BookWyrmConnector(TestCase):
         self.assertEqual(result.key, 'https://example.com/book/122')
         self.assertEqual(result.author, 'Susanna Clarke')
         self.assertEqual(result.year, 2017)
-
-
-    def test_get_date(self):
-        date = get_date(self.edition_data['published_date'])
-        expected = parser.parse("2020-09-15T00:00:00+00:00")
-        self.assertEqual(date, expected)

bookwyrm/tests/incoming/test_favorite.py

@@ -1,5 +1,6 @@
 import json
 import pathlib
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models, incoming
@@ -7,6 +8,8 @@ from bookwyrm import models, incoming
 
 class Favorite(TestCase):
     def setUp(self):
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            with patch('bookwyrm.models.user.get_remote_reviews.delay'):
                 self.remote_user = models.User.objects.create_user(
                     'rat', 'rat@rat.com', 'ratword',
                     local=False,
@@ -15,7 +18,7 @@ class Favorite(TestCase):
                     outbox='https://example.com/users/rat/outbox',
                 )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword',
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True,
             remote_id='http://local.com/user/mouse')
 
         self.status = models.Status.objects.create(

bookwyrm/tests/incoming/test_follow.py

@@ -6,6 +6,8 @@ from bookwyrm import models, incoming
 
 class IncomingFollow(TestCase):
     def setUp(self):
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            with patch('bookwyrm.models.user.get_remote_reviews.delay'):
                 self.remote_user = models.User.objects.create_user(
                     'rat', 'rat@rat.com', 'ratword',
                     local=False,
@@ -14,7 +16,7 @@ class IncomingFollow(TestCase):
                     outbox='https://example.com/users/rat/outbox',
                 )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword')
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
         self.local_user.remote_id = 'http://local.com/user/mouse'
         self.local_user.save()
 
@@ -73,24 +75,3 @@ class IncomingFollow(TestCase):
         # the follow relationship should not exist
         follow = models.UserFollows.objects.all()
         self.assertEqual(list(follow), [])
-
-
-    def test_nonexistent_user_follow(self):
-        activity = {
-            "@context": "https://www.w3.org/ns/activitystreams",
-            "id": "https://example.com/users/rat/follows/123",
-            "type": "Follow",
-            "actor": "https://example.com/users/rat",
-            "object": "http://local.com/user/nonexistent-user"
-        }
-
-        with patch('bookwyrm.broadcast.broadcast_task.delay') as _:
-            incoming.handle_follow(activity)
-
-        # do nothing
-        notifications = models.Notification.objects.all()
-        self.assertEqual(list(notifications), [])
-        requests = models.UserFollowRequest.objects.all()
-        self.assertEqual(list(requests), [])
-        follows = models.UserFollows.objects.all()
-        self.assertEqual(list(follows), [])

bookwyrm/tests/incoming/test_follow_accept.py

@@ -1,3 +1,4 @@
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models, incoming
@@ -5,6 +6,8 @@ from bookwyrm import models, incoming
 
 class IncomingFollowAccept(TestCase):
     def setUp(self):
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            with patch('bookwyrm.models.user.get_remote_reviews.delay'):
                 self.remote_user = models.User.objects.create_user(
                     'rat', 'rat@rat.com', 'ratword',
                     local=False,
@@ -13,7 +16,7 @@ class IncomingFollowAccept(TestCase):
                     outbox='https://example.com/users/rat/outbox',
                 )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword')
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
         self.local_user.remote_id = 'http://local.com/user/mouse'
         self.local_user.save()
 

bookwyrm/tests/incoming/test_update_user.py

@@ -1,31 +0,0 @@
-''' when a remote user changes their profile '''
-import json
-import pathlib
-from django.test import TestCase
-
-from bookwyrm import models, incoming
-
-
-class UpdateUser(TestCase):
-    def setUp(self):
-        self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword',
-            remote_id='https://example.com/user/mouse',
-            local=False,
-            localname='mouse'
-        )
-
-        datafile = pathlib.Path(__file__).parent.joinpath(
-            '../data/ap_user.json'
-        )
-        self.user_data = json.loads(datafile.read_bytes())
-
-    def test_handle_update_user(self):
-        self.assertIsNone(self.user.name)
-        self.assertEqual(self.user.localname, 'mouse')
-
-        incoming.handle_update_user({'object': self.user_data})
-        self.user = models.User.objects.get(id=self.user.id)
-
-        self.assertEqual(self.user.name, 'MOUSE?? MOUSE!!')
-        self.assertEqual(self.user.localname, 'mouse')

bookwyrm/tests/models/test_base_model.py

@@ -1,25 +1,202 @@
 ''' testing models '''
+from collections import namedtuple
+from dataclasses import dataclass
+import re
 from django.test import TestCase
 
+from bookwyrm.activitypub.base_activity import ActivityObject
 from bookwyrm import models
-from bookwyrm.models.base_model import BookWyrmModel
+from bookwyrm.models import base_model
+from bookwyrm.models.base_model import ActivitypubMixin
 from bookwyrm.settings import DOMAIN
 
-
 class BaseModel(TestCase):
+    ''' functionality shared across models '''
     def test_remote_id(self):
-        instance = BookWyrmModel()
+        ''' these should be generated '''
+        instance = base_model.BookWyrmModel()
         instance.id = 1
         expected = instance.get_remote_id()
         self.assertEqual(expected, 'https://%s/bookwyrmmodel/1' % DOMAIN)
 
     def test_remote_id_with_user(self):
+        ''' format of remote id when there's a user object '''
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword')
-        instance = BookWyrmModel()
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+        instance = base_model.BookWyrmModel()
         instance.user = user
         instance.id = 1
         expected = instance.get_remote_id()
         self.assertEqual(
             expected,
             'https://%s/user/mouse/bookwyrmmodel/1' % DOMAIN)
+
+    def test_execute_after_save(self):
+        ''' this function sets remote ids after creation '''
+        # using Work because it BookWrymModel is abstract and this requires save
+        # Work is a relatively not-fancy model.
+        instance = models.Work.objects.create(title='work title')
+        instance.remote_id = None
+        base_model.execute_after_save(None, instance, True)
+        self.assertEqual(
+            instance.remote_id,
+            'https://%s/book/%d' % (DOMAIN, instance.id)
+        )
+
+        # shouldn't set remote_id if it's not created
+        instance.remote_id = None
+        base_model.execute_after_save(None, instance, False)
+        self.assertIsNone(instance.remote_id)
+
+    def test_to_create_activity(self):
+        ''' wrapper for ActivityPub "create" action '''
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+
+        object_activity = {
+            'to': 'to field', 'cc': 'cc field',
+            'content': 'hi',
+            'published': '2020-12-04T17:52:22.623807+00:00',
+        }
+        MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
+        mock_self = MockSelf(
+            'https://example.com/status/1',
+            lambda *args: object_activity
+        )
+        activity = ActivitypubMixin.to_create_activity(mock_self, user)
+        self.assertEqual(
+            activity['id'],
+            'https://example.com/status/1/activity'
+        )
+        self.assertEqual(activity['actor'], user.remote_id)
+        self.assertEqual(activity['type'], 'Create')
+        self.assertEqual(activity['to'], 'to field')
+        self.assertEqual(activity['cc'], 'cc field')
+        self.assertEqual(activity['object'], object_activity)
+        self.assertEqual(
+            activity['signature'].creator,
+            '%s#main-key' % user.remote_id
+        )
+
+    def test_to_delete_activity(self):
+        ''' wrapper for Delete activity '''
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+
+        MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
+        mock_self = MockSelf(
+            'https://example.com/status/1',
+            lambda *args: {}
+        )
+        activity = ActivitypubMixin.to_delete_activity(mock_self, user)
+        self.assertEqual(
+            activity['id'],
+            'https://example.com/status/1/activity'
+        )
+        self.assertEqual(activity['actor'], user.remote_id)
+        self.assertEqual(activity['type'], 'Delete')
+        self.assertEqual(
+            activity['to'],
+            ['%s/followers' % user.remote_id])
+        self.assertEqual(
+            activity['cc'],
+            ['https://www.w3.org/ns/activitystreams#Public'])
+
+    def test_to_update_activity(self):
+        ''' ditto above but for Update '''
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+
+        MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
+        mock_self = MockSelf(
+            'https://example.com/status/1',
+            lambda *args: {}
+        )
+        activity = ActivitypubMixin.to_update_activity(mock_self, user)
+        self.assertIsNotNone(
+            re.match(
+                r'^https:\/\/example\.com\/status\/1#update\/.*',
+                activity['id']
+            )
+        )
+        self.assertEqual(activity['actor'], user.remote_id)
+        self.assertEqual(activity['type'], 'Update')
+        self.assertEqual(
+            activity['to'],
+            ['https://www.w3.org/ns/activitystreams#Public'])
+        self.assertEqual(activity['object'], {})
+
+    def test_to_undo_activity(self):
+        ''' and again, for Undo '''
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+
+        MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
+        mock_self = MockSelf(
+            'https://example.com/status/1',
+            lambda *args: {}
+        )
+        activity = ActivitypubMixin.to_undo_activity(mock_self, user)
+        self.assertEqual(
+            activity['id'],
+            'https://example.com/status/1#undo'
+        )
+        self.assertEqual(activity['actor'], user.remote_id)
+        self.assertEqual(activity['type'], 'Undo')
+        self.assertEqual(activity['object'], {})
+
+
+    def test_to_activity(self):
+        ''' model to ActivityPub json '''
+        @dataclass(init=False)
+        class TestActivity(ActivityObject):
+            ''' real simple mock '''
+            type: str = 'Test'
+
+        class TestModel(ActivitypubMixin, base_model.BookWyrmModel):
+            ''' real simple mock model because BookWyrmModel is abstract '''
+
+        instance = TestModel()
+        instance.remote_id = 'https://www.example.com/test'
+        instance.activity_serializer = TestActivity
+
+        activity = instance.to_activity()
+        self.assertIsInstance(activity, dict)
+        self.assertEqual(activity['id'], 'https://www.example.com/test')
+        self.assertEqual(activity['type'], 'Test')
+
+
+    def test_find_existing_by_remote_id(self):
+        ''' attempt to match a remote id to an object in the db '''
+        # uses a different remote id scheme
+        # this isn't really part of this test directly but it's helpful to state
+        book = models.Edition.objects.create(
+            title='Test Edition', remote_id='http://book.com/book')
+        user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        user.remote_id = 'http://example.com/a/b'
+        user.save()
+
+        self.assertEqual(book.origin_id, 'http://book.com/book')
+        self.assertNotEqual(book.remote_id, 'http://book.com/book')
+
+        # uses subclasses
+        models.Comment.objects.create(
+            user=user, content='test status', book=book, \
+            remote_id='https://comment.net')
+
+        result = models.User.find_existing_by_remote_id('hi')
+        self.assertIsNone(result)
+
+        result = models.User.find_existing_by_remote_id(
+            'http://example.com/a/b')
+        self.assertEqual(result, user)
+
+        # test using origin id
+        result = models.Edition.find_existing_by_remote_id(
+            'http://book.com/book')
+        self.assertEqual(result, book)
+
+        # test subclass match
+        result = models.Status.find_existing_by_remote_id(
+            'https://comment.net')

bookwyrm/tests/models/test_book_model.py

@@ -24,7 +24,7 @@ class Book(TestCase):
     def test_remote_id(self):
         remote_id = 'https://%s/book/%d' % (settings.DOMAIN, self.work.id)
         self.assertEqual(self.work.get_remote_id(), remote_id)
-        self.assertEqual(self.work.remote_id, 'https://example.com/book/1')
+        self.assertEqual(self.work.remote_id, remote_id)
 
     def test_create_book(self):
         ''' you shouldn't be able to create Books (only editions and works) '''
@@ -59,7 +59,7 @@ class Book(TestCase):
 class Shelf(TestCase):
     def setUp(self):
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
         models.Shelf.objects.create(
             name='Test Shelf', identifier='test-shelf', user=user)
 

bookwyrm/tests/models/test_fields.py

@@ -0,0 +1,410 @@
+''' testing models '''
+from io import BytesIO
+from collections import namedtuple
+from dataclasses import dataclass
+import json
+import pathlib
+import re
+from typing import List
+from unittest.mock import patch
+
+from PIL import Image
+import responses
+
+from django.core.exceptions import ValidationError
+from django.core.files.base import ContentFile
+from django.db import models
+from django.test import TestCase
+from django.utils import timezone
+
+from bookwyrm.activitypub.base_activity import ActivityObject
+from bookwyrm.models import fields, User, Status
+from bookwyrm.models.base_model import ActivitypubMixin, BookWyrmModel
+
+class ActivitypubFields(TestCase):
+    ''' overwrites standard model feilds to work with activitypub '''
+    def test_validate_remote_id(self):
+        ''' should look like a url '''
+        self.assertIsNone(fields.validate_remote_id(
+            'http://www.example.com'
+        ))
+        self.assertIsNone(fields.validate_remote_id(
+            'https://www.example.com'
+        ))
+        self.assertIsNone(fields.validate_remote_id(
+            'http://example.com/dlfjg-23/x'
+        ))
+        self.assertRaises(
+            ValidationError, fields.validate_remote_id,
+            'http:/example.com/dlfjg-23/x'
+        )
+        self.assertRaises(
+            ValidationError, fields.validate_remote_id,
+            'www.example.com/dlfjg-23/x'
+        )
+        self.assertRaises(
+            ValidationError, fields.validate_remote_id,
+            'http://www.example.com/dlfjg 23/x'
+        )
+
+    def test_activitypub_field_mixin(self):
+        ''' generic mixin with super basic to and from functionality '''
+        instance = fields.ActivitypubFieldMixin()
+        self.assertEqual(instance.field_to_activity('fish'), 'fish')
+        self.assertEqual(instance.field_from_activity('fish'), 'fish')
+        self.assertFalse(instance.deduplication_field)
+
+        instance = fields.ActivitypubFieldMixin(
+            activitypub_wrapper='endpoints', activitypub_field='outbox'
+        )
+        self.assertEqual(
+            instance.field_to_activity('fish'),
+            {'outbox': 'fish'}
+        )
+        self.assertEqual(
+            instance.field_from_activity({'outbox': 'fish'}),
+            'fish'
+        )
+        self.assertEqual(instance.get_activitypub_field(), 'endpoints')
+
+        instance = fields.ActivitypubFieldMixin()
+        instance.name = 'snake_case_name'
+        self.assertEqual(instance.get_activitypub_field(), 'snakeCaseName')
+
+    def test_remote_id_field(self):
+        ''' just sets some defaults on charfield '''
+        instance = fields.RemoteIdField()
+        self.assertEqual(instance.max_length, 255)
+        self.assertTrue(instance.deduplication_field)
+
+        with self.assertRaises(ValidationError):
+            instance.run_validators('http://www.example.com/dlfjg 23/x')
+
+    def test_username_field(self):
+        ''' again, just setting defaults on username field '''
+        instance = fields.UsernameField()
+        self.assertEqual(instance.activitypub_field, 'preferredUsername')
+        self.assertEqual(instance.max_length, 150)
+        self.assertEqual(instance.unique, True)
+        with self.assertRaises(ValidationError):
+            instance.run_validators('one two')
+            instance.run_validators('a*&')
+            instance.run_validators('trailingwhite ')
+        self.assertIsNone(instance.run_validators('aksdhf'))
+
+        self.assertEqual(instance.field_to_activity('test@example.com'), 'test')
+
+
+    def test_privacy_field_defaults(self):
+        ''' post privacy field's many default values '''
+        instance = fields.PrivacyField()
+        self.assertEqual(instance.max_length, 255)
+        self.assertEqual(
+            [c[0] for c in instance.choices],
+            ['public', 'unlisted', 'followers', 'direct'])
+        self.assertEqual(instance.default, 'public')
+        self.assertEqual(
+            instance.public, 'https://www.w3.org/ns/activitystreams#Public')
+
+    def test_privacy_field_set_field_from_activity(self):
+        ''' translate between to/cc fields and privacy '''
+        @dataclass(init=False)
+        class TestActivity(ActivityObject):
+            ''' real simple mock '''
+            to: List[str]
+            cc: List[str]
+            id: str = 'http://hi.com'
+            type: str = 'Test'
+
+        class TestPrivacyModel(ActivitypubMixin, BookWyrmModel):
+            ''' real simple mock model because BookWyrmModel is abstract '''
+            privacy_field = fields.PrivacyField()
+            mention_users = fields.TagField(User)
+            user = fields.ForeignKey(User, on_delete=models.CASCADE)
+
+        public = 'https://www.w3.org/ns/activitystreams#Public'
+        data = TestActivity(
+            to=[public],
+            cc=['bleh'],
+        )
+        model_instance = TestPrivacyModel(privacy_field='direct')
+        self.assertEqual(model_instance.privacy_field, 'direct')
+
+        instance = fields.PrivacyField()
+        instance.name = 'privacy_field'
+        instance.set_field_from_activity(model_instance, data)
+        self.assertEqual(model_instance.privacy_field, 'public')
+
+        data.to = ['bleh']
+        data.cc = []
+        instance.set_field_from_activity(model_instance, data)
+        self.assertEqual(model_instance.privacy_field, 'direct')
+
+        data.to = ['bleh']
+        data.cc = [public, 'waah']
+        instance.set_field_from_activity(model_instance, data)
+        self.assertEqual(model_instance.privacy_field, 'unlisted')
+
+
+    def test_privacy_field_set_activity_from_field(self):
+        ''' translate between to/cc fields and privacy '''
+        user = User.objects.create_user(
+            'rat', 'rat@rat.rat', 'ratword', local=True)
+        public = 'https://www.w3.org/ns/activitystreams#Public'
+        followers = '%s/followers' % user.remote_id
+
+        instance = fields.PrivacyField()
+        instance.name = 'privacy_field'
+
+        model_instance = Status.objects.create(user=user, content='hi')
+        activity = {}
+        instance.set_activity_from_field(activity, model_instance)
+        self.assertEqual(activity['to'], [public])
+        self.assertEqual(activity['cc'], [followers])
+
+        model_instance = Status.objects.create(user=user, privacy='unlisted')
+        activity = {}
+        instance.set_activity_from_field(activity, model_instance)
+        self.assertEqual(activity['to'], [followers])
+        self.assertEqual(activity['cc'], [public])
+
+        model_instance = Status.objects.create(user=user, privacy='followers')
+        activity = {}
+        instance.set_activity_from_field(activity, model_instance)
+        self.assertEqual(activity['to'], [followers])
+        self.assertEqual(activity['cc'], [])
+
+        model_instance = Status.objects.create(
+            user=user,
+            privacy='direct',
+        )
+        model_instance.mention_users.set([user])
+        activity = {}
+        instance.set_activity_from_field(activity, model_instance)
+        self.assertEqual(activity['to'], [user.remote_id])
+        self.assertEqual(activity['cc'], [])
+
+
+    def test_foreign_key(self):
+        ''' should be able to format a related model '''
+        instance = fields.ForeignKey('User', on_delete=models.CASCADE)
+        Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
+        item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c')
+        # returns the remote_id field of the related object
+        self.assertEqual(instance.field_to_activity(item), 'https://e.b/c')
+
+
+    @responses.activate
+    def test_foreign_key_from_activity_str(self):
+        ''' create a new object from a foreign key '''
+        instance = fields.ForeignKey(User, on_delete=models.CASCADE)
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json')
+        userdata = json.loads(datafile.read_bytes())
+        # don't try to load the user icon
+        del userdata['icon']
+
+        # it shouldn't match with this unrelated user:
+        unrelated_user = User.objects.create_user(
+            'rat', 'rat@rat.rat', 'ratword', local=True)
+
+        # test receiving an unknown remote id and loading data
+        responses.add(
+            responses.GET,
+            'https://example.com/user/mouse',
+            json=userdata,
+            status=200)
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            value = instance.field_from_activity(
+                'https://example.com/user/mouse')
+        self.assertIsInstance(value, User)
+        self.assertNotEqual(value, unrelated_user)
+        self.assertEqual(value.remote_id, 'https://example.com/user/mouse')
+        self.assertEqual(value.name, 'MOUSE?? MOUSE!!')
+
+
+    def test_foreign_key_from_activity_dict(self):
+        ''' test recieving activity json '''
+        instance = fields.ForeignKey(User, on_delete=models.CASCADE)
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json')
+        userdata = json.loads(datafile.read_bytes())
+        # don't try to load the user icon
+        del userdata['icon']
+
+        # it shouldn't match with this unrelated user:
+        unrelated_user = User.objects.create_user(
+            'rat', 'rat@rat.rat', 'ratword', local=True)
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            value = instance.field_from_activity(userdata)
+        self.assertIsInstance(value, User)
+        self.assertNotEqual(value, unrelated_user)
+        self.assertEqual(value.remote_id, 'https://example.com/user/mouse')
+        self.assertEqual(value.name, 'MOUSE?? MOUSE!!')
+        # et cetera but we're not testing serializing user json
+
+
+    def test_foreign_key_from_activity_dict_existing(self):
+        ''' test receiving a dict of an existing object in the db '''
+        instance = fields.ForeignKey(User, on_delete=models.CASCADE)
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json'
+        )
+        userdata = json.loads(datafile.read_bytes())
+        user = User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        user.remote_id = 'https://example.com/user/mouse'
+        user.save()
+        User.objects.create_user(
+            'rat', 'rat@rat.rat', 'ratword', local=True)
+
+        value = instance.field_from_activity(userdata)
+        self.assertEqual(value, user)
+
+
+    def test_foreign_key_from_activity_str_existing(self):
+        ''' test receiving a remote id of an existing object in the db '''
+        instance = fields.ForeignKey(User, on_delete=models.CASCADE)
+        user = User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        User.objects.create_user(
+            'rat', 'rat@rat.rat', 'ratword', local=True)
+
+        value = instance.field_from_activity(user.remote_id)
+        self.assertEqual(value, user)
+
+
+    def test_one_to_one_field(self):
+        ''' a gussied up foreign key '''
+        instance = fields.OneToOneField('User', on_delete=models.CASCADE)
+        Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
+        item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c')
+        self.assertEqual(instance.field_to_activity(item), {'a': 'b'})
+
+    def test_many_to_many_field(self):
+        ''' lists! '''
+        instance = fields.ManyToManyField('User')
+
+        Serializable = namedtuple('Serializable', ('to_activity', 'remote_id'))
+        Queryset = namedtuple('Queryset', ('all', 'instance'))
+        item = Serializable(lambda: {'a': 'b'}, 'https://e.b/c')
+        another_item = Serializable(lambda: {}, 'example.com')
+
+        items = Queryset(lambda: [item], another_item)
+
+        self.assertEqual(instance.field_to_activity(items), ['https://e.b/c'])
+
+        instance = fields.ManyToManyField('User', link_only=True)
+        instance.name = 'snake_case'
+        self.assertEqual(
+            instance.field_to_activity(items),
+            'example.com/snake_case'
+        )
+
+    @responses.activate
+    def test_many_to_many_field_from_activity(self):
+        ''' resolve related fields for a list, takes a list of remote ids '''
+        instance = fields.ManyToManyField(User)
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            '../data/ap_user.json'
+        )
+        userdata = json.loads(datafile.read_bytes())
+        # don't try to load the user icon
+        del userdata['icon']
+
+        # test receiving an unknown remote id and loading data
+        responses.add(
+            responses.GET,
+            'https://example.com/user/mouse',
+            json=userdata,
+            status=200)
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            value = instance.field_from_activity(
+                ['https://example.com/user/mouse', 'bleh']
+            )
+        self.assertIsInstance(value, list)
+        self.assertEqual(len(value), 1)
+        self.assertIsInstance(value[0], User)
+
+    def test_tag_field(self):
+        ''' a special type of many to many field '''
+        instance = fields.TagField('User')
+
+        Serializable = namedtuple(
+            'Serializable',
+            ('to_activity', 'remote_id', 'name_field', 'name')
+        )
+        Queryset = namedtuple('Queryset', ('all', 'instance'))
+        item = Serializable(
+            lambda: {'a': 'b'}, 'https://e.b/c', 'name', 'Name')
+        another_item = Serializable(
+            lambda: {}, 'example.com', '', '')
+        items = Queryset(lambda: [item], another_item)
+
+        result = instance.field_to_activity(items)
+        self.assertIsInstance(result, list)
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0].href, 'https://e.b/c')
+        self.assertEqual(result[0].name, 'Name')
+        self.assertEqual(result[0].type, 'Serializable')
+
+
+    def test_tag_field_from_activity(self):
+        ''' loadin' a list of items from Links '''
+        # TODO
+
+
+    @responses.activate
+    def test_image_field(self):
+        ''' storing images '''
+        user = User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+        image_file = pathlib.Path(__file__).parent.joinpath(
+            '../../static/images/default_avi.jpg')
+        image = Image.open(image_file)
+        output = BytesIO()
+        image.save(output, format=image.format)
+        user.avatar.save(
+            'test.jpg',
+            ContentFile(output.getvalue())
+        )
+
+        output = fields.image_serializer(user.avatar)
+        self.assertIsNotNone(
+            re.match(
+                r'.*\.jpg',
+                output.url,
+            )
+        )
+        self.assertEqual(output.type, 'Image')
+
+        instance = fields.ImageField()
+
+        self.assertEqual(instance.field_to_activity(user.avatar), output)
+
+        responses.add(
+            responses.GET,
+            'http://www.example.com/image.jpg',
+            body=user.avatar.file.read(),
+            status=200)
+        loaded_image = instance.field_from_activity(
+            'http://www.example.com/image.jpg')
+        self.assertIsInstance(loaded_image, list)
+        self.assertIsInstance(loaded_image[1], ContentFile)
+
+
+    def test_datetime_field(self):
+        ''' this one is pretty simple, it just has to use isoformat '''
+        instance = fields.DateTimeField()
+        now = timezone.now()
+        self.assertEqual(instance.field_to_activity(now), now.isoformat())
+        self.assertEqual(
+            instance.field_from_activity(now.isoformat()), now
+        )
+        self.assertEqual(instance.field_from_activity('bip'), None)
+
+
+    def test_array_field(self):
+        ''' idk why it makes them strings but probably for a good reason '''
+        instance = fields.ArrayField(fields.IntegerField)
+        self.assertEqual(instance.field_to_activity([0, 1]), ['0', '1'])

bookwyrm/tests/models/test_import_model.py

@@ -52,7 +52,7 @@ class ImportJob(TestCase):
         unknown_read_data['Date Read'] = ''
 
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
         job = models.ImportJob.objects.create(user=user)
         models.ImportItem.objects.create(
             job=job, index=1, data=currently_reading_data)

bookwyrm/tests/models/test_relationship_models.py

@@ -1,4 +1,5 @@
 ''' testing models '''
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models
@@ -6,6 +7,7 @@ from bookwyrm import models
 
 class Relationship(TestCase):
     def setUp(self):
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
             self.remote_user = models.User.objects.create_user(
                 'rat', 'rat@rat.com', 'ratword',
                 local=False,
@@ -14,7 +16,7 @@ class Relationship(TestCase):
                 outbox='https://example.com/users/rat/outbox',
             )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword')
+            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
         self.local_user.remote_id = 'http://local.com/user/mouse'
         self.local_user.save()
 

bookwyrm/tests/models/test_status_model.py

@@ -7,7 +7,7 @@ from bookwyrm import models, settings
 class Status(TestCase):
     def setUp(self):
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
         book = models.Edition.objects.create(title='Example Edition')
 
         models.Status.objects.create(user=user, content='Blah blah')
@@ -40,13 +40,3 @@ class Status(TestCase):
         expected_id = 'https://%s/user/mouse/review/%d' % \
                 (settings.DOMAIN, review.id)
         self.assertEqual(review.remote_id, expected_id)
-
-
-class Tag(TestCase):
-    def test_tag(self):
-        book = models.Edition.objects.create(title='Example Edition')
-        user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
-        tag = models.Tag.objects.create(user=user, book=book, name='t/est tag')
-        self.assertEqual(tag.identifier, 't%2Fest+tag')
-

bookwyrm/tests/models/test_user_model.py

@@ -1,4 +1,5 @@
 ''' testing models '''
+from unittest.mock import patch
 from django.test import TestCase
 
 from bookwyrm import models
@@ -8,7 +9,7 @@ from bookwyrm.settings import DOMAIN
 class User(TestCase):
     def setUp(self):
         self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword')
+            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
 
     def test_computed_fields(self):
         ''' username instead of id here '''
@@ -19,8 +20,15 @@ class User(TestCase):
         self.assertEqual(self.user.shared_inbox, 'https://%s/inbox' % DOMAIN)
         self.assertEqual(self.user.inbox, '%s/inbox' % expected_id)
         self.assertEqual(self.user.outbox, '%s/outbox' % expected_id)
-        self.assertIsNotNone(self.user.private_key)
-        self.assertIsNotNone(self.user.public_key)
+        self.assertIsNotNone(self.user.key_pair.private_key)
+        self.assertIsNotNone(self.user.key_pair.public_key)
+
+    def test_remote_user(self):
+        with patch('bookwyrm.models.user.set_remote_server.delay'):
+            user = models.User.objects.create_user(
+                'rat', 'rat@rat.rat', 'ratword', local=False,
+                remote_id='https://example.com/dfjkg')
+        self.assertEqual(user.username, 'rat@example.com')
 
 
     def test_user_shelves(self):
@@ -53,7 +61,6 @@ class User(TestCase):
         self.assertEqual(activity['name'], self.user.name)
         self.assertEqual(activity['inbox'], self.user.inbox)
         self.assertEqual(activity['outbox'], self.user.outbox)
-        self.assertEqual(activity['followers'], self.user.ap_followers)
         self.assertEqual(activity['bookwyrmUser'], True)
         self.assertEqual(activity['discoverable'], True)
         self.assertEqual(activity['type'], 'Person')