Merge branch 'main' into progress_update

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: bookwyrm
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

README.md

@@ -8,6 +8,7 @@ Social reading and reviewing, decentralized with ActivityPub
    - [The role of federation](#the-role-of-federation)
    - [Features](#features)
  - [Setting up the developer environment](#setting-up-the-developer-environment)
+ - [Installing in Production](#installing-in-production)
  - [Project structure](#project-structure)
  - [Book data](#book-data)
  - [Contributing](#contributing)
@@ -59,8 +60,6 @@ cp .env.example .env
 
 For most testing, you'll want to use ngrok. Remember to set the DOMAIN in `.env` to your ngrok domain.
 
-
-#### With Docker
 You'll have to install the Docker and docker-compose. When you're ready, run:
 
 ```bash
@@ -69,33 +68,7 @@ docker-compose run --rm web python manage.py migrate
 docker-compose run --rm web python manage.py initdb
 ```
 
-### Without Docker
+Once the build is complete, you can access the instance at `localhost:1333`
-You will need postgres installed and running on your computer.
-
-``` bash
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements.txt
-createdb bookwyrm
-```
-
-Create the psql user in `psql bookwyrm`:
-``` psql
-CREATE ROLE bookwyrm WITH LOGIN PASSWORD 'bookwyrm';
-GRANT ALL PRIVILEGES ON DATABASE bookwyrm TO bookwyrm;
-```
-
-Initialize the database (or, more specifically, delete the existing database, run migrations, and start fresh):
-``` bash
-./rebuilddb.sh
-```
-This creates two users, `mouse` with password `password123` and `rat` with password `ratword`.
-
-The application uses Celery and Redis for task management, which must also be installed and configured.
-
-And go to the app at `localhost:8000`
-
-
 
 ## Installing in Production
 
@@ -114,7 +87,7 @@ This project is still young and isn't, at the momoment, very stable, so please p
   `cp .env.example .env`
    - Add your domain, email address, mailgun credentials
    - Set a secure redis password and secret key
- - Update your nginx configuration in `nginx/defautl.conf`
+ - Update your nginx configuration in `nginx/default.conf`
    - Replace `your-domain.com` with your domain name
  - Run the application (this should also set up a Certbot ssl cert for your domain)
   `docker-compose up --build`
@@ -124,13 +97,13 @@ This project is still young and isn't, at the momoment, very stable, so please p
  - Run docker-compose in the background
   `docker-compose up -d`
  - Initialize the database
-  `./fr-dev initdb`
+  `./bw-dev initdb`
  - Congrats! You did it, go to your domain and enjoy the fruits of your labors
 ### Configure your instance
  - Register a user account in the applcation UI
  - Make your account a superuser (warning: do *not* use django's `createsuperuser` command)
    - On your server, open the django shell
-    `./fr-dev shell`
+    `./bw-dev shell`
    - Load your user and make it a superuser
     ```python
     from bookwyrm import models

bookwyrm/activitypub/__init__.py

@@ -2,20 +2,20 @@
 import inspect
 import sys
 
-from .base_activity import ActivityEncoder, PublicKey, Signature
+from .base_activity import ActivityEncoder, Signature
 from .base_activity import Link, Mention
-from .base_activity import ActivitySerializerError
+from .base_activity import ActivitySerializerError, resolve_remote_id
-from .base_activity import tag_formatter
 from .image import Image
 from .note import Note, GeneratedNote, Article, Comment, Review, Quotation
 from .note import Tombstone
 from .interaction import Boost, Like
 from .ordered_collection import OrderedCollection, OrderedCollectionPage
-from .person import Person
+from .person import Person, PublicKey
+from .response import ActivitypubResponse
 from .book import Edition, Work, Author
 from .verbs import Create, Delete, Undo, Update
 from .verbs import Follow, Accept, Reject
-from .verbs import Add, Remove
+from .verbs import Add, AddBook, Remove
 
 # this creates a list of all the Activity types that we can serialize,
 # so when an Activity comes in from outside, we can check if it's known

bookwyrm/activitypub/base_activity.py

@@ -1,18 +1,12 @@
 ''' basics for an activitypub serializer '''
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
-from uuid import uuid4
 
-from django.core.files.base import ContentFile
+from django.apps import apps
-from django.db import transaction
+from django.db import IntegrityError, transaction
-from django.db.models.fields.related_descriptors \
-        import ForwardManyToOneDescriptor, ManyToManyDescriptor, \
-        ReverseManyToOneDescriptor
-from django.db.models.fields.files import ImageFileDescriptor
-import requests
-
-from bookwyrm import books_manager, models
 
+from bookwyrm.connectors import ConnectorException, get_data
+from bookwyrm.tasks import app
 
 class ActivitySerializerError(ValueError):
     ''' routine problems serializing activitypub json '''
@@ -25,26 +19,19 @@ class ActivityEncoder(JSONEncoder):
 
 
 @dataclass
-class Link():
+class Link:
     ''' for tagging a book in a status '''
     href: str
     name: str
     type: str = 'Link'
 
+
 @dataclass
 class Mention(Link):
     ''' a subtype of Link for mentioning an actor '''
     type: str = 'Mention'
 
 
-@dataclass
-class PublicKey:
-    ''' public key block '''
-    id: str
-    owner: str
-    publicKeyPem: str
-
-
 @dataclass
 class Signature:
     ''' public key block '''
@@ -76,88 +63,67 @@ class ActivityObject:
             setattr(self, field.name, value)
 
 
-    def to_model(self, model, instance=None):
+    def to_model(self, model, instance=None, save=True):
         ''' convert from an activity to a model instance '''
         if not isinstance(self, model.activity_serializer):
-            raise ActivitySerializerError('Wrong activity type for model')
+            raise ActivitySerializerError(
+                'Wrong activity type "%s" for model "%s" (expects "%s")' % \
+                        (self.__class__,
+                         model.__name__,
+                         model.activity_serializer)
+            )
+
+        if hasattr(model, 'ignore_activity') and model.ignore_activity(self):
+            return instance
 
         # check for an existing instance, if we're not updating a known obj
-        if not instance:
+        instance = instance or model.find_existing(self.serialize()) or model()
-            try:
-                return model.objects.get(remote_id=self.id)
-            except model.DoesNotExist:
-                pass
 
-        model_fields = [m.name for m in model._meta.get_fields()]
+        for field in instance.simple_fields:
-        mapped_fields = {}
+            field.set_field_from_activity(instance, self)
-        many_to_many_fields = {}
-        one_to_many_fields = {}
-        image_fields = {}
 
-        for mapping in model.activity_mappings:
+        # image fields have to be set after other fields because they can save
-            if mapping.model_key not in model_fields:
+        # too early and jank up users
-                continue
+        for field in instance.image_fields:
-            # value is None if there's a default that isn't supplied
+            field.set_field_from_activity(instance, self, save=save)
-            # in the activity but is supplied in the formatter
-            value = None
-            if mapping.activity_key:
-                value = getattr(self, mapping.activity_key)
-            model_field = getattr(model, mapping.model_key)
 
-            formatted_value = mapping.model_formatter(value)
+        if not save:
-            if isinstance(model_field, ForwardManyToOneDescriptor) and \
+            return instance
-                    formatted_value:
-                # foreign key remote id reolver (work on Edition, for example)
-                fk_model = model_field.field.related_model
-                reference = resolve_foreign_key(fk_model, formatted_value)
-                mapped_fields[mapping.model_key] = reference
-            elif isinstance(model_field, ManyToManyDescriptor):
-                # status mentions book/users
-                many_to_many_fields[mapping.model_key] = formatted_value
-            elif isinstance(model_field, ReverseManyToOneDescriptor):
-                # attachments on Status, for example
-                one_to_many_fields[mapping.model_key] = formatted_value
-            elif isinstance(model_field, ImageFileDescriptor):
-                # image fields need custom handling
-                image_fields[mapping.model_key] = formatted_value
-            else:
-                mapped_fields[mapping.model_key] = formatted_value
 
         with transaction.atomic():
-            if instance:
+            # we can't set many to many and reverse fields on an unsaved object
-                # updating an existing model isntance
+            try:
-                for k, v in mapped_fields.items():
-                    setattr(instance, k, v)
                 instance.save()
-            else:
+            except IntegrityError as e:
-                # creating a new model instance
+                raise ActivitySerializerError(e)
-                instance = model.objects.create(**mapped_fields)
 
-            # add images
+            # add many to many fields, which have to be set post-save
-            for (model_key, value) in image_fields.items():
+            for field in instance.many_to_many_fields:
-                formatted_value = image_formatter(value)
+                # mention books/users, for example
-                if not formatted_value:
+                field.set_field_from_activity(instance, self)
-                    continue
-                getattr(instance, model_key).save(*formatted_value, save=True)
 
-            for (model_key, values) in many_to_many_fields.items():
+        # reversed relationships in the models
-                # mention books, mention users
+        for (model_field_name, activity_field_name) in \
-                getattr(instance, model_key).set(values)
+                instance.deserialize_reverse_fields:
+            # attachments on Status, for example
+            values = getattr(self, activity_field_name)
+            if values is None or values is MISSING:
+                continue
 
-            # add one to many fields
+            model_field = getattr(model, model_field_name)
-            for (model_key, values) in one_to_many_fields.items():
+            # creating a Work, model_field is 'editions'
-                if values == MISSING:
+            # creating a User, model field is 'key_pair'
-                    continue
+            related_model = model_field.field.model
-                model_field = getattr(instance, model_key)
+            related_field_name = model_field.field.name
-                model = model_field.model
-                for item in values:
-                    item = model.activity_serializer(**item)
-                    field_name = instance.__class__.__name__.lower()
-                    with transaction.atomic():
-                        item = item.to_model(model)
-                        setattr(item, field_name, instance)
-                        item.save()
 
+            for item in values:
+                set_related_field.delay(
+                    related_model.__name__,
+                    instance.__class__.__name__,
+                    related_field_name,
+                    instance.remote_id,
+                    item
+                )
         return instance
 
 
@@ -168,66 +134,72 @@ class ActivityObject:
         return data
 
 
-def resolve_foreign_key(model, remote_id):
+@app.task
-    ''' look up the remote_id on an activity json field '''
+@transaction.atomic
-    if model in [models.Edition, models.Work, models.Book]:
+def set_related_field(
-        return books_manager.get_or_create_book(remote_id)
+        model_name, origin_model_name, related_field_name,
+        related_remote_id, data):
+    ''' load reverse related fields (editions, attachments) without blocking '''
+    model = apps.get_model('bookwyrm.%s' % model_name, require_ready=True)
+    origin_model = apps.get_model(
+        'bookwyrm.%s' % origin_model_name,
+        require_ready=True
+    )
 
-    result = model.objects
+    with transaction.atomic():
-    if hasattr(model.objects, 'select_subclasses'):
+        if isinstance(data, str):
-        result = result.select_subclasses()
+            existing = model.find_existing_by_remote_id(data)
+            if existing:
+                data = existing.to_activity()
+            else:
+                data = get_data(data)
+        activity = model.activity_serializer(**data)
 
-    result = result.filter(
+        # this must exist because it's the object that triggered this function
-        remote_id=remote_id
+        instance = origin_model.find_existing_by_remote_id(related_remote_id)
-    ).first()
+        if not instance:
+            raise ValueError(
+                'Invalid related remote id: %s' % related_remote_id)
 
-    if not result:
+        # set the origin's remote id on the activity so it will be there when
-        raise ActivitySerializerError(
+        # the model instance is created
-            'Could not resolve remote_id in %s model: %s' % \
+        # edition.parentWork = instance, for example
-                (model.__name__, remote_id))
+        model_field = getattr(model, related_field_name)
-    return result
+        if hasattr(model_field, 'activitypub_field'):
+            setattr(
+                activity,
+                getattr(model_field, 'activitypub_field'),
+                instance.remote_id
+            )
+        item = activity.to_model(model)
+
+        # if the related field isn't serialized (attachments on Status), then
+        # we have to set it post-creation
+        if not hasattr(model_field, 'activitypub_field'):
+            setattr(item, related_field_name, instance)
+            item.save()
 
 
-def tag_formatter(tags, tag_type):
+def resolve_remote_id(model, remote_id, refresh=False, save=True):
-    ''' helper function to extract foreign keys from tag activity json '''
+    ''' take a remote_id and return an instance, creating if necessary '''
-    if not isinstance(tags, list):
+    result = model.find_existing_by_remote_id(remote_id)
-        return []
+    if result and not refresh:
-    items = []
+        return result
-    types = {
-        'Book': models.Book,
-        'Mention': models.User,
-    }
-    for tag in [t for t in tags if t.get('type') == tag_type]:
-        if not tag_type in types:
-            continue
-        remote_id = tag.get('href')
-        try:
-            item = resolve_foreign_key(types[tag_type], remote_id)
-        except ActivitySerializerError:
-            continue
-        items.append(item)
-    return items
 
-
+    # load the data and create the object
-def image_formatter(image_slug):
-    ''' helper function to load images and format them for a model '''
-    # when it's an inline image (User avatar/icon, Book cover), it's a json
-    # blob, but when it's an attached image, it's just a url
-    if isinstance(image_slug, dict):
-        url = image_slug.get('url')
-    elif isinstance(image_slug, str):
-        url = image_slug
-    else:
-        return None
-    if not url:
-        return None
     try:
-        response = requests.get(url)
+        data = get_data(remote_id)
-    except ConnectionError:
+    except (ConnectorException, ConnectionError):
-        return None
+        raise ActivitySerializerError(
-    if not response.ok:
+            'Could not connect to host for remote_id in %s model: %s' % \
-        return None
+                (model.__name__, remote_id))
 
-    image_name = str(uuid4()) + '.' + url.split('.')[-1]
+    # check for existing items with shared unique identifiers
-    image_content = ContentFile(response.content)
+    if not result:
-    return [image_name, image_content]
+        result = model.find_existing(data)
+        if result and not refresh:
+            return result
+
+    item = model.activity_serializer(**data)
+    # if we're refreshing, "result" will be set and we'll update it
+    return item.to_model(model, instance=result, save=save)

bookwyrm/activitypub/book.py

@@ -12,13 +12,13 @@ class Book(ActivityObject):
     sortTitle: str = ''
     subtitle: str = ''
     description: str = ''
-    languages: List[str]
+    languages: List[str] = field(default_factory=lambda: [])
     series: str = ''
     seriesNumber: str = ''
-    subjects: List[str]
+    subjects: List[str] = field(default_factory=lambda: [])
-    subjectPlaces: List[str]
+    subjectPlaces: List[str] = field(default_factory=lambda: [])
 
-    authors: List[str]
+    authors: List[str] = field(default_factory=lambda: [])
     firstPublishedDate: str = ''
     publishedDate: str = ''
 
@@ -33,23 +33,25 @@ class Book(ActivityObject):
 @dataclass(init=False)
 class Edition(Book):
     ''' Edition instance of a book object '''
-    isbn10: str
-    isbn13: str
-    oclcNumber: str
-    asin: str
-    pages: str
-    physicalFormat: str
-    publishers: List[str]
-
     work: str
+    isbn10: str = ''
+    isbn13: str = ''
+    oclcNumber: str = ''
+    asin: str = ''
+    pages: int = None
+    physicalFormat: str = ''
+    publishers: List[str] = field(default_factory=lambda: [])
+    editionRank: int = 0
+
     type: str = 'Edition'
 
 
 @dataclass(init=False)
 class Work(Book):
     ''' work instance of a book object '''
-    lccn: str
+    lccn: str = ''
-    editions: List[str]
+    defaultEdition: str = ''
+    editions: List[str] = field(default_factory=lambda: [])
     type: str = 'Work'
 
 
@@ -57,10 +59,12 @@ class Work(Book):
 class Author(ActivityObject):
     ''' author of a book '''
     name: str
-    born: str = ''
+    born: str = None
-    died: str = ''
+    died: str = None
-    aliases: str = ''
+    aliases: List[str] = field(default_factory=lambda: [])
     bio: str = ''
     openlibraryKey: str = ''
+    librarythingKey: str = ''
+    goodreadsKey: str = ''
     wikipediaLink: str = ''
     type: str = 'Person'

bookwyrm/activitypub/note.py

@@ -8,7 +8,6 @@ from .image import Image
 @dataclass(init=False)
 class Tombstone(ActivityObject):
     ''' the placeholder for a deleted status '''
-    url: str
     published: str
     deleted: str
     type: str = 'Tombstone'
@@ -17,14 +16,14 @@ class Tombstone(ActivityObject):
 @dataclass(init=False)
 class Note(ActivityObject):
     ''' Note activity '''
-    url: str
-    inReplyTo: str
     published: str
     attributedTo: str
-    to: List[str]
-    cc: List[str]
     content: str
-    replies: Dict
+    to: List[str] = field(default_factory=lambda: [])
+    cc: List[str] = field(default_factory=lambda: [])
+    replies: Dict = field(default_factory=lambda: {})
+    inReplyTo: str = ''
+    summary: str = ''
     tag: List[Link] = field(default_factory=lambda: [])
     attachment: List[Image] = field(default_factory=lambda: [])
     sensitive: bool = False
@@ -54,8 +53,8 @@ class Comment(Note):
 @dataclass(init=False)
 class Review(Comment):
     ''' a full book review '''
-    name: str
+    name: str = None
-    rating: int
+    rating: int = None
     type: str = 'Review'
 
 

bookwyrm/activitypub/ordered_collection.py

@@ -12,6 +12,7 @@ class OrderedCollection(ActivityObject):
     first: str
     last: str = ''
     name: str = ''
+    owner: str = ''
     type: str = 'OrderedCollection'
 
 

bookwyrm/activitypub/person.py

@@ -2,20 +2,29 @@
 from dataclasses import dataclass, field
 from typing import Dict
 
-from .base_activity import ActivityObject, PublicKey
+from .base_activity import ActivityObject
 from .image import Image
 
+
+@dataclass(init=False)
+class PublicKey(ActivityObject):
+    ''' public key block '''
+    owner: str
+    publicKeyPem: str
+    type: str = 'PublicKey'
+
+
 @dataclass(init=False)
 class Person(ActivityObject):
     ''' actor activitypub json '''
     preferredUsername: str
-    name: str
     inbox: str
     outbox: str
     followers: str
-    summary: str
     publicKey: PublicKey
     endpoints: Dict
+    name: str = None
+    summary: str = None
     icon: Image = field(default_factory=lambda: {})
     bookwyrmUser: bool = False
     manuallyApprovesFollowers: str = False

bookwyrm/activitypub/response.py

@@ -0,0 +1,18 @@
+from django.http import JsonResponse
+
+from .base_activity import ActivityEncoder
+
+class ActivitypubResponse(JsonResponse):
+    """
+    A class to be used in any place that's serializing responses for
+    Activitypub enabled clients. Uses JsonResponse under the hood, but already
+    configures some stuff beforehand. Made to be a drop-in replacement of
+    JsonResponse.
+    """
+    def __init__(self, data, encoder=ActivityEncoder, safe=True,
+                 json_dumps_params=None, **kwargs):
+
+        if 'content_type' not in kwargs:
+            kwargs['content_type'] = 'application/activity+json'
+
+        super().__init__(data, encoder, safe, json_dumps_params, **kwargs)

bookwyrm/activitypub/verbs.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from typing import List
 
 from .base_activity import ActivityObject, Signature
+from .book import Edition
 
 @dataclass(init=False)
 class Verb(ActivityObject):
@@ -69,6 +70,13 @@ class Add(Verb):
     type: str = 'Add'
 
 
+@dataclass(init=False)
+class AddBook(Verb):
+    '''Add activity that's aware of the book obj '''
+    target: Edition
+    type: str = 'Add'
+
+
 @dataclass(init=False)
 class Remove(Verb):
     '''Remove activity '''

bookwyrm/broadcast.py

@@ -3,7 +3,7 @@ import json
 from django.utils.http import http_date
 import requests
 
-from bookwyrm import models
+from bookwyrm import models, settings
 from bookwyrm.activitypub import ActivityEncoder
 from bookwyrm.tasks import app
 from bookwyrm.signatures import make_signature, make_digest
@@ -65,7 +65,7 @@ def sign_and_send(sender, data, destination):
     ''' crpyto whatever and http junk '''
     now = http_date()
 
-    if not sender.private_key:
+    if not sender.key_pair.private_key:
         # this shouldn't happen. it would be bad if it happened.
         raise ValueError('No private key found for sender')
 
@@ -79,6 +79,7 @@ def sign_and_send(sender, data, destination):
             'Digest': digest,
             'Signature': make_signature(sender, destination, now, digest),
             'Content-Type': 'application/activity+json; charset=utf-8',
+            'User-Agent': settings.USER_AGENT,
         },
     )
     if not response.ok:

bookwyrm/connectors/__init__.py

@@ -1,3 +1,6 @@
 ''' bring connectors into the namespace '''
 from .settings import CONNECTORS
 from .abstract_connector import ConnectorException
+from .abstract_connector import get_data, get_image
+
+from .connector_manager import search, local_search, first_search_result

bookwyrm/connectors/abstract_connector.py

@@ -1,35 +1,25 @@
 ''' functionality outline for a book data connector '''
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
-import pytz
+import logging
 from urllib3.exceptions import RequestError
 
 from django.db import transaction
-from dateutil import parser
 import requests
-from requests import HTTPError
+from requests.exceptions import SSLError
 
-from bookwyrm import models
+from bookwyrm import activitypub, models, settings
+from .connector_manager import load_more_data, ConnectorException
 
 
-class ConnectorException(HTTPError):
+logger = logging.getLogger(__name__)
-    ''' when the connector can't do what was asked '''
+class AbstractMinimalConnector(ABC):
-
+    ''' just the bare bones, for other bookwyrm instances '''
-
-class AbstractConnector(ABC):
-    ''' generic book data connector '''
-
     def __init__(self, identifier):
         # load connector settings
         info = models.Connector.objects.get(identifier=identifier)
         self.connector = info
 
-        self.key_mappings = []
-
-        # fields we want to look for in book data to copy over
-        # title we handle separately.
-        self.book_mappings = []
-
         # the things in the connector model to copy over
         self_fields = [
             'base_url',
@@ -44,6 +34,54 @@ class AbstractConnector(ABC):
         for field in self_fields:
             setattr(self, field, getattr(info, field))
 
+    def search(self, query, min_confidence=None):
+        ''' free text search '''
+        params = {}
+        if min_confidence:
+            params['min_confidence'] = min_confidence
+
+        resp = requests.get(
+            '%s%s' % (self.search_url, query),
+            params=params,
+            headers={
+                'Accept': 'application/json; charset=utf-8',
+                'User-Agent': settings.USER_AGENT,
+            },
+        )
+        if not resp.ok:
+            resp.raise_for_status()
+        try:
+            data = resp.json()
+        except ValueError as e:
+            logger.exception(e)
+            raise ConnectorException('Unable to parse json response', e)
+        results = []
+
+        for doc in self.parse_search_data(data)[:10]:
+            results.append(self.format_search_result(doc))
+        return results
+
+    @abstractmethod
+    def get_or_create_book(self, remote_id):
+        ''' pull up a book record by whatever means possible '''
+
+    @abstractmethod
+    def parse_search_data(self, data):
+        ''' turn the result json from a search into a list '''
+
+    @abstractmethod
+    def format_search_result(self, search_result):
+        ''' create a SearchResult obj from json '''
+
+
+class AbstractConnector(AbstractMinimalConnector):
+    ''' generic book data connector '''
+    def __init__(self, identifier):
+        super().__init__(identifier)
+        # fields we want to look for in book data to copy over
+        # title we handle separately.
+        self.book_mappings = []
+
 
     def is_available(self):
         ''' check if you're allowed to use this connector '''
@@ -53,248 +91,113 @@ class AbstractConnector(ABC):
         return True
 
 
-    def search(self, query, min_confidence=None):
-        ''' free text search '''
-        resp = requests.get(
-            '%s%s' % (self.search_url, query),
-            headers={
-                'Accept': 'application/json; charset=utf-8',
-            },
-        )
-        if not resp.ok:
-            resp.raise_for_status()
-        data = resp.json()
-        results = []
-
-        for doc in self.parse_search_data(data)[:10]:
-            results.append(self.format_search_result(doc))
-        return results
-
-
     def get_or_create_book(self, remote_id):
-        ''' pull up a book record by whatever means possible '''
+        ''' translate arbitrary json into an Activitypub dataclass '''
-        # try to load the book
+        # first, check if we have the origin_id saved
-        book = models.Book.objects.select_subclasses().filter(
+        existing = models.Edition.find_existing_by_remote_id(remote_id) or \
-            origin_id=remote_id
+                models.Work.find_existing_by_remote_id(remote_id)
-        ).first()
+        if existing:
-        if book:
+            if hasattr(existing, 'get_default_editon'):
-            if isinstance(book, models.Work):
+                return existing.get_default_editon()
-                return book.default_edition
+            return existing
-            return book
 
-        # no book was found, so we start creating a new one
+        # load the json
         data = get_data(remote_id)
-
+        mapped_data = dict_from_mappings(data, self.book_mappings)
-        work = None
-        edition = None
         if self.is_work_data(data):
-            work_data = data
-            # if we requested a work and there's already an edition, we're set
-            work = self.match_from_mappings(work_data, models.Work)
-            if work and work.default_edition:
-                return work.default_edition
-
-            # no such luck, we need more information.
             try:
-                edition_data = self.get_edition_from_work_data(work_data)
+                edition_data = self.get_edition_from_work_data(data)
             except KeyError:
                 # hack: re-use the work data as the edition data
                 # this is why remote ids aren't necessarily unique
                 edition_data = data
+            work_data = mapped_data
         else:
-            edition_data = data
-            edition = self.match_from_mappings(edition_data, models.Edition)
-            # no need to figure out about the work if we already know about it
-            if edition and edition.parent_work:
-                return edition
-
-            # no such luck, we need more information.
             try:
-                work_data = self.get_work_from_edition_date(edition_data)
+                work_data = self.get_work_from_edition_data(data)
+                work_data = dict_from_mappings(work_data, self.book_mappings)
             except KeyError:
-                # remember this hack: re-use the work data as the edition data
+                work_data = mapped_data
-                work_data = data
+            edition_data = data
 
         if not work_data or not edition_data:
             raise ConnectorException('Unable to load book data: %s' % remote_id)
 
-        # at this point, we need to figure out the work, edition, or both
-        # atomic so that we don't save a work with no edition for vice versa
         with transaction.atomic():
-            if not work:
+            # create activitypub object
-                work_key = self.get_remote_id_from_data(work_data)
+            work_activity = activitypub.Work(**work_data)
-                work = self.create_book(work_key, work_data, models.Work)
+            # this will dedupe automatically
+            work = work_activity.to_model(models.Work)
+            for author in self.get_authors_from_data(data):
+                work.authors.add(author)
 
-            if not edition:
+            edition = self.create_edition_from_data(work, edition_data)
-                ed_key = self.get_remote_id_from_data(edition_data)
+        load_more_data.delay(self.connector.id, work.id)
-                edition = self.create_book(ed_key, edition_data, models.Edition)
+        return edition
-                edition.parent_work = work
-                edition.save()
-            work.default_edition = edition
-            work.save()
 
-        # now's our change to fill in author gaps
+
+    def create_edition_from_data(self, work, edition_data):
+        ''' if we already have the work, we're ready '''
+        mapped_data = dict_from_mappings(edition_data, self.book_mappings)
+        mapped_data['work'] = work.remote_id
+        edition_activity = activitypub.Edition(**mapped_data)
+        edition = edition_activity.to_model(models.Edition)
+        edition.connector = self.connector
+        edition.save()
+
+        work.default_edition = edition
+        work.save()
+
+        for author in self.get_authors_from_data(edition_data):
+            edition.authors.add(author)
         if not edition.authors.exists() and work.authors.exists():
             edition.authors.set(work.authors.all())
-            edition.author_text = work.author_text
-            edition.save()
-
-        if not edition:
-            raise ConnectorException('Unable to create book: %s' % remote_id)
 
         return edition
 
 
-    def create_book(self, remote_id, data, model):
+    def get_or_create_author(self, remote_id):
-        ''' create a work or edition from data '''
+        ''' load that author '''
-        book = model.objects.create(
+        existing = models.Author.find_existing_by_remote_id(remote_id)
-            origin_id=remote_id,
+        if existing:
-            title=data['title'],
+            return existing
-            connector=self.connector,
-        )
-        return self.update_book_from_data(book, data)
 
+        data = get_data(remote_id)
 
-    def update_book_from_data(self, book, data, update_cover=True):
+        mapped_data = dict_from_mappings(data, self.author_mappings)
-        ''' for creating a new book or syncing with data '''
+        activity = activitypub.Author(**mapped_data)
-        book = self.update_from_mappings(book, data, self.book_mappings)
+        # this will dedupe
-
+        return activity.to_model(models.Author)
-        author_text = []
-        for author in self.get_authors_from_data(data):
-            book.authors.add(author)
-            if author.display_name:
-                author_text.append(author.display_name)
-        book.author_text = ', '.join(author_text)
-        book.save()
-
-        if not update_cover:
-            return book
-
-        cover = self.get_cover_from_data(data)
-        if cover:
-            book.cover.save(*cover, save=True)
-        return book
-
-
-    def update_book(self, book, data=None):
-        ''' load new data '''
-        if not book.sync and not book.sync_cover:
-            return book
-
-        if not data:
-            key = getattr(book, self.key_name)
-            data = self.load_book_data(key)
-
-        if book.sync:
-            book = self.update_book_from_data(
-                book, data, update_cover=book.sync_cover)
-        else:
-            cover = self.get_cover_from_data(data)
-            if cover:
-                book.cover.save(*cover, save=True)
-
-        return book
-
-
-    def match_from_mappings(self, data, model):
-        ''' try to find existing copies of this book using various keys '''
-        relevent_mappings = [m for m in self.key_mappings if \
-                not m.model or model == m.model]
-        for mapping in relevent_mappings:
-            # check if this field is present in the data
-            value = data.get(mapping.remote_field)
-            if not value:
-                continue
-
-            # extract the value in the right format
-            value = mapping.formatter(value)
-
-            # search our database for a matching book
-            kwargs = {mapping.local_field: value}
-            match = model.objects.filter(**kwargs).first()
-            if match:
-                return match
-        return None
-
-
-    @abstractmethod
-    def get_remote_id_from_data(self, data):
-        ''' otherwise we won't properly set the remote_id in the db '''
 
 
     @abstractmethod
     def is_work_data(self, data):
         ''' differentiate works and editions '''
 
-
     @abstractmethod
     def get_edition_from_work_data(self, data):
         ''' every work needs at least one edition '''
 
-
     @abstractmethod
-    def get_work_from_edition_date(self, data):
+    def get_work_from_edition_data(self, data):
         ''' every edition needs a work '''
 
-
     @abstractmethod
     def get_authors_from_data(self, data):
         ''' load author data '''
 
-
-    @abstractmethod
-    def get_cover_from_data(self, data):
-        ''' load cover '''
-
-
-    @abstractmethod
-    def parse_search_data(self, data):
-        ''' turn the result json from a search into a list '''
-
-
-    @abstractmethod
-    def format_search_result(self, search_result):
-        ''' create a SearchResult obj from json '''
-
-
     @abstractmethod
     def expand_book_data(self, book):
         ''' get more info on a book '''
 
 
-    def update_from_mappings(self, obj, data, mappings):
+def dict_from_mappings(data, mappings):
-        ''' assign data to model with mappings '''
+    ''' create a dict in Activitypub format, using mappings supplies by
-        for mapping in mappings:
+    the subclass '''
-            # check if this field is present in the data
+    result = {}
-            value = data.get(mapping.remote_field)
+    for mapping in mappings:
-            if not value:
+        result[mapping.local_field] = mapping.get_value(data)
-                continue
+    return result
-
-            # extract the value in the right format
-            try:
-                value = mapping.formatter(value)
-            except:
-                continue
-
-            # assign the formatted value to the model
-            obj.__setattr__(mapping.local_field, value)
-        return obj
-
-
-def get_date(date_string):
-    ''' helper function to try to interpret dates '''
-    if not date_string:
-        return None
-
-    try:
-        return pytz.utc.localize(parser.parse(date_string))
-    except ValueError:
-        pass
-
-    try:
-        return parser.parse(date_string)
-    except ValueError:
-        return None
 
 
 def get_data(url):
@@ -304,16 +207,37 @@ def get_data(url):
             url,
             headers={
                 'Accept': 'application/json; charset=utf-8',
+                'User-Agent': settings.USER_AGENT,
             },
         )
-    except RequestError:
+    except (RequestError, SSLError):
         raise ConnectorException()
     if not resp.ok:
         resp.raise_for_status()
-    data = resp.json()
+    try:
+        data = resp.json()
+    except ValueError:
+        raise ConnectorException()
+
     return data
 
 
+def get_image(url):
+    ''' wrapper for requesting an image '''
+    try:
+        resp = requests.get(
+            url,
+            headers={
+                'User-Agent': settings.USER_AGENT,
+            },
+        )
+    except (RequestError, SSLError):
+        return None
+    if not resp.ok:
+        return None
+    return resp
+
+
 @dataclass
 class SearchResult:
     ''' standardized search result object '''
@@ -321,20 +245,35 @@ class SearchResult:
     key: str
     author: str
     year: str
+    connector: object
     confidence: int = 1
 
     def __repr__(self):
         return "<SearchResult key={!r} title={!r} author={!r}>".format(
             self.key, self.title, self.author)
 
+    def json(self):
+        ''' serialize a connector for json response '''
+        serialized = asdict(self)
+        del serialized['connector']
+        return serialized
+
 
 class Mapping:
     ''' associate a local database field with a field in an external dataset '''
-    def __init__(
+    def __init__(self, local_field, remote_field=None, formatter=None):
-            self, local_field, remote_field=None, formatter=None, model=None):
         noop = lambda x: x
 
         self.local_field = local_field
         self.remote_field = remote_field or local_field
         self.formatter = formatter or noop
-        self.model = model
+
+    def get_value(self, data):
+        ''' pull a field from incoming json and return the formatted version '''
+        value = data.get(self.remote_field)
+        if not value:
+            return None
+        try:
+            return self.formatter(value)
+        except:# pylint: disable=bare-except
+            return None

bookwyrm/connectors/bookwyrm_connector.py

@@ -1,83 +1,21 @@
 ''' using another bookwyrm instance as a source of book data '''
-from django.db import transaction
-
 from bookwyrm import activitypub, models
-from .abstract_connector import AbstractConnector, SearchResult
+from .abstract_connector import AbstractMinimalConnector, SearchResult
-from .abstract_connector import get_data
 
 
-class Connector(AbstractConnector):
+class Connector(AbstractMinimalConnector):
-    ''' interact with other instances '''
+    ''' this is basically just for search '''
-
-    def update_from_mappings(self, obj, data, mappings):
-        ''' serialize book data into a model '''
-        if self.is_work_data(data):
-            work_data = activitypub.Work(**data)
-            return work_data.to_model(models.Work, instance=obj)
-        edition_data = activitypub.Edition(**data)
-        return edition_data.to_model(models.Edition, instance=obj)
-
-
-    def get_remote_id_from_data(self, data):
-        return data.get('id')
-
-
-    def is_work_data(self, data):
-        return data.get('type') == 'Work'
-
-
-    def get_edition_from_work_data(self, data):
-        ''' we're served a list of edition urls '''
-        path = data['editions'][0]
-        return get_data(path)
-
-
-    def get_work_from_edition_date(self, data):
-        return get_data(data['work'])
-
-
-    def get_authors_from_data(self, data):
-        ''' load author data '''
-        for author_id in data.get('authors', []):
-            try:
-                yield models.Author.objects.get(origin_id=author_id)
-            except models.Author.DoesNotExist:
-                pass
-            data = get_data(author_id)
-            author_data = activitypub.Author(**data)
-            author = author_data.to_model(models.Author)
-            yield author
-
-
-    def get_cover_from_data(self, data):
-        pass
 
+    def get_or_create_book(self, remote_id):
+        edition = activitypub.resolve_remote_id(models.Edition, remote_id)
+        work = edition.parent_work
+        work.default_edition = work.get_default_edition()
+        work.save()
+        return edition
 
     def parse_search_data(self, data):
         return data
 
-
     def format_search_result(self, search_result):
+        search_result['connector'] = self
         return SearchResult(**search_result)
-
-
-    def expand_book_data(self, book):
-        work = book
-        # go from the edition to the work, if necessary
-        if isinstance(book, models.Edition):
-            work = book.parent_work
-
-        # it may be that we actually want to request this url
-        editions_url = '%s/editions?page=true' % work.remote_id
-        edition_options = get_data(editions_url)
-        for edition_data in edition_options['orderedItems']:
-            with transaction.atomic():
-                edition = self.create_book(
-                    edition_data['id'],
-                    edition_data,
-                    models.Edition
-                )
-                edition.parent_work = work
-                edition.save()
-            if not edition.authors.exists() and work.authors.exists():
-                edition.authors.set(work.authors.all())

bookwyrm/connectors/connector_manager.py

@@ -1,4 +1,4 @@
-''' select and call a connector for whatever book task needs doing '''
+''' interface with whatever connectors the app has '''
 import importlib
 from urllib.parse import urlparse
 
@@ -8,60 +8,8 @@ from bookwyrm import models
 from bookwyrm.tasks import app
 
 
-def get_edition(book_id):
+class ConnectorException(HTTPError):
-    ''' look up a book in the db and return an edition '''
+    ''' when the connector can't do what was asked '''
-    book = models.Book.objects.select_subclasses().get(id=book_id)
-    if isinstance(book, models.Work):
-        book = book.default_edition
-    return book
-
-
-def get_or_create_book(remote_id):
-    ''' pull up a book record by whatever means possible '''
-    book = models.Book.objects.select_subclasses().filter(
-        remote_id=remote_id
-    ).first()
-    if book:
-        return book
-
-    connector = get_or_create_connector(remote_id)
-
-    # raises ConnectorException
-    book = connector.get_or_create_book(remote_id)
-    if book:
-        load_more_data.delay(book.id)
-    return book
-
-
-def get_or_create_connector(remote_id):
-    ''' get the connector related to the author's server '''
-    url = urlparse(remote_id)
-    identifier = url.netloc
-    if not identifier:
-        raise ValueError('Invalid remote id')
-
-    try:
-        connector_info = models.Connector.objects.get(identifier=identifier)
-    except models.Connector.DoesNotExist:
-        connector_info = models.Connector.objects.create(
-            identifier=identifier,
-            connector_file='bookwyrm_connector',
-            base_url='https://%s' % identifier,
-            books_url='https://%s/book' % identifier,
-            covers_url='https://%s/images/covers' % identifier,
-            search_url='https://%s/search?q=' % identifier,
-            priority=2
-        )
-
-    return load_connector(connector_info)
-
-
-@app.task
-def load_more_data(book_id):
-    ''' background the work of getting all 10,000 editions of LoTR '''
-    book = models.Book.objects.select_subclasses().get(id=book_id)
-    connector = load_connector(book.connector)
-    connector.expand_book_data(book)
 
 
 def search(query, min_confidence=0.1):
@@ -72,7 +20,7 @@ def search(query, min_confidence=0.1):
     for connector in get_connectors():
         try:
             result_set = connector.search(query, min_confidence=min_confidence)
-        except HTTPError:
+        except (HTTPError, ConnectorException):
             continue
 
         result_set = [r for r in result_set \
@@ -102,18 +50,44 @@ def first_search_result(query, min_confidence=0.1):
     return None
 
 
-def update_book(book, data=None):
-    ''' re-sync with the original data source '''
-    connector = load_connector(book.connector)
-    connector.update_book(book, data=data)
-
-
 def get_connectors():
     ''' load all connectors '''
     for info in models.Connector.objects.order_by('priority').all():
         yield load_connector(info)
 
 
+def get_or_create_connector(remote_id):
+    ''' get the connector related to the author's server '''
+    url = urlparse(remote_id)
+    identifier = url.netloc
+    if not identifier:
+        raise ValueError('Invalid remote id')
+
+    try:
+        connector_info = models.Connector.objects.get(identifier=identifier)
+    except models.Connector.DoesNotExist:
+        connector_info = models.Connector.objects.create(
+            identifier=identifier,
+            connector_file='bookwyrm_connector',
+            base_url='https://%s' % identifier,
+            books_url='https://%s/book' % identifier,
+            covers_url='https://%s/images/covers' % identifier,
+            search_url='https://%s/search?q=' % identifier,
+            priority=2
+        )
+
+    return load_connector(connector_info)
+
+
+@app.task
+def load_more_data(connector_id, book_id):
+    ''' background the work of getting all 10,000 editions of LoTR '''
+    connector_info = models.Connector.objects.get(id=connector_id)
+    connector = load_connector(connector_info)
+    book = models.Book.objects.select_subclasses().get(id=book_id)
+    connector.expand_book_data(book)
+
+
 def load_connector(connector_info):
     ''' instantiate the connector class '''
     connector = importlib.import_module(

bookwyrm/connectors/openlibrary.py

@@ -1,13 +1,10 @@
 ''' openlibrary data connector '''
 import re
-import requests
-
-from django.core.files.base import ContentFile
 
 from bookwyrm import models
 from .abstract_connector import AbstractConnector, SearchResult, Mapping
-from .abstract_connector import ConnectorException
+from .abstract_connector import get_data
-from .abstract_connector import get_date, get_data
+from .connector_manager import ConnectorException
 from .openlibrary_languages import languages
 
 
@@ -17,66 +14,62 @@ class Connector(AbstractConnector):
         super().__init__(identifier)
 
         get_first = lambda a: a[0]
-        self.key_mappings = [
+        get_remote_id = lambda a: self.base_url + a
-            Mapping('isbn_13', model=models.Edition, formatter=get_first),
+        self.book_mappings = [
-            Mapping('isbn_10', model=models.Edition, formatter=get_first),
+            Mapping('title'),
-            Mapping('lccn', model=models.Work, formatter=get_first),
+            Mapping('id', remote_field='key', formatter=get_remote_id),
             Mapping(
-                'oclc_number',
+                'cover', remote_field='covers', formatter=self.get_cover_url),
-                remote_field='oclc_numbers',
+            Mapping('sortTitle', remote_field='sort_title'),
-                model=models.Edition,
-                formatter=get_first
-            ),
-            Mapping(
-                'openlibrary_key',
-                remote_field='key',
-                formatter=get_openlibrary_key
-            ),
-            Mapping('goodreads_key'),
-            Mapping('asin'),
-        ]
-
-        self.book_mappings = self.key_mappings + [
-            Mapping('sort_title'),
             Mapping('subtitle'),
             Mapping('description', formatter=get_description),
             Mapping('languages', formatter=get_languages),
             Mapping('series', formatter=get_first),
-            Mapping('series_number'),
+            Mapping('seriesNumber', remote_field='series_number'),
             Mapping('subjects'),
-            Mapping('subject_places'),
+            Mapping('subjectPlaces'),
+            Mapping('isbn13', formatter=get_first),
+            Mapping('isbn10', formatter=get_first),
+            Mapping('lccn', formatter=get_first),
             Mapping(
-                'first_published_date',
+                'oclcNumber', remote_field='oclc_numbers',
-                remote_field='first_publish_date',
+                formatter=get_first
-                formatter=get_date
             ),
             Mapping(
-                'published_date',
+                'openlibraryKey', remote_field='key',
-                remote_field='publish_date',
+                formatter=get_openlibrary_key
-                formatter=get_date
             ),
+            Mapping('goodreadsKey', remote_field='goodreads_key'),
+            Mapping('asin'),
             Mapping(
-                'pages',
+                'firstPublishedDate', remote_field='first_publish_date',
-                model=models.Edition,
-                remote_field='number_of_pages'
             ),
-            Mapping('physical_format', model=models.Edition),
+            Mapping('publishedDate', remote_field='publish_date'),
+            Mapping('pages', remote_field='number_of_pages'),
+            Mapping('physicalFormat', remote_field='physical_format'),
             Mapping('publishers'),
         ]
 
         self.author_mappings = [
-            Mapping('born', remote_field='birth_date', formatter=get_date),
+            Mapping('id', remote_field='key', formatter=get_remote_id),
-            Mapping('died', remote_field='death_date', formatter=get_date),
+            Mapping('name'),
+            Mapping(
+                'openlibraryKey', remote_field='key',
+                formatter=get_openlibrary_key
+            ),
+            Mapping('born', remote_field='birth_date'),
+            Mapping('died', remote_field='death_date'),
             Mapping('bio', formatter=get_description),
         ]
 
 
     def get_remote_id_from_data(self, data):
+        ''' format a url from an openlibrary id field '''
         try:
             key = data['key']
         except KeyError:
             raise ConnectorException('Invalid book data')
-        return '%s/%s' % (self.books_url, key)
+        return '%s%s' % (self.books_url, key)
 
 
     def is_work_data(self, data):
@@ -88,17 +81,17 @@ class Connector(AbstractConnector):
             key = data['key']
         except KeyError:
             raise ConnectorException('Invalid book data')
-        url = '%s/%s/editions' % (self.books_url, key)
+        url = '%s%s/editions' % (self.books_url, key)
         data = get_data(url)
         return pick_default_edition(data['entries'])
 
 
-    def get_work_from_edition_date(self, data):
+    def get_work_from_edition_data(self, data):
         try:
             key = data['works'][0]['key']
         except (IndexError, KeyError):
             raise ConnectorException('No work found for edition')
-        url = '%s/%s' % (self.books_url, key)
+        url = '%s%s' % (self.books_url, key)
         return get_data(url)
 
 
@@ -106,24 +99,17 @@ class Connector(AbstractConnector):
         ''' parse author json and load or create authors '''
         for author_blob in data.get('authors', []):
             author_blob = author_blob.get('author', author_blob)
-            # this id is "/authors/OL1234567A" and we want just "OL1234567A"
+            # this id is "/authors/OL1234567A"
-            author_id = author_blob['key'].split('/')[-1]
+            author_id = author_blob['key']
-            yield self.get_or_create_author(author_id)
+            url = '%s%s' % (self.base_url, author_id)
+            yield self.get_or_create_author(url)
 
 
-    def get_cover_from_data(self, data):
+    def get_cover_url(self, cover_blob):
         ''' ask openlibrary for the cover '''
-        if not data.get('covers'):
+        cover_id = cover_blob[0]
-            return None
+        image_name = '%s-L.jpg' % cover_id
-
+        return '%s/b/id/%s' % (self.covers_url, image_name)
-        cover_id = data.get('covers')[0]
-        image_name = '%s-M.jpg' % cover_id
-        url = '%s/b/id/%s' % (self.covers_url, image_name)
-        response = requests.get(url)
-        if not response.ok:
-            response.raise_for_status()
-        image_content = ContentFile(response.content)
-        return [image_name, image_content]
 
 
     def parse_search_data(self, data):
@@ -138,13 +124,14 @@ class Connector(AbstractConnector):
             title=search_result.get('title'),
             key=key,
             author=', '.join(author),
+            connector=self,
             year=search_result.get('first_publish_year'),
         )
 
 
     def load_edition_data(self, olkey):
         ''' query openlibrary for editions of a work '''
-        url = '%s/works/%s/editions.json' % (self.books_url, olkey)
+        url = '%s/works/%s/editions' % (self.books_url, olkey)
         return get_data(url)
 
 
@@ -157,49 +144,14 @@ class Connector(AbstractConnector):
         # we can mass download edition data from OL to avoid repeatedly querying
         edition_options = self.load_edition_data(work.openlibrary_key)
         for edition_data in edition_options.get('entries'):
-            olkey = edition_data.get('key').split('/')[-1]
+            self.create_edition_from_data(work, edition_data)
-            # make sure the edition isn't already in the database
-            if models.Edition.objects.filter(openlibrary_key=olkey).count():
-                continue
-
-            # creates and populates the book from the data
-            edition = self.create_book(olkey, edition_data, models.Edition)
-            # ensures that the edition is associated with the work
-            edition.parent_work = work
-            edition.save()
-            # get author data from the work if it's missing from the edition
-            if not edition.authors and work.authors:
-                edition.authors.set(work.authors.all())
-
-
-    def get_or_create_author(self, olkey):
-        ''' load that author '''
-        if not re.match(r'^OL\d+A$', olkey):
-            raise ValueError('Invalid OpenLibrary author ID')
-        author = models.Author.objects.filter(openlibrary_key=olkey).first()
-        if author:
-            return author
-
-        url = '%s/authors/%s.json' % (self.base_url, olkey)
-        data = get_data(url)
-
-        author = models.Author(openlibrary_key=olkey)
-        author = self.update_from_mappings(author, data, self.author_mappings)
-        name = data.get('name')
-        # TODO this is making some BOLD assumption
-        if name:
-            author.last_name = name.split(' ')[-1]
-            author.first_name = ' '.join(name.split(' ')[:-1])
-        author.save()
-
-        return author
 
 
 def get_description(description_blob):
     ''' descriptions can be a string or a dict '''
     if isinstance(description_blob, dict):
         return description_blob.get('value')
-    return  description_blob
+    return description_blob
 
 
 def get_openlibrary_key(key):
@@ -224,7 +176,7 @@ def pick_default_edition(options):
     if len(options) == 1:
         return options[0]
 
-    options = [e for e in options if e.get('cover')] or options
+    options = [e for e in options if e.get('covers')] or options
     options = [e for e in options if \
         '/languages/eng' in str(e.get('languages'))] or options
     formats = ['paperback', 'hardcover', 'mass market paperback']

bookwyrm/connectors/self_connector.py

@@ -1,6 +1,9 @@
 ''' using a bookwyrm instance as a source of book data '''
+from functools import reduce
+import operator
+
 from django.contrib.postgres.search import SearchRank, SearchVector
-from django.db.models import F
+from django.db.models import Count, F, Q
 
 from bookwyrm import models
 from .abstract_connector import AbstractConnector, SearchResult
@@ -9,38 +12,20 @@ from .abstract_connector import AbstractConnector, SearchResult
 class Connector(AbstractConnector):
     ''' instantiate a connector  '''
     def search(self, query, min_confidence=0.1):
-        ''' right now you can't search bookwyrm sorry, but when
+        ''' search your local database '''
-        that gets implemented it will totally rule '''
+        if not query:
-        vector = SearchVector('title', weight='A') +\
+            return []
-            SearchVector('subtitle', weight='B') +\
+        # first, try searching unqiue identifiers
-            SearchVector('author_text', weight='C') +\
+        results = search_identifiers(query)
-            SearchVector('isbn_13', weight='A') +\
+        if not results:
-            SearchVector('isbn_10', weight='A') +\
+            # then try searching title/author
-            SearchVector('openlibrary_key', weight='C') +\
+            results = search_title_author(query, min_confidence)
-            SearchVector('goodreads_key', weight='C') +\
-            SearchVector('asin', weight='C') +\
-            SearchVector('oclc_number', weight='C') +\
-            SearchVector('remote_id', weight='C') +\
-            SearchVector('description', weight='D') +\
-            SearchVector('series', weight='D')
-
-        results = models.Edition.objects.annotate(
-            search=vector
-        ).annotate(
-            rank=SearchRank(vector, query)
-        ).filter(
-            rank__gt=min_confidence
-        ).order_by('-rank')
-
-        # remove non-default editions, if possible
-        results = results.filter(parent_work__default_edition__id=F('id')) \
-                    or results
-
         search_results = []
-        for book in results[:10]:
+        for result in results:
-            search_results.append(
+            search_results.append(self.format_search_result(result))
-                self.format_search_result(book)
+            if len(search_results) >= 10:
-            )
+                break
+        search_results.sort(key=lambda r: r.confidence, reverse=True)
         return search_results
 
 
@@ -51,31 +36,74 @@ class Connector(AbstractConnector):
             author=search_result.author_text,
             year=search_result.published_date.year if \
                     search_result.published_date else None,
-            confidence=search_result.rank,
+            connector=self,
+            confidence=search_result.rank if \
+                    hasattr(search_result, 'rank') else 1,
         )
 
 
-    def get_remote_id_from_data(self, data):
-        pass
-
     def is_work_data(self, data):
         pass
 
     def get_edition_from_work_data(self, data):
         pass
 
-    def get_work_from_edition_date(self, data):
+    def get_work_from_edition_data(self, data):
         pass
 
     def get_authors_from_data(self, data):
         return None
 
-    def get_cover_from_data(self, data):
-        return None
-
     def parse_search_data(self, data):
         ''' it's already in the right format, don't even worry about it '''
         return data
 
     def expand_book_data(self, book):
         pass
+
+
+def search_identifiers(query):
+    ''' tries remote_id, isbn; defined as dedupe fields on the model '''
+    filters = [{f.name: query} for f in models.Edition._meta.get_fields() \
+        if hasattr(f, 'deduplication_field') and f.deduplication_field]
+    results = models.Edition.objects.filter(
+        reduce(operator.or_, (Q(**f) for f in filters))
+    ).distinct()
+
+    # when there are multiple editions of the same work, pick the default.
+    # it would be odd for this to happen.
+    return results.filter(parent_work__default_edition__id=F('id')) \
+            or results
+
+
+def search_title_author(query, min_confidence):
+    ''' searches for title and author '''
+    vector = SearchVector('title', weight='A') +\
+        SearchVector('subtitle', weight='B') +\
+        SearchVector('authors__name', weight='C') +\
+        SearchVector('series', weight='D')
+
+    results = models.Edition.objects.annotate(
+        search=vector
+    ).annotate(
+        rank=SearchRank(vector, query)
+    ).filter(
+        rank__gt=min_confidence
+    ).order_by('-rank')
+
+    # when there are multiple editions of the same work, pick the closest
+    editions_of_work = results.values(
+        'parent_work'
+    ).annotate(
+        Count('parent_work')
+    ).values_list('parent_work')
+
+    for work_id in set(editions_of_work):
+        editions = results.filter(parent_work=work_id)
+        default = editions.filter(parent_work__default_edition=F('id'))
+        default_rank = default.first().rank if default.exists() else 0
+        # if mutliple books have the top rank, pick the default edition
+        if default_rank == editions.first().rank:
+            yield default.first()
+        else:
+            yield editions.first()

bookwyrm/context_processors.py

@@ -0,0 +1,8 @@
+''' customize the info available in context for rendering templates '''
+from bookwyrm import models
+
+def site_settings(request):# pylint: disable=unused-argument
+    ''' include the custom info about the site '''
+    return {
+        'site': models.SiteSettings.objects.get()
+    }

bookwyrm/forms.py

@@ -31,10 +31,11 @@ class CustomForm(ModelForm):
             visible.field.widget.attrs['class'] = css_classes[input_type]
 
 
+# pylint: disable=missing-class-docstring
 class LoginForm(CustomForm):
     class Meta:
         model = models.User
-        fields = ['username', 'password']
+        fields = ['localname', 'password']
         help_texts = {f: None for f in fields}
         widgets = {
             'password': PasswordInput(),
@@ -44,7 +45,7 @@ class LoginForm(CustomForm):
 class RegisterForm(CustomForm):
     class Meta:
         model = models.User
-        fields = ['username', 'email', 'password']
+        fields = ['localname', 'email', 'password']
         help_texts = {f: None for f in fields}
         widgets = {
             'password': PasswordInput()
@@ -60,25 +61,36 @@ class RatingForm(CustomForm):
 class ReviewForm(CustomForm):
     class Meta:
         model = models.Review
-        fields = ['user', 'book', 'name', 'content', 'rating', 'privacy']
+        fields = [
+            'user', 'book',
+            'name', 'content', 'rating',
+            'content_warning', 'sensitive',
+            'privacy']
 
 
 class CommentForm(CustomForm):
     class Meta:
         model = models.Comment
-        fields = ['user', 'book', 'content', 'privacy']
+        fields = [
+            'user', 'book', 'content',
+            'content_warning', 'sensitive',
+            'privacy']
 
 
 class QuotationForm(CustomForm):
     class Meta:
         model = models.Quotation
-        fields = ['user', 'book', 'quote', 'content', 'privacy']
+        fields = [
+            'user', 'book', 'quote', 'content',
+            'content_warning', 'sensitive', 'privacy']
 
 
 class ReplyForm(CustomForm):
     class Meta:
         model = models.Status
-        fields = ['user', 'content', 'reply_parent', 'privacy']
+        fields = [
+            'user', 'content', 'content_warning', 'sensitive',
+            'reply_parent', 'privacy']
 
 
 class EditUserForm(CustomForm):
@@ -110,14 +122,14 @@ class EditionForm(CustomForm):
         model = models.Edition
         exclude = [
             'remote_id',
+            'origin_id',
             'created_date',
             'updated_date',
-            'last_sync_date',
+            'edition_rank',
 
             'authors',# TODO
             'parent_work',
             'shelves',
-            'misc_identifiers',
 
             'subjects',# TODO
             'subject_places',# TODO
@@ -125,12 +137,23 @@ class EditionForm(CustomForm):
             'connector',
         ]
 
+class AuthorForm(CustomForm):
+    class Meta:
+        model = models.Author
+        exclude = [
+            'remote_id',
+            'origin_id',
+            'created_date',
+            'updated_date',
+        ]
+
 
 class ImportForm(forms.Form):
     csv_file = forms.FileField()
 
 class ExpiryWidget(widgets.Select):
     def value_from_datadict(self, data, files, name):
+        ''' human-readable exiration time buckets '''
         selected_string = super().value_from_datadict(data, files, name)
 
         if selected_string == 'day':

bookwyrm/goodreads_import.py

@@ -8,8 +8,6 @@ from bookwyrm.models import ImportJob, ImportItem
 from bookwyrm.status import create_notification
 
 logger = logging.getLogger(__name__)
-# TODO: remove or increase once we're confident it's not causing problems.
-MAX_ENTRIES = 500
 
 
 def create_job(user, csv_file, include_reviews, privacy):
@@ -19,12 +17,13 @@ def create_job(user, csv_file, include_reviews, privacy):
         include_reviews=include_reviews,
         privacy=privacy
     )
-    for index, entry in enumerate(list(csv.DictReader(csv_file))[:MAX_ENTRIES]):
+    for index, entry in enumerate(list(csv.DictReader(csv_file))):
         if not all(x in entry for x in ('ISBN13', 'Title', 'Author')):
             raise ValueError('Author, title, and isbn must be in data.')
         ImportItem(job=job, index=index, data=entry).save()
     return job
 
+
 def create_retry_job(user, original_job, items):
     ''' retry items that didn't import '''
     job = ImportJob.objects.create(
@@ -37,6 +36,7 @@ def create_retry_job(user, original_job, items):
         ImportItem(job=job, index=item.index, data=item.data).save()
     return job
 
+
 def start_import(job):
     ''' initalizes a csv import job '''
     result = import_data.delay(job.id)
@@ -49,11 +49,10 @@ def import_data(job_id):
     ''' does the actual lookup work in a celery task '''
     job = ImportJob.objects.get(id=job_id)
     try:
-        results = []
         for item in job.items.all():
             try:
                 item.resolve()
-            except Exception as e:
+            except Exception as e:# pylint: disable=broad-except
                 logger.exception(e)
                 item.fail_reason = 'Error loading book'
                 item.save()
@@ -61,7 +60,6 @@ def import_data(job_id):
 
             if item.book:
                 item.save()
-                results.append(item)
 
                 # shelves book and handles reviews
                 outgoing.handle_imported_book(
@@ -71,3 +69,5 @@ def import_data(job_id):
                 item.save()
     finally:
         create_notification(job.user, 'IMPORT', related_import=job)
+        job.complete = True
+        job.save()

bookwyrm/incoming.py

@@ -1,26 +1,24 @@
 ''' handles all of the activity coming in to the server '''
 import json
-from urllib.parse import urldefrag, unquote_plus
+from urllib.parse import urldefrag
 
 import django.db.utils
 from django.http import HttpResponse
 from django.http import HttpResponseBadRequest, HttpResponseNotFound
 from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_POST
 import requests
 
-from bookwyrm import activitypub, books_manager, models, outgoing
+from bookwyrm import activitypub, models, outgoing
 from bookwyrm import status as status_builder
-from bookwyrm.remote_user import get_or_create_remote_user, refresh_remote_user
 from bookwyrm.tasks import app
 from bookwyrm.signatures import Signature
 
 
 @csrf_exempt
+@require_POST
 def inbox(request, username):
     ''' incoming activitypub events '''
-    # TODO: should do some kind of checking if the user accepts
-    # this action from the sender probably? idk
-    # but this will just throw a 404 if the user doesn't exist
     try:
         models.User.objects.get(localname=username)
     except models.User.DoesNotExist:
@@ -30,11 +28,9 @@ def inbox(request, username):
 
 
 @csrf_exempt
+@require_POST
 def shared_inbox(request):
     ''' incoming activitypub events '''
-    if request.method == 'GET':
-        return HttpResponseNotFound()
-
     try:
         resp = request.body
         activity = json.loads(resp)
@@ -60,9 +56,7 @@ def shared_inbox(request):
         'Like': handle_favorite,
         'Announce': handle_boost,
         'Add': {
-            'Tag': handle_tag,
+            'Edition': handle_add,
-            'Edition': handle_shelve,
-            'Work': handle_shelve,
         },
         'Undo': {
             'Follow': handle_unfollow,
@@ -71,8 +65,8 @@ def shared_inbox(request):
         },
         'Update': {
             'Person': handle_update_user,
-            'Edition': handle_update_book,
+            'Edition': handle_update_edition,
-            'Work': handle_update_book,
+            'Work': handle_update_work,
         },
     }
     activity_type = activity['type']
@@ -97,16 +91,20 @@ def has_valid_signature(request, activity):
         if key_actor != activity.get('actor'):
             raise ValueError("Wrong actor created signature.")
 
-        remote_user = get_or_create_remote_user(key_actor)
+        remote_user = activitypub.resolve_remote_id(models.User, key_actor)
+        if not remote_user:
+            return False
 
         try:
-            signature.verify(remote_user.public_key, request)
+            signature.verify(remote_user.key_pair.public_key, request)
         except ValueError:
-            old_key = remote_user.public_key
+            old_key = remote_user.key_pair.public_key
-            refresh_remote_user(remote_user)
+            remote_user = activitypub.resolve_remote_id(
-            if remote_user.public_key == old_key:
+                models.User, remote_user.remote_id, refresh=True
+            )
+            if remote_user.key_pair.public_key == old_key:
                 raise # Key unchanged.
-            signature.verify(remote_user.public_key, request)
+            signature.verify(remote_user.key_pair.public_key, request)
     except (ValueError, requests.exceptions.HTTPError):
         return False
     return True
@@ -115,26 +113,10 @@ def has_valid_signature(request, activity):
 @app.task
 def handle_follow(activity):
     ''' someone wants to follow a local user '''
-    # figure out who they want to follow -- not using get_or_create because
-    # we only care if you want to follow local users
     try:
-        to_follow = models.User.objects.get(remote_id=activity['object'])
+        relationship = activitypub.Follow(
-    except models.User.DoesNotExist:
+            **activity
-        # some rando, who cares
+        ).to_model(models.UserFollowRequest)
-        return
-    if not to_follow.local:
-        # just ignore follow alerts about other servers. maybe they should be
-        # handled. maybe they shouldn't be sent at all.
-        return
-
-    # figure out who the actor is
-    actor = get_or_create_remote_user(activity['actor'])
-    try:
-        relationship = models.UserFollowRequest.objects.create(
-            user_subject=actor,
-            user_object=to_follow,
-            remote_id=activity['id']
-        )
     except django.db.utils.IntegrityError as err:
         if err.__cause__.diag.constraint_name != 'userfollowrequest_unique':
             raise
@@ -143,27 +125,22 @@ def handle_follow(activity):
         )
         # send the accept normally for a duplicate request
 
-    if not to_follow.manually_approves_followers:
+    manually_approves = relationship.user_object.manually_approves_followers
-        status_builder.create_notification(
+
-            to_follow,
+    status_builder.create_notification(
-            'FOLLOW',
+        relationship.user_object,
-            related_user=actor
+        'FOLLOW_REQUEST' if manually_approves else 'FOLLOW',
-        )
+        related_user=relationship.user_subject
+    )
+    if not manually_approves:
         outgoing.handle_accept(relationship)
-    else:
-        # Accept will be triggered manually
-        status_builder.create_notification(
-            to_follow,
-            'FOLLOW_REQUEST',
-            related_user=actor
-        )
 
 
 @app.task
 def handle_unfollow(activity):
     ''' unfollow a local user '''
     obj = activity['object']
-    requester = get_or_create_remote_user(obj['actor'])
+    requester = activitypub.resolve_remote_id(models.User, obj['actor'])
     to_unfollow = models.User.objects.get(remote_id=obj['object'])
     # raises models.User.DoesNotExist
 
@@ -176,7 +153,7 @@ def handle_follow_accept(activity):
     # figure out who they want to follow
     requester = models.User.objects.get(remote_id=activity['object']['actor'])
     # figure out who they are
-    accepter = get_or_create_remote_user(activity['actor'])
+    accepter = activitypub.resolve_remote_id(models.User, activity['actor'])
 
     try:
         request = models.UserFollowRequest.objects.get(
@@ -193,7 +170,7 @@ def handle_follow_accept(activity):
 def handle_follow_reject(activity):
     ''' someone is rejecting a follow request '''
     requester = models.User.objects.get(remote_id=activity['object']['actor'])
-    rejecter = get_or_create_remote_user(activity['actor'])
+    rejecter = activitypub.resolve_remote_id(models.User, activity['actor'])
 
     request = models.UserFollowRequest.objects.get(
         user_subject=requester,
@@ -206,33 +183,49 @@ def handle_follow_reject(activity):
 @app.task
 def handle_create(activity):
     ''' someone did something, good on them '''
-    if activity['object'].get('type') not in \
-            ['Note', 'Comment', 'Quotation', 'Review', 'GeneratedNote']:
-        # if it's an article or unknown type, ignore it
-        return
-
-    user = get_or_create_remote_user(activity['actor'])
-    if user.local:
-        # we really oughtn't even be sending in this case
-        return
-
     # deduplicate incoming activities
-    status_id = activity['object']['id']
+    activity = activity['object']
+    status_id = activity.get('id')
     if models.Status.objects.filter(remote_id=status_id).count():
         return
 
-    status = status_builder.create_status(activity['object'])
+    try:
+        serializer = activitypub.activity_objects[activity['type']]
+    except KeyError:
+        return
+
+    activity = serializer(**activity)
+    try:
+        model = models.activity_models[activity.type]
+    except KeyError:
+        # not a type of status we are prepared to deserialize
+        return
+
+    status = activity.to_model(model)
     if not status:
+        # it was discarded because it's not a bookwyrm type
         return
 
     # create a notification if this is a reply
+    notified = []
     if status.reply_parent and status.reply_parent.user.local:
+        notified.append(status.reply_parent.user)
         status_builder.create_notification(
             status.reply_parent.user,
             'REPLY',
             related_user=status.user,
             related_status=status,
         )
+    if status.mention_users.exists():
+        for mentioned_user in status.mention_users.all():
+            if not mentioned_user.local or mentioned_user in notified:
+                continue
+            status_builder.create_notification(
+                mentioned_user,
+                'MENTION',
+                related_user=status.user,
+                related_status=status,
+            )
 
 
 @app.task
@@ -245,11 +238,12 @@ def handle_delete_status(activity):
         # is trying to delete a user.
         return
     try:
-        status = models.Status.objects.select_subclasses().get(
+        status = models.Status.objects.get(
             remote_id=status_id
         )
     except models.Status.DoesNotExist:
         return
+    models.Notification.objects.filter(related_status=status).all().delete()
     status_builder.delete_status(status)
 
 
@@ -257,17 +251,18 @@ def handle_delete_status(activity):
 def handle_favorite(activity):
     ''' approval of your good good post '''
     fav = activitypub.Like(**activity)
-
+    # we dont know this status, we don't care about this status
-    liker = get_or_create_remote_user(activity['actor'])
+    if not models.Status.objects.filter(remote_id=fav.object).exists():
-    if liker.local:
         return
 
     fav = fav.to_model(models.Favorite)
+    if fav.user.local:
+        return
 
     status_builder.create_notification(
         fav.status.user,
         'FAVORITE',
-        related_user=liker,
+        related_user=fav.user,
         related_status=fav.status,
     )
 
@@ -312,35 +307,13 @@ def handle_unboost(activity):
 
 
 @app.task
-def handle_tag(activity):
+def handle_add(activity):
-    ''' someone is tagging a book '''
-    user = get_or_create_remote_user(activity['actor'])
-    if not user.local:
-        # ordered collection weirndess so we can't just to_model
-        book = books_manager.get_or_create_book(activity['object']['id'])
-        name = activity['object']['target'].split('/')[-1]
-        name = unquote_plus(name)
-        models.Tag.objects.get_or_create(
-            user=user,
-            book=book,
-            name=name
-        )
-
-
-@app.task
-def handle_shelve(activity):
     ''' putting a book on a shelf '''
-    user = get_or_create_remote_user(activity['actor'])
+    #this is janky as heck but I haven't thought of a better solution
-    book = books_manager.get_or_create_book(activity['object'])
     try:
-        shelf = models.Shelf.objects.get(remote_id=activity['target'])
+        activitypub.AddBook(**activity).to_model(models.ShelfBook)
-    except models.Shelf.DoesNotExist:
+    except activitypub.ActivitySerializerError:
-        return
+        activitypub.AddBook(**activity).to_model(models.Tag)
-    if shelf.user != user:
-        # this doesn't add up.
-        return
-    shelf.books.add(book)
-    shelf.save()
 
 
 @app.task
@@ -358,15 +331,12 @@ def handle_update_user(activity):
 
 
 @app.task
-def handle_update_book(activity):
+def handle_update_edition(activity):
     ''' a remote instance changed a book (Document) '''
-    document = activity['object']
+    activitypub.Edition(**activity['object']).to_model(models.Edition)
-    # check if we have their copy and care about their updates
-    book = models.Book.objects.select_subclasses().filter(
-        remote_id=document['id'],
-        sync=True,
-    ).first()
-    if not book:
-        return
 
-    books_manager.update_book(book, data=document)
+
+@app.task
+def handle_update_work(activity):
+    ''' a remote instance changed a book (Document) '''
+    activitypub.Work(**activity['object']).to_model(models.Work)

bookwyrm/management/commands/deduplicate_book_data.py

@@ -0,0 +1,83 @@
+''' PROCEED WITH CAUTION: uses deduplication fields to permanently
+merge book data objects '''
+from django.core.management.base import BaseCommand
+from django.db.models import Count
+from bookwyrm import models
+
+
+def update_related(canonical, obj):
+    ''' update all the models with fk to the object being removed '''
+    # move related models to canonical
+    related_models = [
+        (r.remote_field.name, r.related_model) for r in \
+            canonical._meta.related_objects]
+    for (related_field, related_model) in related_models:
+        related_objs = related_model.objects.filter(
+            **{related_field: obj})
+        for related_obj in related_objs:
+            print(
+                'replacing in',
+                related_model.__name__,
+                related_field,
+                related_obj.id
+            )
+            try:
+                setattr(related_obj, related_field, canonical)
+                related_obj.save()
+            except TypeError:
+                getattr(related_obj, related_field).add(canonical)
+                getattr(related_obj, related_field).remove(obj)
+
+
+def copy_data(canonical, obj):
+    ''' try to get the most data possible '''
+    for data_field in obj._meta.get_fields():
+        if not hasattr(data_field, 'activitypub_field'):
+            continue
+        data_value = getattr(obj, data_field.name)
+        if not data_value:
+            continue
+        if not getattr(canonical, data_field.name):
+            print('setting data field', data_field.name, data_value)
+            setattr(canonical, data_field.name, data_value)
+    canonical.save()
+
+
+def dedupe_model(model):
+    ''' combine duplicate editions and update related models '''
+    fields = model._meta.get_fields()
+    dedupe_fields = [f for f in fields if \
+            hasattr(f, 'deduplication_field') and f.deduplication_field]
+    for field in dedupe_fields:
+        dupes = model.objects.values(field.name).annotate(
+            Count(field.name)
+        ).filter(**{'%s__count__gt' % field.name: 1})
+
+        for dupe in dupes:
+            value = dupe[field.name]
+            if not value or value == '':
+                continue
+            print('----------')
+            print(dupe)
+            objs = model.objects.filter(
+                **{field.name: value}
+            ).order_by('id')
+            canonical = objs.first()
+            print('keeping', canonical.remote_id)
+            for obj in objs[1:]:
+                print(obj.remote_id)
+                copy_data(canonical, obj)
+                update_related(canonical, obj)
+                # remove the outdated entry
+                obj.delete()
+
+
+class Command(BaseCommand):
+    ''' dedplucate allllll the book data models '''
+    help = 'merges duplicate book data'
+    # pylint: disable=no-self-use,unused-argument
+    def handle(self, *args, **options):
+        ''' run deudplications '''
+        dedupe_model(models.Edition)
+        dedupe_model(models.Work)
+        dedupe_model(models.Author)

bookwyrm/management/commands/initdb.py

@@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand, CommandError
 from django.contrib.auth.models import Group, Permission
 from django.contrib.contenttypes.models import ContentType
 
-from bookwyrm.models import Connector, User
+from bookwyrm.models import Connector, SiteSettings, User
 from bookwyrm.settings import DOMAIN
 
 def init_groups():
@@ -73,7 +73,7 @@ def init_connectors():
         identifier='bookwyrm.social',
         name='BookWyrm dot Social',
         connector_file='bookwyrm_connector',
-        base_url='https://bookwyrm.social' ,
+        base_url='https://bookwyrm.social',
         books_url='https://bookwyrm.social/book',
         covers_url='https://bookwyrm.social/images/covers',
         search_url='https://bookwyrm.social/search?q=',
@@ -91,10 +91,14 @@ def init_connectors():
         priority=3,
     )
 
+def init_settings():
+    SiteSettings.objects.create()
+
 class Command(BaseCommand):
     help = 'Initializes the database with starter data'
 
     def handle(self, *args, **options):
         init_groups()
         init_permissions()
-        init_connectors()
+        init_connectors()
+        init_settings()

bookwyrm/migrations/0001_initial.py

@@ -7,7 +7,7 @@ import django.core.validators
 from django.db import migrations, models
 import django.db.models.deletion
 import django.utils.timezone
-import bookwyrm.utils.fields
+from django.contrib.postgres.fields import JSONField
 
 
 class Migration(migrations.Migration):
@@ -62,7 +62,7 @@ class Migration(migrations.Migration):
                 ('content', models.TextField(blank=True, null=True)),
                 ('created_date', models.DateTimeField(auto_now_add=True)),
                 ('openlibrary_key', models.CharField(max_length=255)),
-                ('data', bookwyrm.utils.fields.JSONField()),
+                ('data', JSONField()),
             ],
             options={
                 'abstract': False,
@@ -75,7 +75,7 @@ class Migration(migrations.Migration):
                 ('content', models.TextField(blank=True, null=True)),
                 ('created_date', models.DateTimeField(auto_now_add=True)),
                 ('openlibrary_key', models.CharField(max_length=255, unique=True)),
-                ('data', bookwyrm.utils.fields.JSONField()),
+                ('data', JSONField()),
                 ('cover', models.ImageField(blank=True, null=True, upload_to='covers/')),
                 ('added_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
                 ('authors', models.ManyToManyField(to='bookwyrm.Author')),

bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py

@@ -2,7 +2,6 @@
 
 import bookwyrm.models.connector
 import bookwyrm.models.site
-import bookwyrm.utils.fields
 from django.conf import settings
 import django.contrib.postgres.operations
 import django.core.validators
@@ -10,6 +9,7 @@ from django.db import migrations, models
 import django.db.models.deletion
 import django.db.models.expressions
 import django.utils.timezone
+from django.contrib.postgres.fields import JSONField, ArrayField
 import uuid
 
 
@@ -148,7 +148,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='misc_identifiers',
-            field=bookwyrm.utils.fields.JSONField(null=True),
+            field=JSONField(null=True),
         ),
         migrations.AddField(
             model_name='book',
@@ -226,7 +226,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='author',
             name='aliases',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='user',
@@ -394,17 +394,17 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='subject_places',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='book',
             name='subjects',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='edition',
             name='publishers',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AlterField(
             model_name='connector',
@@ -578,7 +578,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='book',
             name='languages',
-            field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
         ),
         migrations.AddField(
             model_name='edition',
@@ -676,7 +676,7 @@ class Migration(migrations.Migration):
             name='ImportItem',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('data', bookwyrm.utils.fields.JSONField()),
+                ('data', JSONField()),
             ],
         ),
         migrations.CreateModel(

bookwyrm/migrations/0016_auto_20201129_0304.py

@@ -0,0 +1,62 @@
+# Generated by Django 3.0.7 on 2020-11-29 03:04
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+from django.contrib.postgres.fields import ArrayField
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0015_auto_20201128_0349'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='book',
+            name='subject_places',
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subjects',
+            field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='parent_work',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=models.CharField(max_length=100, unique=True),
+        ),
+        migrations.AlterUniqueTogether(
+            name='tag',
+            unique_together=set(),
+        ),
+        migrations.RemoveField(
+            model_name='tag',
+            name='book',
+        ),
+        migrations.RemoveField(
+            model_name='tag',
+            name='user',
+        ),
+        migrations.CreateModel(
+            name='UserTag',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_date', models.DateTimeField(auto_now_add=True)),
+                ('updated_date', models.DateTimeField(auto_now=True)),
+                ('remote_id', models.CharField(max_length=255, null=True)),
+                ('book', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition')),
+                ('tag', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'unique_together': {('user', 'book', 'tag')},
+            },
+        ),
+    ]

bookwyrm/migrations/0016_auto_20201211_2026.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.0.7 on 2020-12-11 20:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0015_auto_20201128_0349'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sitesettings',
+            name='admin_email',
+            field=models.EmailField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='sitesettings',
+            name='support_link',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='sitesettings',
+            name='support_title',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

bookwyrm/migrations/0017_auto_20201130_1819.py

@@ -0,0 +1,189 @@
+# Generated by Django 3.0.7 on 2020-11-30 18:19
+
+import bookwyrm.models.base_model
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+def copy_rsa_keys(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    users = app_registry.get_model('bookwyrm', 'User')
+    keypair = app_registry.get_model('bookwyrm', 'KeyPair')
+    for user in users.objects.using(db_alias):
+        if user.public_key or user.private_key:
+            user.key_pair = keypair.objects.create(
+                remote_id='%s/#main-key' % user.remote_id,
+                private_key=user.private_key,
+                public_key=user.public_key
+            )
+            user.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0016_auto_20201129_0304'),
+    ]
+    operations = [
+        migrations.CreateModel(
+            name='KeyPair',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_date', models.DateTimeField(auto_now_add=True)),
+                ('updated_date', models.DateTimeField(auto_now=True)),
+                ('remote_id', bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id])),
+                ('private_key', models.TextField(blank=True, null=True)),
+                ('public_key', bookwyrm.models.fields.TextField(blank=True, null=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(bookwyrm.models.base_model.ActivitypubMixin, models.Model),
+        ),
+        migrations.AddField(
+            model_name='user',
+            name='followers',
+            field=bookwyrm.models.fields.ManyToManyField(related_name='following', through='bookwyrm.UserFollows', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='connector',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='favorite',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='federatedserver',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='image',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='notification',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='readthrough',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='avatar',
+            field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='avatars/'),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='bookwyrm_user',
+            field=bookwyrm.models.fields.BooleanField(default=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='inbox',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='local',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='manually_approves_followers',
+            field=bookwyrm.models.fields.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='name',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='outbox',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='shared_inbox',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='summary',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='username',
+            field=bookwyrm.models.fields.UsernameField(),
+        ),
+        migrations.AlterField(
+            model_name='userblocks',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='userfollowrequest',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='userfollows',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='remote_id',
+            field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]),
+        ),
+        migrations.AddField(
+            model_name='user',
+            name='key_pair',
+            field=bookwyrm.models.fields.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='owner', to='bookwyrm.KeyPair'),
+        ),
+        migrations.RunPython(copy_rsa_keys),
+    ]

bookwyrm/migrations/0017_auto_20201212_0059.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.0.7 on 2020-12-12 00:59
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0016_auto_20201211_2026'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='readthrough',
+            name='book',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+    ]

bookwyrm/migrations/0018_auto_20201130_1832.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.0.7 on 2020-11-30 18:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0017_auto_20201130_1819'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='user',
+            name='following',
+        ),
+        migrations.RemoveField(
+            model_name='user',
+            name='private_key',
+        ),
+        migrations.RemoveField(
+            model_name='user',
+            name='public_key',
+        ),
+    ]

bookwyrm/migrations/0019_auto_20201130_1939.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.0.7 on 2020-11-30 19:39
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+def update_notnull(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    users = app_registry.get_model('bookwyrm', 'User')
+    for user in users.objects.using(db_alias):
+        if user.name and user.summary:
+            continue
+        if not user.summary:
+            user.summary = ''
+        if not user.name:
+            user.name = ''
+        user.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0018_auto_20201130_1832'),
+    ]
+
+    operations = [
+        migrations.RunPython(update_notnull),
+        migrations.AlterField(
+            model_name='user',
+            name='name',
+            field=bookwyrm.models.fields.CharField(default='', max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='summary',
+            field=bookwyrm.models.fields.TextField(default=''),
+        ),
+    ]

bookwyrm/migrations/0020_auto_20201208_0213.py

@@ -0,0 +1,353 @@
+# Generated by Django 3.0.7 on 2020-12-08 02:13
+
+import bookwyrm.models.fields
+from django.conf import settings
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0019_auto_20201130_1939'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='author',
+            name='aliases',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='bio',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='born',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='died',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='openlibrary_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='wikipedia_link',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='authors',
+            field=bookwyrm.models.fields.ManyToManyField(to='bookwyrm.Author'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='cover',
+            field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='covers/'),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='description',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='first_published_date',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='goodreads_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='languages',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='librarything_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='openlibrary_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='published_date',
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='series',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='series_number',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='sort_title',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subject_places',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subjects',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='subtitle',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='title',
+            field=bookwyrm.models.fields.CharField(max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='boost',
+            name='boosted_status',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='boosters', to='bookwyrm.Status'),
+        ),
+        migrations.AlterField(
+            model_name='comment',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='asin',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='isbn_10',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='isbn_13',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='oclc_number',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='pages',
+            field=bookwyrm.models.fields.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='parent_work',
+            field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='physical_format',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='edition',
+            name='publishers',
+            field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None),
+        ),
+        migrations.AlterField(
+            model_name='favorite',
+            name='status',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'),
+        ),
+        migrations.AlterField(
+            model_name='favorite',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='image',
+            name='caption',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='image',
+            name='image',
+            field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='status/'),
+        ),
+        migrations.AlterField(
+            model_name='quotation',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='quotation',
+            name='quote',
+            field=bookwyrm.models.fields.TextField(),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='rating',
+            field=bookwyrm.models.fields.IntegerField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(5)]),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='privacy',
+            field=bookwyrm.models.fields.CharField(choices=[('public', 'Public'), ('unlisted', 'Unlisted'), ('followers', 'Followers'), ('direct', 'Direct')], default='public', max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='shelf',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='added_by',
+            field=bookwyrm.models.fields.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='shelfbook',
+            name='shelf',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Shelf'),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='content',
+            field=bookwyrm.models.fields.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='mention_books',
+            field=bookwyrm.models.fields.TagField(related_name='mention_book', to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='mention_users',
+            field=bookwyrm.models.fields.TagField(related_name='mention_user', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='published_date',
+            field=bookwyrm.models.fields.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='reply_parent',
+            field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='sensitive',
+            field=bookwyrm.models.fields.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=bookwyrm.models.fields.CharField(max_length=100, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='userblocks',
+            name='user_object',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_object', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userblocks',
+            name='user_subject',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_subject', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollowrequest',
+            name='user_object',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_object', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollowrequest',
+            name='user_subject',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_subject', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollows',
+            name='user_object',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_object', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='userfollows',
+            name='user_subject',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_subject', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='book',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='tag',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag'),
+        ),
+        migrations.AlterField(
+            model_name='usertag',
+            name='user',
+            field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='work',
+            name='default_edition',
+            field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+        ),
+        migrations.AlterField(
+            model_name='work',
+            name='lccn',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

bookwyrm/migrations/0021_merge_20201212_1737.py

@@ -0,0 +1,14 @@
+# Generated by Django 3.0.7 on 2020-12-12 17:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0020_auto_20201208_0213'),
+        ('bookwyrm', '0016_auto_20201211_2026'),
+    ]
+
+    operations = [
+    ]

bookwyrm/migrations/0022_auto_20201212_1744.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.0.7 on 2020-12-12 17:44
+
+from django.db import migrations
+
+
+def set_author_name(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    authors = app_registry.get_model('bookwyrm', 'Author')
+    for author in authors.objects.using(db_alias):
+        if not author.name:
+            author.name = '%s %s' % (author.first_name, author.last_name)
+            author.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0021_merge_20201212_1737'),
+    ]
+
+    operations = [
+        migrations.RunPython(set_author_name),
+        migrations.RemoveField(
+            model_name='author',
+            name='first_name',
+        ),
+        migrations.RemoveField(
+            model_name='author',
+            name='last_name',
+        ),
+    ]

bookwyrm/migrations/0023_auto_20201214_0511.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.0.7 on 2020-12-14 05:11
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0022_auto_20201212_1744'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='status',
+            name='privacy',
+            field=bookwyrm.models.fields.PrivacyField(choices=[('public', 'Public'), ('unlisted', 'Unlisted'), ('followers', 'Followers'), ('direct', 'Direct')], default='public', max_length=255),
+        ),
+    ]

bookwyrm/migrations/0023_merge_20201216_0112.py

@@ -0,0 +1,14 @@
+# Generated by Django 3.0.7 on 2020-12-16 01:12
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0017_auto_20201212_0059'),
+        ('bookwyrm', '0022_auto_20201212_1744'),
+    ]
+
+    operations = [
+    ]

bookwyrm/migrations/0024_merge_20201216_1721.py

@@ -0,0 +1,14 @@
+# Generated by Django 3.0.7 on 2020-12-16 17:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0023_auto_20201214_0511'),
+        ('bookwyrm', '0023_merge_20201216_0112'),
+    ]
+
+    operations = [
+    ]

bookwyrm/migrations/0025_auto_20201217_0046.py

@@ -0,0 +1,39 @@
+# Generated by Django 3.0.7 on 2020-12-17 00:46
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0024_merge_20201216_1721'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='author',
+            name='bio',
+            field=bookwyrm.models.fields.HtmlField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='description',
+            field=bookwyrm.models.fields.HtmlField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='quotation',
+            name='quote',
+            field=bookwyrm.models.fields.HtmlField(),
+        ),
+        migrations.AlterField(
+            model_name='status',
+            name='content',
+            field=bookwyrm.models.fields.HtmlField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='summary',
+            field=bookwyrm.models.fields.HtmlField(default=''),
+        ),
+    ]

bookwyrm/migrations/0026_status_content_warning.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.0.7 on 2020-12-17 03:17
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0025_auto_20201217_0046'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='status',
+            name='content_warning',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=500, null=True),
+        ),
+    ]

bookwyrm/migrations/0027_auto_20201220_2007.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.0.7 on 2020-12-20 20:07
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0026_status_content_warning'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='name',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='summary',
+            field=bookwyrm.models.fields.HtmlField(blank=True, null=True),
+        ),
+    ]

bookwyrm/migrations/0028_remove_book_author_text.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.0.7 on 2020-12-21 19:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0027_auto_20201220_2007'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='book',
+            name='author_text',
+        ),
+    ]

bookwyrm/migrations/0029_auto_20201221_2014.py

@@ -0,0 +1,61 @@
+# Generated by Django 3.0.7 on 2020-12-21 20:14
+
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0028_remove_book_author_text'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='author',
+            name='last_sync_date',
+        ),
+        migrations.RemoveField(
+            model_name='author',
+            name='sync',
+        ),
+        migrations.RemoveField(
+            model_name='book',
+            name='last_sync_date',
+        ),
+        migrations.RemoveField(
+            model_name='book',
+            name='sync',
+        ),
+        migrations.RemoveField(
+            model_name='book',
+            name='sync_cover',
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='goodreads_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='last_edited_by',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='librarything_key',
+            field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='last_edited_by',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='origin_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

bookwyrm/migrations/0030_auto_20201224_1939.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.0.7 on 2020-12-24 19:39
+
+import bookwyrm.models.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0029_auto_20201221_2014'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='localname',
+            field=models.CharField(max_length=255, null=True, unique=True, validators=[bookwyrm.models.fields.validate_localname]),
+        ),
+    ]

bookwyrm/migrations/0031_auto_20210104_2040.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.0.7 on 2021-01-04 20:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0030_auto_20201224_1939'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sitesettings',
+            name='favicon',
+            field=models.ImageField(blank=True, null=True, upload_to='logos/'),
+        ),
+        migrations.AddField(
+            model_name='sitesettings',
+            name='logo',
+            field=models.ImageField(blank=True, null=True, upload_to='logos/'),
+        ),
+        migrations.AddField(
+            model_name='sitesettings',
+            name='logo_small',
+            field=models.ImageField(blank=True, null=True, upload_to='logos/'),
+        ),
+    ]

bookwyrm/migrations/0032_auto_20210104_2055.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0.7 on 2021-01-04 20:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0031_auto_20210104_2040'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sitesettings',
+            name='instance_tagline',
+            field=models.CharField(default='Social Reading and Reviewing', max_length=150),
+        ),
+        migrations.AddField(
+            model_name='sitesettings',
+            name='registration_closed_text',
+            field=models.TextField(default='Contact an administrator to get an invite'),
+        ),
+    ]

bookwyrm/migrations/0033_siteinvite_created_date.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.0.7 on 2021-01-05 19:08
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0032_auto_20210104_2055'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='siteinvite',
+            name='created_date',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+    ]

bookwyrm/migrations/0034_importjob_complete.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.7 on 2021-01-07 16:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0033_siteinvite_created_date'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='importjob',
+            name='complete',
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0035_edition_edition_rank.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.0.7 on 2021-01-11 17:18
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+def set_rank(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    books = app_registry.get_model('bookwyrm', 'Edition')
+    for book in books.objects.using(db_alias):
+        book.edition_rank = book.get_rank
+        book.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0034_importjob_complete'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='edition',
+            name='edition_rank',
+            field=bookwyrm.models.fields.IntegerField(default=0),
+        ),
+        migrations.RunPython(set_rank),
+    ]

bookwyrm/models/__init__.py

@@ -2,19 +2,22 @@
 import inspect
 import sys
 
-from .book import Book, Work, Edition
+from .book import Book, Work, Edition, BookDataModel
 from .author import Author
 from .connector import Connector
 
 from .shelf import Shelf, ShelfBook
 
 from .status import Status, GeneratedNote, Review, Comment, Quotation
-from .status import Favorite, Boost, Notification, ReadThrough, ProgressMode, ProgressUpdate
+from .status import Boost
 from .attachment import Image
+from .favorite import Favorite
+from .notification import Notification
+from .readthrough import ReadThrough, ProgressUpdate, ProgressMode
 
-from .tag import Tag
+from .tag import Tag, UserTag
 
-from .user import User
+from .user import User, KeyPair
 from .relationship import UserFollows, UserFollowRequest, UserBlocks
 from .federated_server import FederatedServer
 
@@ -25,3 +28,6 @@ from .site import SiteSettings, SiteInvite, PasswordReset
 cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass)
 activity_models = {c[1].activity_serializer.__name__: c[1] \
     for c in cls_members if hasattr(c[1], 'activity_serializer')}
+
+status_models = [
+    c.__name__ for (_, c) in activity_models.items() if issubclass(c, Status)]

bookwyrm/models/attachment.py

@@ -3,7 +3,8 @@ from django.db import models
 
 from bookwyrm import activitypub
 from .base_model import ActivitypubMixin
-from .base_model import ActivityMapping, BookWyrmModel
+from .base_model import BookWyrmModel
+from . import fields
 
 
 class Attachment(ActivitypubMixin, BookWyrmModel):
@@ -14,19 +15,16 @@ class Attachment(ActivitypubMixin, BookWyrmModel):
         related_name='attachments',
         null=True
     )
+    reverse_unfurl = True
     class Meta:
         ''' one day we'll have other types of attachments besides images '''
         abstract = True
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('url', 'image'),
-        ActivityMapping('name', 'caption'),
-    ]
 
 class Image(Attachment):
     ''' an image attachment '''
-    image = models.ImageField(upload_to='status/', null=True, blank=True)
+    image = fields.ImageField(
-    caption = models.TextField(null=True, blank=True)
+        upload_to='status/', null=True, blank=True, activitypub_field='url')
+    caption = fields.TextField(null=True, blank=True, activitypub_field='name')
 
     activity_serializer = activitypub.Image

bookwyrm/models/author.py

@@ -1,50 +1,28 @@
 ''' database schema for info about authors '''
 from django.db import models
-from django.utils import timezone
 
 from bookwyrm import activitypub
-from bookwyrm.utils.fields import ArrayField
+from bookwyrm.settings import DOMAIN
 
-from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel
+from .book import BookDataModel
+from . import fields
 
 
-class Author(ActivitypubMixin, BookWyrmModel):
+class Author(BookDataModel):
     ''' basic biographic info '''
-    origin_id = models.CharField(max_length=255, null=True)
+    wikipedia_link = fields.CharField(
-    ''' copy of an author from OL '''
+        max_length=255, blank=True, null=True, deduplication_field=True)
-    openlibrary_key = models.CharField(max_length=255, blank=True, null=True)
-    sync = models.BooleanField(default=True)
-    last_sync_date = models.DateTimeField(default=timezone.now)
-    wikipedia_link = models.CharField(max_length=255, blank=True, null=True)
     # idk probably other keys would be useful here?
-    born = models.DateTimeField(blank=True, null=True)
+    born = fields.DateTimeField(blank=True, null=True)
-    died = models.DateTimeField(blank=True, null=True)
+    died = fields.DateTimeField(blank=True, null=True)
-    name = models.CharField(max_length=255)
+    name = fields.CharField(max_length=255, deduplication_field=True)
-    last_name = models.CharField(max_length=255, blank=True, null=True)
+    aliases = fields.ArrayField(
-    first_name = models.CharField(max_length=255, blank=True, null=True)
-    aliases = ArrayField(
         models.CharField(max_length=255), blank=True, default=list
     )
-    bio = models.TextField(null=True, blank=True)
+    bio = fields.HtmlField(null=True, blank=True)
 
-    @property
+    def get_remote_id(self):
-    def display_name(self):
+        ''' editions and works both use "book" instead of model_name '''
-        ''' Helper to return a displayable name'''
+        return 'https://%s/author/%s' % (DOMAIN, self.id)
-        if self.name:
-            return self.name
-        # don't want to return a spurious space if all of these are None
-        if self.first_name and self.last_name:
-            return self.first_name + ' ' + self.last_name
-        return self.last_name or self.first_name
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('name', 'name'),
-        ActivityMapping('born', 'born'),
-        ActivityMapping('died', 'died'),
-        ActivityMapping('aliases', 'aliases'),
-        ActivityMapping('bio', 'bio'),
-        ActivityMapping('openlibraryKey', 'openlibrary_key'),
-        ActivityMapping('wikipediaLink', 'wikipedia_link'),
-    ]
     activity_serializer = activitypub.Author

bookwyrm/models/base_model.py

@@ -1,34 +1,27 @@
 ''' base model with default fields '''
-from datetime import datetime
 from base64 import b64encode
-from dataclasses import dataclass
+from functools import reduce
-from typing import Callable
+import operator
 from uuid import uuid4
-from urllib.parse import urlencode
 
 from Crypto.PublicKey import RSA
 from Crypto.Signature import pkcs1_15
 from Crypto.Hash import SHA256
+from django.core.paginator import Paginator
 from django.db import models
-from django.db.models.fields.files import ImageFieldFile
+from django.db.models import Q
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import DOMAIN, PAGE_LENGTH
+from .fields import ImageField, ManyToManyField, RemoteIdField
 
 
-PrivacyLevels = models.TextChoices('Privacy', [
-    'public',
-    'unlisted',
-    'followers',
-    'direct'
-])
-
 class BookWyrmModel(models.Model):
     ''' shared fields '''
     created_date = models.DateTimeField(auto_now_add=True)
     updated_date = models.DateTimeField(auto_now=True)
-    remote_id = models.CharField(max_length=255, null=True)
+    remote_id = RemoteIdField(null=True, activitypub_field='id')
 
     def get_remote_id(self):
         ''' generate a url that resolves to the local object '''
@@ -42,8 +35,14 @@ class BookWyrmModel(models.Model):
         ''' this is just here to provide default fields for other models '''
         abstract = True
 
+    @property
+    def local_path(self):
+        ''' how to link to this object in the local app '''
+        return self.get_remote_id().replace('https://%s' % DOMAIN, '')
+
 
 @receiver(models.signals.post_save)
+#pylint: disable=unused-argument
 def execute_after_save(sender, instance, created, *args, **kwargs):
     ''' set the remote_id after save (when the id is available) '''
     if not created or not hasattr(instance, 'get_remote_id'):
@@ -53,58 +52,116 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
         instance.save()
 
 
+def unfurl_related_field(related_field, sort_field=None):
+    ''' load reverse lookups (like public key owner or Status attachment '''
+    if hasattr(related_field, 'all'):
+        return [unfurl_related_field(i) for i in related_field.order_by(
+            sort_field).all()]
+    if related_field.reverse_unfurl:
+        return related_field.field_to_activity()
+    return related_field.remote_id
+
+
 class ActivitypubMixin:
     ''' add this mixin for models that are AP serializable '''
     activity_serializer = lambda: {}
+    reverse_unfurl = False
 
-    def to_activity(self, pure=False):
+    def __init__(self, *args, **kwargs):
-        ''' convert from a model to an activity '''
+        ''' collect some info on model fields '''
-        if pure:
+        self.image_fields = []
-            # works around bookwyrm-specific fields for vanilla AP services
+        self.many_to_many_fields = []
-            mappings = self.pure_activity_mappings
+        self.simple_fields = [] # "simple"
-        else:
+        for field in self._meta.get_fields():
-            # may include custom fields that bookwyrm instances will understand
+            if not hasattr(field, 'field_to_activity'):
-            mappings = self.activity_mappings
-
-        fields = {}
-        for mapping in mappings:
-            if not hasattr(self, mapping.model_key) or not mapping.activity_key:
-                # this field on the model isn't serialized
                 continue
-            value = getattr(self, mapping.model_key)
-            if hasattr(value, 'remote_id'):
-                # this is probably a foreign key field, which we want to
-                # serialize as just the remote_id url reference
-                value = value.remote_id
-            elif isinstance(value, datetime):
-                value = value.isoformat()
-            elif isinstance(value, ImageFieldFile):
-                value = image_formatter(value)
 
-            # run the custom formatter function set in the model
+            if isinstance(field, ImageField):
-            formatted_value = mapping.activity_formatter(value)
+                self.image_fields.append(field)
-            if mapping.activity_key in fields and \
+            elif isinstance(field, ManyToManyField):
-                    isinstance(fields[mapping.activity_key], list):
+                self.many_to_many_fields.append(field)
-                # there can be two database fields that map to the same AP list
-                # this happens in status tags, which combines user and book tags
-                fields[mapping.activity_key] += formatted_value
             else:
-                fields[mapping.activity_key] = formatted_value
+                self.simple_fields.append(field)
 
-        if pure:
+        self.activity_fields = self.image_fields + \
-            return self.pure_activity_serializer(
+                self.many_to_many_fields + self.simple_fields
-                **fields
+
-            ).serialize()
+        self.deserialize_reverse_fields = self.deserialize_reverse_fields \
-        return self.activity_serializer(
+                if hasattr(self, 'deserialize_reverse_fields') else []
-            **fields
+        self.serialize_reverse_fields = self.serialize_reverse_fields \
-        ).serialize()
+                if hasattr(self, 'serialize_reverse_fields') else []
+
+        super().__init__(*args, **kwargs)
 
 
-    def to_create_activity(self, user, pure=False):
+    @classmethod
+    def find_existing_by_remote_id(cls, remote_id):
+        ''' look up a remote id in the db '''
+        return cls.find_existing({'id': remote_id})
+
+    @classmethod
+    def find_existing(cls, data):
+        ''' compare data to fields that can be used for deduplation.
+        This always includes remote_id, but can also be unique identifiers
+        like an isbn for an edition '''
+        filters = []
+        for field in cls._meta.get_fields():
+            if not hasattr(field, 'deduplication_field') or \
+                    not field.deduplication_field:
+                continue
+
+            value = data.get(field.get_activitypub_field())
+            if not value:
+                continue
+            filters.append({field.name: value})
+
+        if hasattr(cls, 'origin_id') and 'id' in data:
+            # kinda janky, but this handles special case for books
+            filters.append({'origin_id': data['id']})
+
+        if not filters:
+            # if there are no deduplication fields, it will match the first
+            # item no matter what. this shouldn't happen but just in case.
+            return None
+
+        objects = cls.objects
+        if hasattr(objects, 'select_subclasses'):
+            objects = objects.select_subclasses()
+
+        # an OR operation on all the match fields
+        match = objects.filter(
+            reduce(
+                operator.or_, (Q(**f) for f in filters)
+            )
+        )
+        # there OUGHT to be only one match
+        return match.first()
+
+
+    def to_activity(self):
+        ''' convert from a model to an activity '''
+        activity = {}
+        for field in self.activity_fields:
+            field.set_activity_from_field(activity, self)
+
+        if hasattr(self, 'serialize_reverse_fields'):
+            # for example, editions of a work
+            for model_field_name, activity_field_name, sort_field in \
+                    self.serialize_reverse_fields:
+                related_field = getattr(self, model_field_name)
+                activity[activity_field_name] = \
+                        unfurl_related_field(related_field, sort_field)
+
+        if not activity.get('id'):
+            activity['id'] = self.get_remote_id()
+        return self.activity_serializer(**activity).serialize()
+
+
+    def to_create_activity(self, user, **kwargs):
         ''' returns the object wrapped in a Create activity '''
-        activity_object = self.to_activity(pure=pure)
+        activity_object = self.to_activity(**kwargs)
 
-        signer = pkcs1_15.new(RSA.import_key(user.private_key))
+        signer = pkcs1_15.new(RSA.import_key(user.key_pair.private_key))
         content = activity_object['content']
         signed_message = signer.sign(SHA256.new(content.encode('utf8')))
         create_id = self.remote_id + '/activity'
@@ -118,8 +175,8 @@ class ActivitypubMixin:
         return activitypub.Create(
             id=create_id,
             actor=user.remote_id,
-            to=['%s/followers' % user.remote_id],
+            to=activity_object['to'],
-            cc=['https://www.w3.org/ns/activitystreams#Public'],
+            cc=activity_object['cc'],
             object=activity_object,
             signature=signature,
         ).serialize()
@@ -127,21 +184,18 @@ class ActivitypubMixin:
 
     def to_delete_activity(self, user):
         ''' notice of deletion '''
-        # this should be a tombstone
-        activity_object = self.to_activity()
-
         return activitypub.Delete(
             id=self.remote_id + '/activity',
             actor=user.remote_id,
             to=['%s/followers' % user.remote_id],
             cc=['https://www.w3.org/ns/activitystreams#Public'],
-            object=activity_object,
+            object=self.to_activity(),
         ).serialize()
 
 
     def to_update_activity(self, user):
         ''' wrapper for Updates to an activity '''
-        activity_id = '%s#update/%s' % (user.remote_id, uuid4())
+        activity_id = '%s#update/%s' % (self.remote_id, uuid4())
         return activitypub.Update(
             id=activity_id,
             actor=user.remote_id,
@@ -153,10 +207,10 @@ class ActivitypubMixin:
     def to_undo_activity(self, user):
         ''' undo an action '''
         return activitypub.Undo(
-            id='%s#undo' % user.remote_id,
+            id='%s#undo' % self.remote_id,
             actor=user.remote_id,
             object=self.to_activity()
-        )
+        ).serialize()
 
 
 class OrderedCollectionPageMixin(ActivitypubMixin):
@@ -167,77 +221,55 @@ class OrderedCollectionPageMixin(ActivitypubMixin):
         ''' this can be overriden if there's a special remote id, ie outbox '''
         return self.remote_id
 
-    def page(self, min_id=None, max_id=None):
-        ''' helper function to create the pagination url '''
-        params = {'page': 'true'}
-        if min_id:
-            params['min_id'] = min_id
-        if max_id:
-            params['max_id'] = max_id
-        return '?%s' % urlencode(params)
-
-    def next_page(self, items):
-        ''' use the max id of the last item '''
-        if not items.count():
-            return ''
-        return self.page(max_id=items[items.count() - 1].id)
-
-    def prev_page(self, items):
-        ''' use the min id of the first item '''
-        if not items.count():
-            return ''
-        return self.page(min_id=items[0].id)
-
-    def to_ordered_collection_page(self, queryset, remote_id, \
-            id_only=False, min_id=None, max_id=None):
-        ''' serialize and pagiante a queryset '''
-        # TODO: weird place to define this
-        limit = 20
-        # filters for use in the django queryset min/max
-        filters = {}
-        if min_id is not None:
-            filters['id__gt'] = min_id
-        if max_id is not None:
-            filters['id__lte'] = max_id
-        page_id = self.page(min_id=min_id, max_id=max_id)
-
-        items = queryset.filter(
-            **filters
-        ).all()[:limit]
-
-        if id_only:
-            page = [s.remote_id for s in items]
-        else:
-            page = [s.to_activity() for s in items]
-        return activitypub.OrderedCollectionPage(
-            id='%s%s' % (remote_id, page_id),
-            partOf=remote_id,
-            orderedItems=page,
-            next='%s%s' % (remote_id, self.next_page(items)),
-            prev='%s%s' % (remote_id, self.prev_page(items))
-        ).serialize()
 
     def to_ordered_collection(self, queryset, \
             remote_id=None, page=False, **kwargs):
         ''' an ordered collection of whatevers '''
         remote_id = remote_id or self.remote_id
         if page:
-            return self.to_ordered_collection_page(
+            return to_ordered_collection_page(
                 queryset, remote_id, **kwargs)
-        name = ''
+        name = self.name if hasattr(self, 'name') else None
-        if hasattr(self, 'name'):
+        owner = self.user.remote_id if hasattr(self, 'user') else ''
-            name = self.name
 
-        size = queryset.count()
+        paginated = Paginator(queryset, PAGE_LENGTH)
         return activitypub.OrderedCollection(
             id=remote_id,
-            totalItems=size,
+            totalItems=paginated.count,
             name=name,
-            first='%s%s' % (remote_id, self.page()),
+            owner=owner,
-            last='%s%s' % (remote_id, self.page(min_id=0))
+            first='%s?page=1' % remote_id,
+            last='%s?page=%d' % (remote_id, paginated.num_pages)
         ).serialize()
 
 
+# pylint: disable=unused-argument
+def to_ordered_collection_page(
+        queryset, remote_id, id_only=False, page=1, **kwargs):
+    ''' serialize and pagiante a queryset '''
+    paginated = Paginator(queryset, PAGE_LENGTH)
+
+    activity_page = paginated.page(page)
+    if id_only:
+        items = [s.remote_id for s in activity_page.object_list]
+    else:
+        items = [s.to_activity() for s in activity_page.object_list]
+
+    prev_page = next_page = None
+    if activity_page.has_next():
+        next_page = '%s?page=%d' % (remote_id, activity_page.next_page_number())
+    if activity_page.has_previous():
+        prev_page = '%s?page=%d' % \
+                (remote_id, activity_page.previous_page_number())
+    return activitypub.OrderedCollectionPage(
+        id='%s?page=%s' % (remote_id, page),
+        partOf=remote_id,
+        orderedItems=items,
+        next=next_page,
+        prev=prev_page
+    ).serialize()
+
+
 class OrderedCollectionMixin(OrderedCollectionPageMixin):
     ''' extends activitypub models to work as ordered collections '''
     @property
@@ -250,39 +282,3 @@ class OrderedCollectionMixin(OrderedCollectionPageMixin):
     def to_activity(self, **kwargs):
         ''' an ordered collection of the specified model queryset  '''
         return self.to_ordered_collection(self.collection_queryset, **kwargs)
-
-
-@dataclass(frozen=True)
-class ActivityMapping:
-    ''' translate between an activitypub json field and a model field '''
-    activity_key: str
-    model_key: str
-    activity_formatter: Callable = lambda x: x
-    model_formatter: Callable = lambda x: x
-
-
-def tag_formatter(items, name_field, activity_type):
-    ''' helper function to format lists of foreign keys into Tags '''
-    tags = []
-    for item in items.all():
-        tags.append(activitypub.Link(
-            href=item.remote_id,
-            name=getattr(item, name_field),
-            type=activity_type
-        ))
-    return tags
-
-
-def image_formatter(image):
-    ''' convert images into activitypub json '''
-    if image and hasattr(image, 'url'):
-        url = image.url
-    else:
-        return None
-    url = 'https://%s%s' % (DOMAIN, url)
-    return activitypub.Image(url=url)
-
-
-def image_attachments_formatter(images):
-    ''' create a list of image attachments '''
-    return [image_formatter(i) for i in images]

bookwyrm/models/book.py

@@ -2,111 +2,104 @@
 import re
 
 from django.db import models
-from django.db.models import Q
-from django.utils import timezone
 from model_utils.managers import InheritanceManager
 
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
-from bookwyrm.utils.fields import ArrayField
 
-from .base_model import ActivityMapping, BookWyrmModel
+from .base_model import BookWyrmModel
 from .base_model import ActivitypubMixin, OrderedCollectionPageMixin
+from . import fields
 
-class Book(ActivitypubMixin, BookWyrmModel):
+class BookDataModel(ActivitypubMixin, BookWyrmModel):
-    ''' a generic book, which can mean either an edition or a work '''
+    ''' fields shared between editable book data (books, works, authors) '''
     origin_id = models.CharField(max_length=255, null=True, blank=True)
-    # these identifiers apply to both works and editions
+    openlibrary_key = fields.CharField(
-    openlibrary_key = models.CharField(max_length=255, blank=True, null=True)
+        max_length=255, blank=True, null=True, deduplication_field=True)
-    librarything_key = models.CharField(max_length=255, blank=True, null=True)
+    librarything_key = fields.CharField(
-    goodreads_key = models.CharField(max_length=255, blank=True, null=True)
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    goodreads_key = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
 
-    # info about where the data comes from and where/if to sync
+    last_edited_by = models.ForeignKey(
-    sync = models.BooleanField(default=True)
+        'User', on_delete=models.PROTECT, null=True)
-    sync_cover = models.BooleanField(default=True)
+
-    last_sync_date = models.DateTimeField(default=timezone.now)
+    class Meta:
+        ''' can't initialize this model, that wouldn't make sense '''
+        abstract = True
+
+    def save(self, *args, **kwargs):
+        ''' ensure that the remote_id is within this instance '''
+        if self.id:
+            self.remote_id = self.get_remote_id()
+        else:
+            self.origin_id = self.remote_id
+            self.remote_id = None
+        return super().save(*args, **kwargs)
+
+
+class Book(BookDataModel):
+    ''' a generic book, which can mean either an edition or a work '''
     connector = models.ForeignKey(
         'Connector', on_delete=models.PROTECT, null=True)
 
-    # TODO: edit history
-
     # book/work metadata
-    title = models.CharField(max_length=255)
+    title = fields.CharField(max_length=255)
-    sort_title = models.CharField(max_length=255, blank=True, null=True)
+    sort_title = fields.CharField(max_length=255, blank=True, null=True)
-    subtitle = models.CharField(max_length=255, blank=True, null=True)
+    subtitle = fields.CharField(max_length=255, blank=True, null=True)
-    description = models.TextField(blank=True, null=True)
+    description = fields.HtmlField(blank=True, null=True)
-    languages = ArrayField(
+    languages = fields.ArrayField(
         models.CharField(max_length=255), blank=True, default=list
     )
-    series = models.CharField(max_length=255, blank=True, null=True)
+    series = fields.CharField(max_length=255, blank=True, null=True)
-    series_number = models.CharField(max_length=255, blank=True, null=True)
+    series_number = fields.CharField(max_length=255, blank=True, null=True)
-    subjects = ArrayField(
+    subjects = fields.ArrayField(
-        models.CharField(max_length=255), blank=True, default=list
+        models.CharField(max_length=255), blank=True, null=True, default=list
     )
-    subject_places = ArrayField(
+    subject_places = fields.ArrayField(
-        models.CharField(max_length=255), blank=True, default=list
+        models.CharField(max_length=255), blank=True, null=True, default=list
     )
-    # TODO: include an annotation about the type of authorship (ie, translator)
+    authors = fields.ManyToManyField('Author')
-    authors = models.ManyToManyField('Author')
+    cover = fields.ImageField(
-    # preformatted authorship string for search and easier display
+        upload_to='covers/', blank=True, null=True, alt_field='alt_text')
-    author_text = models.CharField(max_length=255, blank=True, null=True)
+    first_published_date = fields.DateTimeField(blank=True, null=True)
-    cover = models.ImageField(upload_to='covers/', blank=True, null=True)
+    published_date = fields.DateTimeField(blank=True, null=True)
-    first_published_date = models.DateTimeField(blank=True, null=True)
+
-    published_date = models.DateTimeField(blank=True, null=True)
     objects = InheritanceManager()
 
     @property
-    def ap_authors(self):
+    def author_text(self):
-        ''' the activitypub serialization should be a list of author ids '''
+        ''' format a list of authors '''
-        return [a.remote_id for a in self.authors.all()]
+        return ', '.join(a.name for a in self.authors.all())
 
     @property
     def latest_readthrough(self):
         return self.readthrough_set.order_by('-updated_date').first()
 
-    activity_mappings = [
+    @property
-        ActivityMapping('id', 'remote_id'),
+    def edition_info(self):
+        ''' properties of this edition, as a string '''
+        items = [
+            self.physical_format if hasattr(self, 'physical_format') else None,
+            self.languages[0] + ' language' if self.languages and \
+                    self.languages[0] != 'English' else None,
+            str(self.published_date.year) if self.published_date else None,
+        ]
+        return ', '.join(i for i in items if i)
 
-        ActivityMapping('authors', 'ap_authors'),
+    @property
-        ActivityMapping('firstPublishedDate', 'firstpublished_date'),
+    def alt_text(self):
-        ActivityMapping('publishedDate', 'published_date'),
+        ''' image alt test '''
-
+        text = '%s cover' % self.title
-        ActivityMapping('title', 'title'),
+        if self.edition_info:
-        ActivityMapping('sortTitle', 'sort_title'),
+            text += ' (%s)' % self.edition_info
-        ActivityMapping('subtitle', 'subtitle'),
+        return text
-        ActivityMapping('description', 'description'),
-        ActivityMapping('languages', 'languages'),
-        ActivityMapping('series', 'series'),
-        ActivityMapping('seriesNumber', 'series_number'),
-        ActivityMapping('subjects', 'subjects'),
-        ActivityMapping('subjectPlaces', 'subject_places'),
-
-        ActivityMapping('openlibraryKey', 'openlibrary_key'),
-        ActivityMapping('librarythingKey', 'librarything_key'),
-        ActivityMapping('goodreadsKey', 'goodreads_key'),
-
-        ActivityMapping('work', 'parent_work'),
-        ActivityMapping('isbn10', 'isbn_10'),
-        ActivityMapping('isbn13', 'isbn_13'),
-        ActivityMapping('oclcNumber', 'oclc_number'),
-        ActivityMapping('asin', 'asin'),
-        ActivityMapping('pages', 'pages'),
-        ActivityMapping('physicalFormat', 'physical_format'),
-        ActivityMapping('publishers', 'publishers'),
-
-        ActivityMapping('lccn', 'lccn'),
-        ActivityMapping('editions', 'editions_path'),
-        ActivityMapping('cover', 'cover'),
-    ]
 
     def save(self, *args, **kwargs):
         ''' can't be abstract for query reasons, but you shouldn't USE it '''
         if not isinstance(self, Edition) and not isinstance(self, Work):
             raise ValueError('Books should be added as Editions or Works')
-        if self.id and not self.remote_id:
+        return super().save(*args, **kwargs)
-            self.remote_id = self.get_remote_id()
-
-        super().save(*args, **kwargs)
 
     def get_remote_id(self):
         ''' editions and works both use "book" instead of model_name '''
@@ -123,47 +116,56 @@ class Book(ActivitypubMixin, BookWyrmModel):
 class Work(OrderedCollectionPageMixin, Book):
     ''' a work (an abstract concept of a book that manifests in an edition) '''
     # library of congress catalog control number
-    lccn = models.CharField(max_length=255, blank=True, null=True)
+    lccn = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
     # this has to be nullable but should never be null
-    default_edition = models.ForeignKey(
+    default_edition = fields.ForeignKey(
         'Edition',
         on_delete=models.PROTECT,
-        null=True
+        null=True,
+        load_remote=False
     )
 
-    @property
+    def save(self, *args, **kwargs):
-    def editions_path(self):
+        ''' set some fields on the edition object '''
-        ''' it'd be nice to serialize the edition instead but, recursion '''
+        # set rank
-        default = self.default_edition
+        for edition in self.editions.all():
-        ed_list = [
+            edition.save()
-            e.remote_id for e in self.edition_set.filter(~Q(id=default.id)).all()
+        return super().save(*args, **kwargs)
-        ]
-        return [default.remote_id] + ed_list
 
+    def get_default_edition(self):
+        ''' in case the default edition is not set '''
+        return self.default_edition or self.editions.order_by(
+            '-edition_rank'
+        ).first()
 
     def to_edition_list(self, **kwargs):
-        ''' activitypub serialization for this work's editions '''
+        ''' an ordered collection of editions '''
-        remote_id = self.remote_id + '/editions'
         return self.to_ordered_collection(
-            self.edition_set,
+            self.editions.order_by('-edition_rank').all(),
-            remote_id=remote_id,
+            remote_id='%s/editions' % self.remote_id,
             **kwargs
         )
 
-
     activity_serializer = activitypub.Work
+    serialize_reverse_fields = [('editions', 'editions', '-edition_rank')]
+    deserialize_reverse_fields = [('editions', 'editions')]
 
 
 class Edition(Book):
     ''' an edition of a book '''
     # these identifiers only apply to editions, not works
-    isbn_10 = models.CharField(max_length=255, blank=True, null=True)
+    isbn_10 = fields.CharField(
-    isbn_13 = models.CharField(max_length=255, blank=True, null=True)
+        max_length=255, blank=True, null=True, deduplication_field=True)
-    oclc_number = models.CharField(max_length=255, blank=True, null=True)
+    isbn_13 = fields.CharField(
-    asin = models.CharField(max_length=255, blank=True, null=True)
+        max_length=255, blank=True, null=True, deduplication_field=True)
-    pages = models.IntegerField(blank=True, null=True)
+    oclc_number = fields.CharField(
-    physical_format = models.CharField(max_length=255, blank=True, null=True)
+        max_length=255, blank=True, null=True, deduplication_field=True)
-    publishers = ArrayField(
+    asin = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True)
+    pages = fields.IntegerField(blank=True, null=True)
+    physical_format = fields.CharField(max_length=255, blank=True, null=True)
+    publishers = fields.ArrayField(
         models.CharField(max_length=255), blank=True, default=list
     )
     shelves = models.ManyToManyField(
@@ -172,17 +174,42 @@ class Edition(Book):
         through='ShelfBook',
         through_fields=('book', 'shelf')
     )
-    parent_work = models.ForeignKey('Work', on_delete=models.PROTECT, null=True)
+    parent_work = fields.ForeignKey(
+        'Work', on_delete=models.PROTECT, null=True,
+        related_name='editions', activitypub_field='work')
+    edition_rank = fields.IntegerField(default=0)
 
     activity_serializer = activitypub.Edition
+    name_field = 'title'
+
+    @property
+    def get_rank(self):
+        ''' calculate how complete the data is on this edition '''
+        if self.parent_work and self.parent_work.default_edition == self:
+            # default edition has the highest rank
+            return 20
+        rank = 0
+        rank += int(bool(self.cover)) * 3
+        rank += int(bool(self.isbn_13))
+        rank += int(bool(self.isbn_10))
+        rank += int(bool(self.oclc_number))
+        rank += int(bool(self.pages))
+        rank += int(bool(self.physical_format))
+        rank += int(bool(self.description))
+        # max rank is 9
+        return rank
 
     def save(self, *args, **kwargs):
-        ''' calculate isbn 10/13 '''
+        ''' set some fields on the edition object '''
+        # calculate isbn 10/13
         if self.isbn_13 and self.isbn_13[:3] == '978' and not self.isbn_10:
             self.isbn_10 = isbn_13_to_10(self.isbn_13)
         if self.isbn_10 and not self.isbn_13:
             self.isbn_13 = isbn_10_to_13(self.isbn_10)
 
+        # set rank
+        self.edition_rank = self.get_rank
+
         return super().save(*args, **kwargs)
 
 

bookwyrm/models/favorite.py

@@ -0,0 +1,26 @@
+''' like/fav/star a status '''
+from django.db import models
+from django.utils import timezone
+
+from bookwyrm import activitypub
+from .base_model import ActivitypubMixin, BookWyrmModel
+from . import fields
+
+class Favorite(ActivitypubMixin, BookWyrmModel):
+    ''' fav'ing a post '''
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='actor')
+    status = fields.ForeignKey(
+        'Status', on_delete=models.PROTECT, activitypub_field='object')
+
+    activity_serializer = activitypub.Like
+
+    def save(self, *args, **kwargs):
+        ''' update user active time '''
+        self.user.last_active_date = timezone.now()
+        self.user.save()
+        super().save(*args, **kwargs)
+
+    class Meta:
+        ''' can't fav things twice '''
+        unique_together = ('user', 'status')

bookwyrm/models/fields.py

@@ -0,0 +1,427 @@
+''' activitypub-aware django model fields '''
+from dataclasses import MISSING
+import re
+from uuid import uuid4
+
+import dateutil.parser
+from dateutil.parser import ParserError
+from django.contrib.postgres.fields import ArrayField as DjangoArrayField
+from django.core.exceptions import ValidationError
+from django.core.files.base import ContentFile
+from django.db import models
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+from bookwyrm import activitypub
+from bookwyrm.connectors import get_image
+from bookwyrm.sanitize_html import InputHtmlParser
+from bookwyrm.settings import DOMAIN
+
+
+def validate_remote_id(value):
+    ''' make sure the remote_id looks like a url '''
+    if not value or not re.match(r'^http.?:\/\/[^\s]+$', value):
+        raise ValidationError(
+            _('%(value)s is not a valid remote_id'),
+            params={'value': value},
+        )
+
+
+def validate_localname(value):
+    ''' make sure localnames look okay '''
+    if not re.match(r'^[A-Za-z\-_\.0-9]+$', value):
+        raise ValidationError(
+            _('%(value)s is not a valid username'),
+            params={'value': value},
+        )
+
+
+def validate_username(value):
+    ''' make sure usernames look okay '''
+    if not re.match(r'^[A-Za-z\-_\.0-9]+@[A-Za-z\-_\.0-9]+\.[a-z]{2,}$', value):
+        raise ValidationError(
+            _('%(value)s is not a valid username'),
+            params={'value': value},
+        )
+
+
+class ActivitypubFieldMixin:
+    ''' make a database field serializable '''
+    def __init__(self, *args, \
+                 activitypub_field=None, activitypub_wrapper=None,
+                 deduplication_field=False, **kwargs):
+        self.deduplication_field = deduplication_field
+        if activitypub_wrapper:
+            self.activitypub_wrapper = activitypub_field
+            self.activitypub_field = activitypub_wrapper
+        else:
+            self.activitypub_field = activitypub_field
+        super().__init__(*args, **kwargs)
+
+
+    def set_field_from_activity(self, instance, data):
+        ''' helper function for assinging a value to the field '''
+        try:
+            value = getattr(data, self.get_activitypub_field())
+        except AttributeError:
+            # masssively hack-y workaround for boosts
+            if self.get_activitypub_field() != 'attributedTo':
+                raise
+            value = getattr(data, 'actor')
+        formatted = self.field_from_activity(value)
+        if formatted is None or formatted is MISSING:
+            return
+        setattr(instance, self.name, formatted)
+
+
+    def set_activity_from_field(self, activity, instance):
+        ''' update the json object '''
+        value = getattr(instance, self.name)
+        formatted = self.field_to_activity(value)
+        if formatted is None:
+            return
+
+        key = self.get_activitypub_field()
+        # TODO: surely there's a better way
+        if instance.__class__.__name__ == 'Boost' and key == 'attributedTo':
+            key = 'actor'
+        if isinstance(activity.get(key), list):
+            activity[key] += formatted
+        else:
+            activity[key] = formatted
+
+
+    def field_to_activity(self, value):
+        ''' formatter to convert a model value into activitypub '''
+        if hasattr(self, 'activitypub_wrapper'):
+            return {self.activitypub_wrapper: value}
+        return value
+
+    def field_from_activity(self, value):
+        ''' formatter to convert activitypub into a model value '''
+        if hasattr(self, 'activitypub_wrapper'):
+            value = value.get(self.activitypub_wrapper)
+        return value
+
+    def get_activitypub_field(self):
+        ''' model_field_name to activitypubFieldName '''
+        if self.activitypub_field:
+            return self.activitypub_field
+        name = self.name.split('.')[-1]
+        components = name.split('_')
+        return components[0] + ''.join(x.title() for x in components[1:])
+
+
+class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
+    ''' default (de)serialization for foreign key and one to one '''
+    def __init__(self, *args, load_remote=True, **kwargs):
+        self.load_remote = load_remote
+        super().__init__(*args, **kwargs)
+
+    def field_from_activity(self, value):
+        if not value:
+            return None
+
+        related_model = self.related_model
+        if isinstance(value, dict) and value.get('id'):
+            if not self.load_remote:
+                # only look in the local database
+                return related_model.find_existing(value)
+            # this is an activitypub object, which we can deserialize
+            activity_serializer = related_model.activity_serializer
+            return activity_serializer(**value).to_model(related_model)
+        try:
+            # make sure the value looks like a remote id
+            validate_remote_id(value)
+        except ValidationError:
+            # we don't know what this is, ignore it
+            return None
+        # gets or creates the model field from the remote id
+        if not self.load_remote:
+            # only look in the local database
+            return related_model.find_existing_by_remote_id(value)
+        return activitypub.resolve_remote_id(related_model, value)
+
+
+class RemoteIdField(ActivitypubFieldMixin, models.CharField):
+    ''' a url that serves as a unique identifier '''
+    def __init__(self, *args, max_length=255, validators=None, **kwargs):
+        validators = validators or [validate_remote_id]
+        super().__init__(
+            *args, max_length=max_length, validators=validators,
+            **kwargs
+        )
+        # for this field, the default is true. false everywhere else.
+        self.deduplication_field = kwargs.get('deduplication_field', True)
+
+
+class UsernameField(ActivitypubFieldMixin, models.CharField):
+    ''' activitypub-aware username field '''
+    def __init__(self, activitypub_field='preferredUsername', **kwargs):
+        self.activitypub_field = activitypub_field
+        # I don't totally know why pylint is mad at this, but it makes it work
+        super( #pylint: disable=bad-super-call
+            ActivitypubFieldMixin, self
+        ).__init__(
+            _('username'),
+            max_length=150,
+            unique=True,
+            validators=[validate_username],
+            error_messages={
+                'unique': _('A user with that username already exists.'),
+            },
+        )
+
+    def deconstruct(self):
+        ''' implementation of models.Field deconstruct '''
+        name, path, args, kwargs = super().deconstruct()
+        del kwargs['verbose_name']
+        del kwargs['max_length']
+        del kwargs['unique']
+        del kwargs['validators']
+        del kwargs['error_messages']
+        return name, path, args, kwargs
+
+    def field_to_activity(self, value):
+        return value.split('@')[0]
+
+
+PrivacyLevels = models.TextChoices('Privacy', [
+    'public',
+    'unlisted',
+    'followers',
+    'direct'
+])
+
+class PrivacyField(ActivitypubFieldMixin, models.CharField):
+    ''' this maps to two differente activitypub fields '''
+    public = 'https://www.w3.org/ns/activitystreams#Public'
+    def __init__(self, *args, **kwargs):
+        super().__init__(
+            *args, max_length=255,
+            choices=PrivacyLevels.choices, default='public')
+
+    def set_field_from_activity(self, instance, data):
+        to = data.to
+        cc = data.cc
+        if to == [self.public]:
+            setattr(instance, self.name, 'public')
+        elif cc == []:
+            setattr(instance, self.name, 'direct')
+        elif self.public in cc:
+            setattr(instance, self.name, 'unlisted')
+        else:
+            setattr(instance, self.name, 'followers')
+
+    def set_activity_from_field(self, activity, instance):
+        mentions = [u.remote_id for u in instance.mention_users.all()]
+        # this is a link to the followers list
+        followers = instance.user.__class__._meta.get_field('followers')\
+                .field_to_activity(instance.user.followers)
+        if instance.privacy == 'public':
+            activity['to'] = [self.public]
+            activity['cc'] = [followers] + mentions
+        elif instance.privacy == 'unlisted':
+            activity['to'] = [followers]
+            activity['cc'] = [self.public] + mentions
+        elif instance.privacy == 'followers':
+            activity['to'] = [followers]
+            activity['cc'] = mentions
+        if instance.privacy == 'direct':
+            activity['to'] = mentions
+            activity['cc'] = []
+
+
+class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey):
+    ''' activitypub-aware foreign key field '''
+    def field_to_activity(self, value):
+        if not value:
+            return None
+        return value.remote_id
+
+
+class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField):
+    ''' activitypub-aware foreign key field '''
+    def field_to_activity(self, value):
+        if not value:
+            return None
+        return value.to_activity()
+
+
+class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
+    ''' activitypub-aware many to many field '''
+    def __init__(self, *args, link_only=False, **kwargs):
+        self.link_only = link_only
+        super().__init__(*args, **kwargs)
+
+    def set_field_from_activity(self, instance, data):
+        ''' helper function for assinging a value to the field '''
+        value = getattr(data, self.get_activitypub_field())
+        formatted = self.field_from_activity(value)
+        if formatted is None or formatted is MISSING:
+            return
+        getattr(instance, self.name).set(formatted)
+
+    def field_to_activity(self, value):
+        if self.link_only:
+            return '%s/%s' % (value.instance.remote_id, self.name)
+        return [i.remote_id for i in value.all()]
+
+    def field_from_activity(self, value):
+        items = []
+        if value is None or value is MISSING:
+            return []
+        for remote_id in value:
+            try:
+                validate_remote_id(remote_id)
+            except ValidationError:
+                continue
+            items.append(
+                activitypub.resolve_remote_id(self.related_model, remote_id)
+            )
+        return items
+
+
+class TagField(ManyToManyField):
+    ''' special case of many to many that uses Tags '''
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.activitypub_field = 'tag'
+
+    def field_to_activity(self, value):
+        tags = []
+        for item in value.all():
+            activity_type = item.__class__.__name__
+            if activity_type == 'User':
+                activity_type = 'Mention'
+            tags.append(activitypub.Link(
+                href=item.remote_id,
+                name=getattr(item, item.name_field),
+                type=activity_type
+            ))
+        return tags
+
+    def field_from_activity(self, value):
+        if not isinstance(value, list):
+            return None
+        items = []
+        for link_json in value:
+            link = activitypub.Link(**link_json)
+            tag_type = link.type if link.type != 'Mention' else 'Person'
+            if tag_type == 'Book':
+                tag_type = 'Edition'
+            if tag_type != self.related_model.activity_serializer.type:
+                # tags can contain multiple types
+                continue
+            items.append(
+                activitypub.resolve_remote_id(self.related_model, link.href)
+            )
+        return items
+
+
+def image_serializer(value, alt):
+    ''' helper for serializing images '''
+    if value and hasattr(value, 'url'):
+        url = value.url
+    else:
+        return None
+    url = 'https://%s%s' % (DOMAIN, url)
+    return activitypub.Image(url=url, name=alt)
+
+
+class ImageField(ActivitypubFieldMixin, models.ImageField):
+    ''' activitypub-aware image field '''
+    def __init__(self, *args, alt_field=None, **kwargs):
+        self.alt_field = alt_field
+        super().__init__(*args, **kwargs)
+
+    # pylint: disable=arguments-differ
+    def set_field_from_activity(self, instance, data, save=True):
+        ''' helper function for assinging a value to the field '''
+        value = getattr(data, self.get_activitypub_field())
+        formatted = self.field_from_activity(value)
+        if formatted is None or formatted is MISSING:
+            return
+        getattr(instance, self.name).save(*formatted, save=save)
+
+    def set_activity_from_field(self, activity, instance):
+        value = getattr(instance, self.name)
+        if value is None:
+            return
+        alt_text = getattr(instance, self.alt_field)
+        formatted = self.field_to_activity(value, alt_text)
+
+        key = self.get_activitypub_field()
+        activity[key] = formatted
+
+
+    def field_to_activity(self, value, alt=None):
+        return image_serializer(value, alt)
+
+
+    def field_from_activity(self, value):
+        image_slug = value
+        # when it's an inline image (User avatar/icon, Book cover), it's a json
+        # blob, but when it's an attached image, it's just a url
+        if isinstance(image_slug, dict):
+            url = image_slug.get('url')
+        elif isinstance(image_slug, str):
+            url = image_slug
+        else:
+            return None
+
+        try:
+            validate_remote_id(url)
+        except ValidationError:
+            return None
+
+        response = get_image(url)
+        if not response:
+            return None
+
+        image_name = str(uuid4()) + '.' + url.split('.')[-1]
+        image_content = ContentFile(response.content)
+        return [image_name, image_content]
+
+
+class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
+    ''' activitypub-aware datetime field '''
+    def field_to_activity(self, value):
+        if not value:
+            return None
+        return value.isoformat()
+
+    def field_from_activity(self, value):
+        try:
+            date_value = dateutil.parser.parse(value)
+            try:
+                return timezone.make_aware(date_value)
+            except ValueError:
+                return date_value
+        except (ParserError, TypeError):
+            return None
+
+class HtmlField(ActivitypubFieldMixin, models.TextField):
+    ''' a text field for storing html '''
+    def field_from_activity(self, value):
+        if not value or value == MISSING:
+            return None
+        sanitizer = InputHtmlParser()
+        sanitizer.feed(value)
+        return sanitizer.get_output()
+
+class ArrayField(ActivitypubFieldMixin, DjangoArrayField):
+    ''' activitypub-aware array field '''
+    def field_to_activity(self, value):
+        return [str(i) for i in value]
+
+class CharField(ActivitypubFieldMixin, models.CharField):
+    ''' activitypub-aware char field '''
+
+class TextField(ActivitypubFieldMixin, models.TextField):
+    ''' activitypub-aware text field '''
+
+class BooleanField(ActivitypubFieldMixin, models.BooleanField):
+    ''' activitypub-aware boolean field '''
+
+class IntegerField(ActivitypubFieldMixin, models.IntegerField):
+    ''' activitypub-aware boolean field '''

bookwyrm/models/import_job.py

@@ -2,14 +2,13 @@
 import re
 import dateutil.parser
 
+from django.contrib.postgres.fields import JSONField
 from django.db import models
 from django.utils import timezone
 
-from bookwyrm import books_manager
+from bookwyrm.connectors import connector_manager
-from bookwyrm.connectors import ConnectorException
 from bookwyrm.models import ReadThrough, User, Book
-from bookwyrm.utils.fields import JSONField
+from .fields import PrivacyLevels
-from .base_model import PrivacyLevels
 
 
 # Mapping goodreads -> bookwyrm shelf titles.
@@ -43,6 +42,7 @@ class ImportJob(models.Model):
     created_date = models.DateTimeField(default=timezone.now)
     task_id = models.CharField(max_length=100, null=True)
     include_reviews = models.BooleanField(default=True)
+    complete = models.BooleanField(default=False)
     privacy = models.CharField(
         max_length=255,
         default='public',
@@ -72,12 +72,12 @@ class ImportItem(models.Model):
 
     def get_book_from_isbn(self):
         ''' search by isbn '''
-        search_result = books_manager.first_search_result(
+        search_result = connector_manager.first_search_result(
             self.isbn, min_confidence=0.999
         )
         if search_result:
             # raises ConnectorException
-            return books_manager.get_or_create_book(search_result.key)
+            return search_result.connector.get_or_create_book(search_result.key)
         return None
 
 
@@ -87,12 +87,12 @@ class ImportItem(models.Model):
             self.data['Title'],
             self.data['Author']
         )
-        search_result = books_manager.first_search_result(
+        search_result = connector_manager.first_search_result(
             search_term, min_confidence=0.999
         )
         if search_result:
             # raises ConnectorException
-            return books_manager.get_or_create_book(search_result.key)
+            return search_result.connector.get_or_create_book(search_result.key)
         return None
 
 

bookwyrm/models/notification.py

@@ -0,0 +1,33 @@
+''' alert a user to activity '''
+from django.db import models
+from .base_model import BookWyrmModel
+
+
+NotificationType = models.TextChoices(
+    'NotificationType',
+    'FAVORITE REPLY MENTION TAG FOLLOW FOLLOW_REQUEST BOOST IMPORT')
+
+class Notification(BookWyrmModel):
+    ''' you've been tagged, liked, followed, etc '''
+    user = models.ForeignKey('User', on_delete=models.PROTECT)
+    related_book = models.ForeignKey(
+        'Edition', on_delete=models.PROTECT, null=True)
+    related_user = models.ForeignKey(
+        'User',
+        on_delete=models.PROTECT, null=True, related_name='related_user')
+    related_status = models.ForeignKey(
+        'Status', on_delete=models.PROTECT, null=True)
+    related_import = models.ForeignKey(
+        'ImportJob', on_delete=models.PROTECT, null=True)
+    read = models.BooleanField(default=False)
+    notification_type = models.CharField(
+        max_length=255, choices=NotificationType.choices)
+
+    class Meta:
+        ''' checks if notifcation is in enum list for valid types '''
+        constraints = [
+            models.CheckConstraint(
+                check=models.Q(notification_type__in=NotificationType.values),
+                name="notification_type_valid",
+            )
+        ]

bookwyrm/models/readthrough.py

@@ -0,0 +1,56 @@
+''' progress in a book '''
+from django.db import models
+from django.utils import timezone
+
+from .base_model import BookWyrmModel
+
+class ProgressMode(models.TextChoices):
+    PAGE = 'PG', 'page'
+    PERCENT = 'PCT', 'percent'
+
+class ReadThrough(BookWyrmModel):
+    ''' Store a read through a book in the database. '''
+    user = models.ForeignKey('User', on_delete=models.PROTECT)
+    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
+    progress = models.IntegerField(
+        null=True,
+        blank=True)
+    progress_mode = models.CharField(
+        max_length=3,
+        choices=ProgressMode.choices,
+        default=ProgressMode.PAGE)
+    start_date = models.DateTimeField(
+        blank=True,
+        null=True)
+    finish_date = models.DateTimeField(
+        blank=True,
+        null=True)
+
+    def save(self, *args, **kwargs):
+        ''' update user active time '''
+        self.user.last_active_date = timezone.now()
+        self.user.save()
+        super().save(*args, **kwargs)
+
+    def create_update(self):
+        if self.progress:
+            return self.progressupdate_set.create(
+                user=self.user,
+                progress=self.progress,
+                mode=self.progress_mode)
+
+class ProgressUpdate(BookWyrmModel):
+    ''' Store progress through a book in the database. '''
+    user = models.ForeignKey('User', on_delete=models.PROTECT)
+    readthrough = models.ForeignKey('ReadThrough', on_delete=models.PROTECT)
+    progress = models.IntegerField()
+    mode = models.CharField(
+        max_length=3,
+        choices=ProgressMode.choices,
+        default=ProgressMode.PAGE)
+
+    def save(self, *args, **kwargs):
+        ''' update user active time '''
+        self.user.last_active_date = timezone.now()
+        self.user.save()
+        super().save(*args, **kwargs)

bookwyrm/models/relationship.py

@@ -2,20 +2,23 @@
 from django.db import models
 
 from bookwyrm import activitypub
-from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel
+from .base_model import ActivitypubMixin, BookWyrmModel
+from . import fields
 
 
 class UserRelationship(ActivitypubMixin, BookWyrmModel):
     ''' many-to-many through table for followers '''
-    user_subject = models.ForeignKey(
+    user_subject = fields.ForeignKey(
         'User',
         on_delete=models.PROTECT,
-        related_name='%(class)s_user_subject'
+        related_name='%(class)s_user_subject',
+        activitypub_field='actor',
     )
-    user_object = models.ForeignKey(
+    user_object = fields.ForeignKey(
         'User',
         on_delete=models.PROTECT,
-        related_name='%(class)s_user_object'
+        related_name='%(class)s_user_object',
+        activitypub_field='object',
     )
 
     class Meta:
@@ -32,14 +35,9 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel):
             )
         ]
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('actor', 'user_subject'),
-        ActivityMapping('object', 'user_object'),
-    ]
     activity_serializer = activitypub.Follow
 
-    def get_remote_id(self, status=None):
+    def get_remote_id(self, status=None):# pylint: disable=arguments-differ
         ''' use shelf identifier in remote_id '''
         status = status or 'follows'
         base_path = self.user_subject.remote_id
@@ -56,7 +54,7 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel):
 
 
     def to_reject_activity(self):
-        ''' generate an Accept for this follow request '''
+        ''' generate a Reject for this follow request '''
         return activitypub.Reject(
             id=self.get_remote_id(status='rejects'),
             actor=self.user_object.remote_id,

bookwyrm/models/shelf.py

@@ -3,19 +3,22 @@ import re
 from django.db import models
 
 from bookwyrm import activitypub
-from .base_model import BookWyrmModel, OrderedCollectionMixin, PrivacyLevels
+from .base_model import ActivitypubMixin, BookWyrmModel
+from .base_model import OrderedCollectionMixin
+from . import fields
 
 
 class Shelf(OrderedCollectionMixin, BookWyrmModel):
     ''' a list of books owned by a user '''
-    name = models.CharField(max_length=100)
+    name = fields.CharField(max_length=100)
     identifier = models.CharField(max_length=100)
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='owner')
     editable = models.BooleanField(default=True)
-    privacy = models.CharField(
+    privacy = fields.CharField(
         max_length=255,
         default='public',
-        choices=PrivacyLevels.choices
+        choices=fields.PrivacyLevels.choices
     )
     books = models.ManyToManyField(
         'Edition',
@@ -36,7 +39,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
     @property
     def collection_queryset(self):
         ''' list of books for this shelf, overrides OrderedCollectionMixin  '''
-        return self.books
+        return self.books.all()
 
     def get_remote_id(self):
         ''' shelf identifier instead of id '''
@@ -48,17 +51,22 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
         unique_together = ('user', 'identifier')
 
 
-class ShelfBook(BookWyrmModel):
+class ShelfBook(ActivitypubMixin, BookWyrmModel):
     ''' many to many join table for books and shelves '''
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
+    book = fields.ForeignKey(
-    shelf = models.ForeignKey('Shelf', on_delete=models.PROTECT)
+        'Edition', on_delete=models.PROTECT, activitypub_field='object')
-    added_by = models.ForeignKey(
+    shelf = fields.ForeignKey(
+        'Shelf', on_delete=models.PROTECT, activitypub_field='target')
+    added_by = fields.ForeignKey(
         'User',
         blank=True,
         null=True,
-        on_delete=models.PROTECT
+        on_delete=models.PROTECT,
+        activitypub_field='actor'
     )
 
+    activity_serializer = activitypub.AddBook
+
     def to_add_activity(self, user):
         ''' AP for shelving a book'''
         return activitypub.Add(

bookwyrm/models/site.py

@@ -12,11 +12,27 @@ from .user import User
 class SiteSettings(models.Model):
     ''' customized settings for this instance '''
     name = models.CharField(default='BookWyrm', max_length=100)
+    instance_tagline = models.CharField(
+        max_length=150, default='Social Reading and Reviewing')
     instance_description = models.TextField(
-        default="This instance has no description.")
+        default='This instance has no description.')
+    registration_closed_text = models.TextField(
+        default='Contact an administrator to get an invite')
     code_of_conduct = models.TextField(
-        default="Add a code of conduct here.")
+        default='Add a code of conduct here.')
     allow_registration = models.BooleanField(default=True)
+    logo = models.ImageField(
+        upload_to='logos/', null=True, blank=True
+    )
+    logo_small = models.ImageField(
+        upload_to='logos/', null=True, blank=True
+    )
+    favicon = models.ImageField(
+        upload_to='logos/', null=True, blank=True
+    )
+    support_link = models.CharField(max_length=255, null=True, blank=True)
+    support_title = models.CharField(max_length=100, null=True, blank=True)
+    admin_email = models.EmailField(max_length=255, null=True, blank=True)
 
     @classmethod
     def get(cls):
@@ -34,6 +50,7 @@ def new_access_code():
 
 class SiteInvite(models.Model):
     ''' gives someone access to create an account on the instance '''
+    created_date = models.DateTimeField(auto_now_add=True)
     code = models.CharField(max_length=32, default=new_access_code)
     expiry = models.DateTimeField(blank=True, null=True)
     use_limit = models.IntegerField(blank=True, null=True)
@@ -49,7 +66,7 @@ class SiteInvite(models.Model):
     @property
     def link(self):
         ''' formats the invite link '''
-        return "https://{}/invite/{}".format(DOMAIN, self.code)
+        return 'https://{}/invite/{}'.format(DOMAIN, self.code)
 
 
 def get_passowrd_reset_expiry():
@@ -71,4 +88,4 @@ class PasswordReset(models.Model):
     @property
     def link(self):
         ''' formats the invite link '''
-        return "https://{}/password-reset/{}".format(DOMAIN, self.code)
+        return 'https://{}/password-reset/{}'.format(DOMAIN, self.code)

bookwyrm/models/status.py

@@ -1,31 +1,34 @@
 ''' models for storing different kinds of Activities '''
-from django.utils import timezone
+from dataclasses import MISSING
+import re
+
+from django.apps import apps
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
+from django.utils import timezone
 from model_utils.managers import InheritanceManager
 
 from bookwyrm import activitypub
 from .base_model import ActivitypubMixin, OrderedCollectionPageMixin
-from .base_model import ActivityMapping, BookWyrmModel, PrivacyLevels
+from .base_model import BookWyrmModel
-from .base_model import tag_formatter, image_attachments_formatter
+from . import fields
-
+from .fields import image_serializer
 
 class Status(OrderedCollectionPageMixin, BookWyrmModel):
     ''' any post, like a reply to a review, etc '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
+    user = fields.ForeignKey(
-    content = models.TextField(blank=True, null=True)
+        'User', on_delete=models.PROTECT, activitypub_field='attributedTo')
-    mention_users = models.ManyToManyField('User', related_name='mention_user')
+    content = fields.HtmlField(blank=True, null=True)
-    mention_books = models.ManyToManyField(
+    mention_users = fields.TagField('User', related_name='mention_user')
-        'Edition', related_name='mention_book')
+    mention_books = fields.TagField('Edition', related_name='mention_book')
     local = models.BooleanField(default=True)
-    privacy = models.CharField(
+    content_warning = fields.CharField(
-        max_length=255,
+        max_length=500, blank=True, null=True, activitypub_field='summary')
-        default='public',
+    privacy = fields.PrivacyField(max_length=255)
-        choices=PrivacyLevels.choices
+    sensitive = fields.BooleanField(default=False)
-    )
+    # created date is different than publish date because of federated posts
-    sensitive = models.BooleanField(default=False)
+    published_date = fields.DateTimeField(
-    # the created date can't be this, because of receiving federated posts
+        default=timezone.now, activitypub_field='published')
-    published_date = models.DateTimeField(default=timezone.now)
     deleted = models.BooleanField(default=False)
     deleted_date = models.DateTimeField(blank=True, null=True)
     favorites = models.ManyToManyField(
@@ -35,94 +38,57 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         through_fields=('status', 'user'),
         related_name='user_favorites'
     )
-    reply_parent = models.ForeignKey(
+    reply_parent = fields.ForeignKey(
         'self',
         null=True,
-        on_delete=models.PROTECT
+        on_delete=models.PROTECT,
+        activitypub_field='inReplyTo',
     )
     objects = InheritanceManager()
 
-    # ---- activitypub serialization settings for this model ----- #
-    @property
-    def ap_to(self):
-        ''' should be related to post privacy I think '''
-        return ['https://www.w3.org/ns/activitystreams#Public']
-
-    @property
-    def ap_cc(self):
-        ''' should be related to post privacy I think '''
-        return [self.user.ap_followers]
-
-    @property
-    def ap_replies(self):
-        ''' structured replies block '''
-        return self.to_replies()
-
-    @property
-    def ap_status_image(self):
-        ''' attach a book cover, if relevent '''
-        if hasattr(self, 'book'):
-            return self.book.ap_cover
-        if self.mention_books.first():
-            return self.mention_books.first().ap_cover
-        return None
-
-
-    shared_mappings = [
-        ActivityMapping('url', 'remote_id', lambda x: None),
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('inReplyTo', 'reply_parent'),
-        ActivityMapping('published', 'published_date'),
-        ActivityMapping('attributedTo', 'user'),
-        ActivityMapping('to', 'ap_to'),
-        ActivityMapping('cc', 'ap_cc'),
-        ActivityMapping('replies', 'ap_replies'),
-        ActivityMapping(
-            'tag', 'mention_books',
-            lambda x: tag_formatter(x, 'title', 'Book'),
-            lambda x: activitypub.tag_formatter(x, 'Book')
-        ),
-        ActivityMapping(
-            'tag', 'mention_users',
-            lambda x: tag_formatter(x, 'username', 'Mention'),
-            lambda x: activitypub.tag_formatter(x, 'Mention')
-        ),
-        ActivityMapping(
-            'attachment', 'attachments',
-            lambda x: image_attachments_formatter(x.all()),
-        )
-    ]
-
-    # serializing to bookwyrm expanded activitypub
-    activity_mappings = shared_mappings + [
-        ActivityMapping('name', 'name'),
-        ActivityMapping('inReplyToBook', 'book'),
-        ActivityMapping('rating', 'rating'),
-        ActivityMapping('quote', 'quote'),
-        ActivityMapping('content', 'content'),
-    ]
-
-    # for serializing to standard activitypub without extended types
-    pure_activity_mappings = shared_mappings + [
-        ActivityMapping('name', 'ap_pure_name'),
-        ActivityMapping('content', 'ap_pure_content'),
-        ActivityMapping('attachment', 'ap_status_image'),
-    ]
-
     activity_serializer = activitypub.Note
+    serialize_reverse_fields = [('attachments', 'attachment', 'id')]
+    deserialize_reverse_fields = [('attachments', 'attachment')]
+
+    @classmethod
+    def ignore_activity(cls, activity):
+        ''' keep notes if they are replies to existing statuses '''
+        if activity.type != 'Note':
+            return False
+        if cls.objects.filter(
+                remote_id=activity.inReplyTo).exists():
+            return False
+
+        # keep notes if they mention local users
+        if activity.tag == MISSING or activity.tag is None:
+            return True
+        tags = [l['href'] for l in activity.tag if l['type'] == 'Mention']
+        for tag in tags:
+            user_model = apps.get_model('bookwyrm.User', require_ready=True)
+            if user_model.objects.filter(
+                    remote_id=tag, local=True).exists():
+                # we found a mention of a known use boost
+                return False
+        return True
 
-    #----- replies collection activitypub ----#
     @classmethod
     def replies(cls, status):
         ''' load all replies to a status. idk if there's a better way
             to write this so it's just a property '''
-        return cls.objects.filter(reply_parent=status).select_subclasses()
+        return cls.objects.filter(
+            reply_parent=status
+        ).select_subclasses().order_by('published_date')
 
     @property
     def status_type(self):
         ''' expose the type of status for the ui using activity type '''
         return self.activity_serializer.__name__
 
+    @property
+    def boostable(self):
+        ''' you can't boost dms '''
+        return self.privacy in ['unlisted', 'public']
+
     def to_replies(self, **kwargs):
         ''' helper function for loading AP serialized replies to a status '''
         return self.to_ordered_collection(
@@ -131,7 +97,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
             **kwargs
         )
 
-    def to_activity(self, pure=False):
+    def to_activity(self, pure=False):# pylint: disable=arguments-differ
         ''' return tombstone if the status is deleted '''
         if self.deleted:
             return activitypub.Tombstone(
@@ -140,7 +106,24 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
                 deleted=self.deleted_date.isoformat(),
                 published=self.deleted_date.isoformat()
             ).serialize()
-        return ActivitypubMixin.to_activity(self, pure=pure)
+        activity = ActivitypubMixin.to_activity(self)
+        activity['replies'] = self.to_replies()
+
+        # "pure" serialization for non-bookwyrm instances
+        if pure and hasattr(self, 'pure_content'):
+            activity['content'] = self.pure_content
+            if 'name' in activity:
+                activity['name'] = self.pure_name
+            activity['type'] = self.pure_type
+            activity['attachment'] = [
+                image_serializer(b.cover, b.alt_text) \
+                    for b in self.mention_books.all()[:4] if b.cover]
+            if hasattr(self, 'book') and self.book.cover:
+                activity['attachment'].append(
+                    image_serializer(self.book.cover, self.book.alt_text)
+                )
+        return activity
+
 
     def save(self, *args, **kwargs):
         ''' update user active time '''
@@ -153,57 +136,62 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
 class GeneratedNote(Status):
     ''' these are app-generated messages about user activity '''
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
         message = self.content
         books = ', '.join(
-            '<a href="%s">"%s"</a>' % (self.book.remote_id, self.book.title) \
+            '<a href="%s">"%s"</a>' % (book.remote_id, book.title) \
             for book in self.mention_books.all()
         )
-        return '%s %s' % (message, books)
+        return '%s %s %s' % (self.user.display_name, message, books)
 
     activity_serializer = activitypub.GeneratedNote
-    pure_activity_serializer = activitypub.Note
+    pure_type = 'Note'
 
 
 class Comment(Status):
     ''' like a review but without a rating and transient '''
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook')
 
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
-        return self.content + '<br><br>(comment on <a href="%s">"%s"</a>)' % \
+        return '%s<p>(comment on <a href="%s">"%s"</a>)</p>' % \
-                (self.book.remote_id, self.book.title)
+                (self.content, self.book.remote_id, self.book.title)
 
     activity_serializer = activitypub.Comment
-    pure_activity_serializer = activitypub.Note
+    pure_type = 'Note'
 
 
 class Quotation(Status):
     ''' like a review but without a rating and transient '''
-    quote = models.TextField()
+    quote = fields.HtmlField()
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook')
 
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
-        return '"%s"<br>-- <a href="%s">"%s"</a><br><br>%s' % (
+        quote = re.sub(r'^<p>', '<p>"', self.quote)
-            self.quote,
+        quote = re.sub(r'</p>$', '"</p>', quote)
+        return '%s <p>-- <a href="%s">"%s"</a></p>%s' % (
+            quote,
             self.book.remote_id,
             self.book.title,
             self.content,
         )
 
     activity_serializer = activitypub.Quotation
-    pure_activity_serializer = activitypub.Note
+    pure_type = 'Note'
 
 
 class Review(Status):
     ''' a book review '''
-    name = models.CharField(max_length=255, null=True)
+    name = fields.CharField(max_length=255, null=True)
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
+    book = fields.ForeignKey(
-    rating = models.IntegerField(
+        'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook')
+    rating = fields.IntegerField(
         default=None,
         null=True,
         blank=True,
@@ -211,9 +199,10 @@ class Review(Status):
     )
 
     @property
-    def ap_pure_name(self):
+    def pure_name(self):
         ''' clarify review names for mastodon serialization '''
         if self.rating:
+            #pylint: disable=bad-string-format-type
             return 'Review of "%s" (%d stars): %s' % (
                 self.book.title,
                 self.rating,
@@ -225,139 +214,37 @@ class Review(Status):
         )
 
     @property
-    def ap_pure_content(self):
+    def pure_content(self):
         ''' indicate the book in question for mastodon (or w/e) users '''
-        return self.content + '<br><br>(<a href="%s">"%s"</a>)' % \
+        return self.content
-                (self.book.remote_id, self.book.title)
 
     activity_serializer = activitypub.Review
-    pure_activity_serializer = activitypub.Article
+    pure_type = 'Article'
-
-
-class Favorite(ActivitypubMixin, BookWyrmModel):
-    ''' fav'ing a post '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    status = models.ForeignKey('Status', on_delete=models.PROTECT)
-
-    # ---- activitypub serialization settings for this model ----- #
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping('actor', 'user'),
-        ActivityMapping('object', 'status'),
-    ]
-
-    activity_serializer = activitypub.Like
-
-    def save(self, *args, **kwargs):
-        ''' update user active time '''
-        self.user.last_active_date = timezone.now()
-        self.user.save()
-        super().save(*args, **kwargs)
-
-
-    class Meta:
-        ''' can't fav things twice '''
-        unique_together = ('user', 'status')
 
 
 class Boost(Status):
     ''' boost'ing a post '''
-    boosted_status = models.ForeignKey(
+    boosted_status = fields.ForeignKey(
         'Status',
         on_delete=models.PROTECT,
-        related_name="boosters")
+        related_name='boosters',
+        activitypub_field='object',
+    )
 
-    activity_mappings = [
+    def __init__(self, *args, **kwargs):
-        ActivityMapping('id', 'remote_id'),
+        ''' the user field is "actor" here instead of "attributedTo" '''
-        ActivityMapping('actor', 'user'),
+        super().__init__(*args, **kwargs)
-        ActivityMapping('object', 'boosted_status'),
+
-    ]
+        reserve_fields = ['user', 'boosted_status']
+        self.simple_fields = [f for f in self.simple_fields if \
+                f.name in reserve_fields]
+        self.activity_fields = self.simple_fields
+        self.many_to_many_fields = []
+        self.image_fields = []
+        self.deserialize_reverse_fields = []
 
     activity_serializer = activitypub.Boost
 
     # This constraint can't work as it would cross tables.
     # class Meta:
     #     unique_together = ('user', 'boosted_status')
-
-
-class ProgressMode(models.TextChoices):
-    PAGE = 'PG', 'page'
-    PERCENT = 'PCT', 'percent'
-
-class ReadThrough(BookWyrmModel):
-    ''' Store a read through a book in the database. '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    book = models.ForeignKey('Book', on_delete=models.PROTECT)
-    progress = models.IntegerField(
-        null=True,
-        blank=True)
-    progress_mode = models.CharField(
-        max_length=3,
-        choices=ProgressMode.choices,
-        default=ProgressMode.PAGE)
-    start_date = models.DateTimeField(
-        blank=True,
-        null=True)
-    finish_date = models.DateTimeField(
-        blank=True,
-        null=True)
-
-    def save(self, *args, **kwargs):
-        ''' update user active time '''
-        self.user.last_active_date = timezone.now()
-        self.user.save()
-        super().save(*args, **kwargs)
-
-    def create_update(self):
-        if self.progress:
-            return self.progressupdate_set.create(
-                user=self.user,
-                progress=self.progress,
-                mode=self.progress_mode)
-
-
-class ProgressUpdate(BookWyrmModel):
-    ''' Store progress through a book in the database. '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    readthrough = models.ForeignKey('ReadThrough', on_delete=models.PROTECT)
-    progress = models.IntegerField()
-    mode = models.CharField(
-        max_length=3,
-        choices=ProgressMode.choices,
-        default=ProgressMode.PAGE)
-
-    def save(self, *args, **kwargs):
-        ''' update user active time '''
-        self.user.last_active_date = timezone.now()
-        self.user.save()
-        super().save(*args, **kwargs)
-
-
-NotificationType = models.TextChoices(
-    'NotificationType',
-    'FAVORITE REPLY MENTION TAG FOLLOW FOLLOW_REQUEST BOOST IMPORT')
-
-class Notification(BookWyrmModel):
-    ''' you've been tagged, liked, followed, etc '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    related_book = models.ForeignKey(
-        'Edition', on_delete=models.PROTECT, null=True)
-    related_user = models.ForeignKey(
-        'User',
-        on_delete=models.PROTECT, null=True, related_name='related_user')
-    related_status = models.ForeignKey(
-        'Status', on_delete=models.PROTECT, null=True)
-    related_import = models.ForeignKey(
-        'ImportJob', on_delete=models.PROTECT, null=True)
-    read = models.BooleanField(default=False)
-    notification_type = models.CharField(
-        max_length=255, choices=NotificationType.choices)
-
-    class Meta:
-        ''' checks if notifcation is in enum list for valid types '''
-        constraints = [
-            models.CheckConstraint(
-                check=models.Q(notification_type__in=NotificationType.values),
-                name="notification_type_valid",
-            )
-        ]

bookwyrm/models/tag.py

@@ -6,19 +6,20 @@ from django.db import models
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
 from .base_model import OrderedCollectionMixin, BookWyrmModel
+from . import fields
 
 
 class Tag(OrderedCollectionMixin, BookWyrmModel):
     ''' freeform tags for books '''
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
+    name = fields.CharField(max_length=100, unique=True)
-    book = models.ForeignKey('Edition', on_delete=models.PROTECT)
-    name = models.CharField(max_length=100)
     identifier = models.CharField(max_length=100)
 
     @classmethod
     def book_queryset(cls, identifier):
         ''' county of books associated with this tag '''
-        return cls.objects.filter(identifier=identifier)
+        return cls.objects.filter(
+            identifier=identifier
+        ).order_by('-updated_date')
 
     @property
     def collection_queryset(self):
@@ -30,6 +31,26 @@ class Tag(OrderedCollectionMixin, BookWyrmModel):
         base_path = 'https://%s' % DOMAIN
         return '%s/tag/%s' % (base_path, self.identifier)
 
+
+    def save(self, *args, **kwargs):
+        ''' create a url-safe lookup key for the tag '''
+        if not self.id:
+            # add identifiers to new tags
+            self.identifier = urllib.parse.quote_plus(self.name)
+        super().save(*args, **kwargs)
+
+
+class UserTag(BookWyrmModel):
+    ''' an instance of a tag on a book by a user '''
+    user = fields.ForeignKey(
+        'User', on_delete=models.PROTECT, activitypub_field='actor')
+    book = fields.ForeignKey(
+        'Edition', on_delete=models.PROTECT, activitypub_field='object')
+    tag = fields.ForeignKey(
+        'Tag', on_delete=models.PROTECT, activitypub_field='target')
+
+    activity_serializer = activitypub.AddBook
+
     def to_add_activity(self, user):
         ''' AP for shelving a book'''
         return activitypub.Add(
@@ -45,16 +66,10 @@ class Tag(OrderedCollectionMixin, BookWyrmModel):
             id='%s#remove' % self.remote_id,
             actor=user.remote_id,
             object=self.book.to_activity(),
-            target=self.to_activity(),
+            target=self.remote_id,
         ).serialize()
 
-    def save(self, *args, **kwargs):
-        ''' create a url-safe lookup key for the tag '''
-        if not self.id:
-            # add identifiers to new tags
-            self.identifier = urllib.parse.quote_plus(self.name)
-        super().save(*args, **kwargs)
 
     class Meta:
         ''' unqiueness constraint '''
-        unique_together = ('user', 'book', 'name')
+        unique_together = ('user', 'book', 'tag')

bookwyrm/models/user.py

@@ -1,49 +1,71 @@
 ''' database schema for user data '''
+import re
 from urllib.parse import urlparse
 
+from django.apps import apps
 from django.contrib.auth.models import AbstractUser
 from django.db import models
 from django.dispatch import receiver
 
 from bookwyrm import activitypub
+from bookwyrm.connectors import get_data
 from bookwyrm.models.shelf import Shelf
-from bookwyrm.models.status import Status
+from bookwyrm.models.status import Status, Review
 from bookwyrm.settings import DOMAIN
 from bookwyrm.signatures import create_key_pair
-from .base_model import ActivityMapping, OrderedCollectionPageMixin
+from bookwyrm.tasks import app
-from .base_model import image_formatter
+from bookwyrm.utils import regex
+from .base_model import OrderedCollectionPageMixin
+from .base_model import ActivitypubMixin, BookWyrmModel
+from .federated_server import FederatedServer
+from . import fields
 
 
 class User(OrderedCollectionPageMixin, AbstractUser):
     ''' a user who wants to read books '''
-    private_key = models.TextField(blank=True, null=True)
+    username = fields.UsernameField()
-    public_key = models.TextField(blank=True, null=True)
+
-    inbox = models.CharField(max_length=255, unique=True)
+    key_pair = fields.OneToOneField(
-    shared_inbox = models.CharField(max_length=255, blank=True, null=True)
+        'KeyPair',
+        on_delete=models.CASCADE,
+        blank=True, null=True,
+        activitypub_field='publicKey',
+        related_name='owner'
+    )
+    inbox = fields.RemoteIdField(unique=True)
+    shared_inbox = fields.RemoteIdField(
+        activitypub_field='sharedInbox',
+        activitypub_wrapper='endpoints',
+        deduplication_field=False,
+        null=True)
     federated_server = models.ForeignKey(
         'FederatedServer',
         on_delete=models.PROTECT,
         null=True,
         blank=True,
     )
-    outbox = models.CharField(max_length=255, unique=True)
+    outbox = fields.RemoteIdField(unique=True)
-    summary = models.TextField(blank=True, null=True)
+    summary = fields.HtmlField(null=True, blank=True)
-    local = models.BooleanField(default=True)
+    local = models.BooleanField(default=False)
-    bookwyrm_user = models.BooleanField(default=True)
+    bookwyrm_user = fields.BooleanField(default=True)
     localname = models.CharField(
         max_length=255,
         null=True,
-        unique=True
+        unique=True,
+        validators=[fields.validate_localname],
     )
     # name is your display name, which you can change at will
-    name = models.CharField(max_length=100, blank=True, null=True)
+    name = fields.CharField(max_length=100, null=True, blank=True)
-    avatar = models.ImageField(upload_to='avatars/', blank=True, null=True)
+    avatar = fields.ImageField(
-    following = models.ManyToManyField(
+        upload_to='avatars/', blank=True, null=True,
+        activitypub_field='icon', alt_field='alt_text')
+    followers = fields.ManyToManyField(
         'self',
+        link_only=True,
         symmetrical=False,
         through='UserFollows',
-        through_fields=('user_subject', 'user_object'),
+        through_fields=('user_object', 'user_subject'),
-        related_name='followers'
+        related_name='following'
     )
     follow_requests = models.ManyToManyField(
         'self',
@@ -66,84 +88,69 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         through_fields=('user', 'status'),
         related_name='favorite_statuses'
     )
-    remote_id = models.CharField(max_length=255, null=True, unique=True)
+    remote_id = fields.RemoteIdField(
+        null=True, unique=True, activitypub_field='id')
     created_date = models.DateTimeField(auto_now_add=True)
     updated_date = models.DateTimeField(auto_now=True)
     last_active_date = models.DateTimeField(auto_now=True)
-    manually_approves_followers = models.BooleanField(default=False)
+    manually_approves_followers = fields.BooleanField(default=False)
 
-    # ---- activitypub serialization settings for this model ----- #
+    name_field = 'username'
     @property
-    def ap_followers(self):
+    def alt_text(self):
-        ''' generates url for activitypub followers page '''
+        ''' alt text with username '''
-        return '%s/followers' % self.remote_id
+        return 'avatar for %s' % (self.localname or self.username)
 
     @property
-    def ap_public_key(self):
+    def display_name(self):
-        ''' format the public key block for activitypub '''
+        ''' show the cleanest version of the user's name possible '''
-        return activitypub.PublicKey(**{
+        if self.name and self.name != '':
-            'id': '%s/#main-key' % self.remote_id,
+            return self.name
-            'owner': self.remote_id,
+        return self.localname or self.username
-            'publicKeyPem': self.public_key,
-        })
 
-    activity_mappings = [
-        ActivityMapping('id', 'remote_id'),
-        ActivityMapping(
-            'preferredUsername',
-            'username',
-            activity_formatter=lambda x: x.split('@')[0]
-        ),
-        ActivityMapping('name', 'name'),
-        ActivityMapping('bookwyrmUser', 'bookwyrm_user'),
-        ActivityMapping('inbox', 'inbox'),
-        ActivityMapping('outbox', 'outbox'),
-        ActivityMapping('followers', 'ap_followers'),
-        ActivityMapping('summary', 'summary'),
-        ActivityMapping(
-            'publicKey',
-            'public_key',
-            model_formatter=lambda x: x.get('publicKeyPem')
-        ),
-        ActivityMapping('publicKey', 'ap_public_key'),
-        ActivityMapping(
-            'endpoints',
-            'shared_inbox',
-            activity_formatter=lambda x: {'sharedInbox': x},
-            model_formatter=lambda x: x.get('sharedInbox')
-        ),
-        ActivityMapping('icon', 'avatar'),
-        ActivityMapping(
-            'manuallyApprovesFollowers',
-            'manually_approves_followers'
-        ),
-        # this field isn't in the activity but should always be false
-        ActivityMapping(None, 'local', model_formatter=lambda x: False),
-    ]
     activity_serializer = activitypub.Person
 
-    def to_outbox(self, **kwargs):
+    def to_outbox(self, filter_type=None, **kwargs):
         ''' an ordered collection of statuses '''
-        queryset = Status.objects.filter(
+        if filter_type:
+            filter_class = apps.get_model(
+                'bookwyrm.%s' % filter_type, require_ready=True)
+            if not issubclass(filter_class, Status):
+                raise TypeError(
+                    'filter_status_class must be a subclass of models.Status')
+            queryset = filter_class.objects
+        else:
+            queryset = Status.objects
+
+        queryset = queryset.filter(
             user=self,
             deleted=False,
-        ).select_subclasses()
+            privacy__in=['public', 'unlisted'],
+        ).select_subclasses().order_by('-published_date')
         return self.to_ordered_collection(queryset, \
                 remote_id=self.outbox, **kwargs)
 
     def to_following_activity(self, **kwargs):
         ''' activitypub following list '''
         remote_id = '%s/following' % self.remote_id
-        return self.to_ordered_collection(self.following, \
+        return self.to_ordered_collection(
-                remote_id=remote_id, id_only=True, **kwargs)
+            self.following.order_by('-updated_date').all(),
+            remote_id=remote_id,
+            id_only=True,
+            **kwargs
+        )
 
     def to_followers_activity(self, **kwargs):
         ''' activitypub followers list '''
         remote_id = '%s/followers' % self.remote_id
-        return self.to_ordered_collection(self.followers, \
+        return self.to_ordered_collection(
-                remote_id=remote_id, id_only=True, **kwargs)
+            self.followers.order_by('-updated_date').all(),
+            remote_id=remote_id,
+            id_only=True,
+            **kwargs
+        )
 
-    def to_activity(self, pure=False):
+    def to_activity(self):
         ''' override default AP serializer to add context object
             idk if this is the best way to go about this '''
         activity_object = super().to_activity()
@@ -163,35 +170,72 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     def save(self, *args, **kwargs):
         ''' populate fields for new local users '''
         # this user already exists, no need to populate fields
-        if self.id:
+        if not self.local and not re.match(regex.full_username, self.username):
-            return super().save(*args, **kwargs)
-
-        if not self.local:
             # generate a username that uses the domain (webfinger format)
             actor_parts = urlparse(self.remote_id)
             self.username = '%s@%s' % (self.username, actor_parts.netloc)
             return super().save(*args, **kwargs)
 
+        if self.id or not self.local:
+            return super().save(*args, **kwargs)
+
         # populate fields for local users
-        self.remote_id = 'https://%s/user/%s' % (DOMAIN, self.username)
+        self.remote_id = 'https://%s/user/%s' % (DOMAIN, self.localname)
-        self.localname = self.username
-        self.username = '%s@%s' % (self.username, DOMAIN)
-        self.actor = self.remote_id
         self.inbox = '%s/inbox' % self.remote_id
         self.shared_inbox = 'https://%s/inbox' % DOMAIN
         self.outbox = '%s/outbox' % self.remote_id
-        if not self.private_key:
-            self.private_key, self.public_key = create_key_pair()
 
         return super().save(*args, **kwargs)
 
+    @property
+    def local_path(self):
+        ''' this model doesn't inherit bookwyrm model, so here we are '''
+        return '/user/%s' % (self.localname or self.username)
+
+
+class KeyPair(ActivitypubMixin, BookWyrmModel):
+    ''' public and private keys for a user '''
+    private_key = models.TextField(blank=True, null=True)
+    public_key = fields.TextField(
+        blank=True, null=True, activitypub_field='publicKeyPem')
+
+    activity_serializer = activitypub.PublicKey
+    serialize_reverse_fields = [('owner', 'owner', 'id')]
+
+    def get_remote_id(self):
+        # self.owner is set by the OneToOneField on User
+        return '%s/#main-key' % self.owner.remote_id
+
+    def save(self, *args, **kwargs):
+        ''' create a key pair '''
+        if not self.public_key:
+            self.private_key, self.public_key = create_key_pair()
+        return super().save(*args, **kwargs)
+
+    def to_activity(self):
+        ''' override default AP serializer to add context object
+            idk if this is the best way to go about this '''
+        activity_object = super().to_activity()
+        del activity_object['@context']
+        del activity_object['type']
+        return activity_object
+
 
 @receiver(models.signals.post_save, sender=User)
+#pylint: disable=unused-argument
 def execute_after_save(sender, instance, created, *args, **kwargs):
     ''' create shelves for new users '''
-    if not instance.local or not created:
+    if not created:
         return
 
+    if not instance.local:
+        set_remote_server.delay(instance.id)
+        return
+
+    instance.key_pair = KeyPair.objects.create(
+        remote_id='%s/#main-key' % instance.remote_id)
+    instance.save()
+
     shelves = [{
         'name': 'To Read',
         'identifier': 'to-read',
@@ -210,3 +254,54 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
             user=instance,
             editable=False
         ).save()
+
+
+@app.task
+def set_remote_server(user_id):
+    ''' figure out the user's remote server in the background '''
+    user = User.objects.get(id=user_id)
+    actor_parts = urlparse(user.remote_id)
+    user.federated_server = \
+        get_or_create_remote_server(actor_parts.netloc)
+    user.save()
+    if user.bookwyrm_user:
+        get_remote_reviews.delay(user.outbox)
+
+
+def get_or_create_remote_server(domain):
+    ''' get info on a remote server '''
+    try:
+        return FederatedServer.objects.get(
+            server_name=domain
+        )
+    except FederatedServer.DoesNotExist:
+        pass
+
+    data = get_data('https://%s/.well-known/nodeinfo' % domain)
+
+    try:
+        nodeinfo_url = data.get('links')[0].get('href')
+    except (TypeError, KeyError):
+        return None
+
+    data = get_data(nodeinfo_url)
+
+    server = FederatedServer.objects.create(
+        server_name=domain,
+        application_type=data['software']['name'],
+        application_version=data['software']['version'],
+    )
+    return server
+
+
+@app.task
+def get_remote_reviews(outbox):
+    ''' ingest reviews by a new remote bookwyrm user '''
+    outbox_page = outbox + '?page=true&type=Review'
+    data = get_data(outbox_page)
+
+    # TODO: pagination?
+    for activity in data['orderedItems']:
+        if not activity['type'] == 'Review':
+            continue
+        activitypub.Review(**activity).to_model(Review)

bookwyrm/outgoing.py

@@ -2,35 +2,36 @@
 import re
 
 from django.db import IntegrityError, transaction
-from django.http import HttpResponseNotFound, JsonResponse
+from django.http import JsonResponse
+from django.shortcuts import get_object_or_404
 from django.views.decorators.csrf import csrf_exempt
-import requests
+from django.views.decorators.http import require_GET
+from markdown import markdown
+from requests import HTTPError
 
 from bookwyrm import activitypub
 from bookwyrm import models
+from bookwyrm.connectors import get_data, ConnectorException
 from bookwyrm.broadcast import broadcast
+from bookwyrm.sanitize_html import InputHtmlParser
 from bookwyrm.status import create_notification
 from bookwyrm.status import create_generated_note
 from bookwyrm.status import delete_status
-from bookwyrm.remote_user import get_or_create_remote_user
 from bookwyrm.settings import DOMAIN
 from bookwyrm.utils import regex
 
 
 @csrf_exempt
+@require_GET
 def outbox(request, username):
     ''' outbox for the requested user '''
-    if request.method != 'GET':
+    user = get_object_or_404(models.User, localname=username)
-        return HttpResponseNotFound()
+    filter_type = request.GET.get('type')
+    if filter_type not in models.status_models:
+        filter_type = None
 
-    try:
-        user = models.User.objects.get(localname=username)
-    except models.User.DoesNotExist:
-        return HttpResponseNotFound()
-
-    # collection overview
     return JsonResponse(
-        user.to_outbox(**request.GET),
+        user.to_outbox(**request.GET, filter_type=filter_type),
         encoder=activitypub.ActivityEncoder
     )
 
@@ -40,6 +41,9 @@ def handle_remote_webfinger(query):
     user = None
 
     # usernames could be @user@domain or user@domain
+    if not query:
+        return None
+
     if query[0] == '@':
         query = query[1:]
 
@@ -54,16 +58,16 @@ def handle_remote_webfinger(query):
         url = 'https://%s/.well-known/webfinger?resource=acct:%s' % \
             (domain, query)
         try:
-            response = requests.get(url)
+            data = get_data(url)
-        except requests.exceptions.ConnectionError:
+        except (ConnectorException, HTTPError):
             return None
-        if not response.ok:
+
-            return None
+        for link in data.get('links'):
-        data = response.json()
+            if link.get('rel') == 'self':
-        for link in data['links']:
-            if link['rel'] == 'self':
                 try:
-                    user = get_or_create_remote_user(link['href'])
+                    user = activitypub.resolve_remote_id(
+                        models.User, link['href']
+                    )
                 except KeyError:
                     return None
     return user
@@ -162,22 +166,30 @@ def handle_imported_book(user, item, include_reviews, privacy):
     if not item.book:
         return
 
-    if item.shelf:
+    existing_shelf = models.ShelfBook.objects.filter(
+        book=item.book, added_by=user).exists()
+
+    # shelve the book if it hasn't been shelved already
+    if item.shelf and not existing_shelf:
         desired_shelf = models.Shelf.objects.get(
             identifier=item.shelf,
             user=user
         )
-        # shelve the book if it hasn't been shelved already
+        shelf_book = models.ShelfBook.objects.create(
-        shelf_book, created = models.ShelfBook.objects.get_or_create(
             book=item.book, shelf=desired_shelf, added_by=user)
-        if created:
+        broadcast(user, shelf_book.to_add_activity(user), privacy=privacy)
-            broadcast(user, shelf_book.to_add_activity(user), privacy=privacy)
 
-            # only add new read-throughs if the item isn't already shelved
+    for read in item.reads:
-            for read in item.reads:
+        # check for an existing readthrough with the same dates
-                read.book = item.book
+        if models.ReadThrough.objects.filter(
-                read.user = user
+                user=user, book=item.book,
-                read.save()
+                start_date=read.start_date,
+                finish_date=read.finish_date
+            ).exists():
+            continue
+        read.book = item.book
+        read.user = user
+        read.save()
 
     if include_reviews and (item.rating or item.review):
         review_title = 'Review of {!r} on Goodreads'.format(
@@ -209,15 +221,72 @@ def handle_delete_status(user, status):
 
 def handle_status(user, form):
     ''' generic handler for statuses '''
-    status = form.save()
+    status = form.save(commit=False)
+    if not status.sensitive and status.content_warning:
+        # the cw text field remains populated when you click "remove"
+        status.content_warning = None
+    status.save()
 
     # inspect the text for user tags
-    text = status.content
+    content = status.content
-    matches = re.finditer(
+    for (mention_text, mention_user) in find_mentions(content):
-        regex.username,
+        # add them to status mentions fk
-        text
+        status.mention_users.add(mention_user)
-    )
+
-    for match in matches:
+        # turn the mention into a link
+        content = re.sub(
+            r'%s([^@]|$)' % mention_text,
+            r'<a href="%s">%s</a>\g<1>' % \
+                (mention_user.remote_id, mention_text),
+            content)
+
+    # add reply parent to mentions and notify
+    if status.reply_parent:
+        status.mention_users.add(status.reply_parent.user)
+        for mention_user in status.reply_parent.mention_users.all():
+            status.mention_users.add(mention_user)
+
+        if status.reply_parent.user.local:
+            create_notification(
+                status.reply_parent.user,
+                'REPLY',
+                related_user=user,
+                related_status=status
+            )
+
+    # deduplicate mentions
+    status.mention_users.set(set(status.mention_users.all()))
+    # create mention notifications
+    for mention_user in status.mention_users.all():
+        if status.reply_parent and mention_user == status.reply_parent.user:
+            continue
+        if mention_user.local:
+            create_notification(
+                mention_user,
+                'MENTION',
+                related_user=user,
+                related_status=status
+            )
+
+    # don't apply formatting to generated notes
+    if not isinstance(status, models.GeneratedNote):
+        status.content = to_markdown(content)
+    # do apply formatting to quotes
+    if hasattr(status, 'quote'):
+        status.quote = to_markdown(status.quote)
+
+    status.save()
+
+    broadcast(user, status.to_create_activity(user), software='bookwyrm')
+
+    # re-format the activity for non-bookwyrm servers
+    remote_activity = status.to_create_activity(user, pure=True)
+    broadcast(user, remote_activity, software='other')
+
+
+def find_mentions(content):
+    ''' detect @mentions in raw status content '''
+    for match in re.finditer(regex.strict_username, content):
         username = match.group().strip().split('@')[1:]
         if len(username) == 1:
             # this looks like a local user (@user), fill in the domain
@@ -228,48 +297,25 @@ def handle_status(user, form):
         if not mention_user:
             # we can ignore users we don't know about
             continue
-        # add them to status mentions fk
+        yield (match.group(), mention_user)
-        status.mention_users.add(mention_user)
-        # create notification if the mentioned user is local
-        if mention_user.local:
-            create_notification(
-                mention_user,
-                'MENTION',
-                related_user=user,
-                related_status=status
-            )
-    status.save()
-
-    # notify reply parent or tagged users
-    if status.reply_parent and status.reply_parent.user.local:
-        create_notification(
-            status.reply_parent.user,
-            'REPLY',
-            related_user=user,
-            related_status=status
-        )
-
-    broadcast(user, status.to_create_activity(user), software='bookwyrm')
-
-    # re-format the activity for non-bookwyrm servers
-    if hasattr(status, 'pure_activity_serializer'):
-        remote_activity = status.to_create_activity(user, pure=True)
-        broadcast(user, remote_activity, software='other')
 
 
-def handle_tag(user, tag):
+def format_links(content):
-    ''' tag a book '''
+    ''' detect and format links '''
-    broadcast(user, tag.to_add_activity(user))
+    return re.sub(
+        r'([^(href=")]|^|\()(https?:\/\/(%s([\w\.\-_\/+&\?=:;,])*))' % \
+                regex.domain,
+        r'\g<1><a href="\g<2>">\g<3></a>',
+        content)
 
-
+def to_markdown(content):
-def handle_untag(user, book, name):
+    ''' catch links and convert to markdown '''
-    ''' tag a book '''
+    content = format_links(content)
-    book = models.Book.objects.get(id=book)
+    content = markdown(content)
-    tag = models.Tag.objects.get(name=name, book=book, user=user)
+    # sanitize resulting html
-    tag_activity = tag.to_remove_activity(user)
+    sanitizer = InputHtmlParser()
-    tag.delete()
+    sanitizer.feed(content)
-
+    return sanitizer.get_output()
-    broadcast(user, tag_activity)
 
 
 def handle_favorite(user, status):
@@ -286,12 +332,13 @@ def handle_favorite(user, status):
     fav_activity = favorite.to_activity()
     broadcast(
         user, fav_activity, privacy='direct', direct_recipients=[status.user])
-    create_notification(
+    if status.user.local:
-        status.user,
+        create_notification(
-        'FAVORITE',
+            status.user,
-        related_user=user,
+            'FAVORITE',
-        related_status=status
+            related_user=user,
-    )
+            related_status=status
+        )
 
 
 def handle_unfavorite(user, status):
@@ -309,28 +356,42 @@ def handle_unfavorite(user, status):
     favorite.delete()
     broadcast(user, fav_activity, direct_recipients=[status.user])
 
+    # check for notification
+    if status.user.local:
+        notification = models.Notification.objects.filter(
+            user=status.user, related_user=user,
+            related_status=status, notification_type='FAVORITE'
+        ).first()
+        if notification:
+            notification.delete()
+
 
 def handle_boost(user, status):
     ''' a user wishes to boost a status '''
+    # is it boostable?
+    if not status.boostable:
+        return
+
     if models.Boost.objects.filter(
             boosted_status=status, user=user).exists():
         # you already boosted that.
         return
     boost = models.Boost.objects.create(
         boosted_status=status,
+        privacy=status.privacy,
         user=user,
     )
-    boost.save()
 
     boost_activity = boost.to_activity()
     broadcast(user, boost_activity)
 
-    create_notification(
+    if status.user.local:
-        status.user,
+        create_notification(
-        'BOOST',
+            status.user,
-        related_user=user,
+            'BOOST',
-        related_status=status
+            related_user=user,
-    )
+            related_status=status
+        )
 
 
 def handle_unboost(user, status):
@@ -343,12 +404,11 @@ def handle_unboost(user, status):
     boost.delete()
     broadcast(user, activity)
 
-
+    # delete related notification
-def handle_update_book(user, book):
+    if status.user.local:
-    ''' broadcast the news about our book '''
+        notification = models.Notification.objects.filter(
-    broadcast(user, book.to_update_activity(user))
+            user=status.user, related_user=user,
-
+            related_status=status, notification_type='BOOST'
-
+        ).first()
-def handle_update_user(user):
+        if notification:
-    ''' broadcast editing a user's profile '''
+            notification.delete()
-    broadcast(user, user.to_update_activity(user))

bookwyrm/remote_user.py

@@ -1,111 +0,0 @@
-''' manage remote users '''
-from urllib.parse import urlparse
-import requests
-
-from django.db import transaction
-
-from bookwyrm import activitypub, models
-from bookwyrm import status as status_builder
-from bookwyrm.tasks import app
-
-
-def get_or_create_remote_user(actor):
-    ''' look up a remote user or add them '''
-    try:
-        return models.User.objects.get(remote_id=actor)
-    except models.User.DoesNotExist:
-        pass
-
-    data = fetch_user_data(actor)
-
-    actor_parts = urlparse(actor)
-    with transaction.atomic():
-        user = activitypub.Person(**data).to_model(models.User)
-        user.federated_server = get_or_create_remote_server(actor_parts.netloc)
-        user.save()
-    if user.bookwyrm_user:
-        get_remote_reviews.delay(user.id)
-    return user
-
-
-def fetch_user_data(actor):
-    ''' load the user's info from the actor url '''
-    try:
-        response = requests.get(
-            actor,
-            headers={'Accept': 'application/activity+json'}
-        )
-    except ConnectionError:
-        return None
-
-    if not response.ok:
-        response.raise_for_status()
-    data = response.json()
-
-    # make sure our actor is who they say they are
-    if actor != data['id']:
-        raise ValueError("Remote actor id must match url.")
-    return data
-
-
-def refresh_remote_user(user):
-    ''' get updated user data from its home instance '''
-    data = fetch_user_data(user.remote_id)
-
-    activity = activitypub.Person(**data)
-    activity.to_model(models.User, instance=user)
-
-
-@app.task
-def get_remote_reviews(user_id):
-    ''' ingest reviews by a new remote bookwyrm user '''
-    try:
-        user = models.User.objects.get(id=user_id)
-    except models.User.DoesNotExist:
-        return
-    outbox_page = user.outbox + '?page=true'
-    response = requests.get(
-        outbox_page,
-        headers={'Accept': 'application/activity+json'}
-    )
-    data = response.json()
-    # TODO: pagination?
-    for activity in data['orderedItems']:
-        status_builder.create_status(activity)
-
-
-def get_or_create_remote_server(domain):
-    ''' get info on a remote server '''
-    try:
-        return models.FederatedServer.objects.get(
-            server_name=domain
-        )
-    except models.FederatedServer.DoesNotExist:
-        pass
-
-    response = requests.get(
-        'https://%s/.well-known/nodeinfo' % domain,
-        headers={'Accept': 'application/activity+json'}
-    )
-
-    if response.status_code != 200:
-        return None
-
-    data = response.json()
-    try:
-        nodeinfo_url = data.get('links')[0].get('href')
-    except (TypeError, KeyError):
-        return None
-
-    response = requests.get(
-        nodeinfo_url,
-        headers={'Accept': 'application/activity+json'}
-    )
-    data = response.json()
-
-    server = models.FederatedServer.objects.create(
-        server_name=domain,
-        application_type=data['software']['name'],
-        application_version=data['software']['version'],
-    )
-    return server

bookwyrm/routine_book_tasks.py

@@ -1,16 +0,0 @@
-''' Routine tasks for keeping your library tidy '''
-from datetime import timedelta
-from django.utils import timezone
-from bookwyrm import books_manager
-from bookwyrm import models
-
-def sync_book_data():
-    ''' update books with any changes to their canonical source '''
-    expiry = timezone.now() - timedelta(days=1)
-    books = models.Edition.objects.filter(
-        sync=True,
-        last_sync_date__lte=expiry
-    ).all()
-    for book in books:
-        # TODO: create background tasks
-        books_manager.update_book(book)

bookwyrm/sanitize_html.py

@@ -1,12 +1,16 @@
 ''' html parser to clean up incoming text from unknown sources '''
 from html.parser import HTMLParser
 
-class InputHtmlParser(HTMLParser):
+class InputHtmlParser(HTMLParser):#pylint: disable=abstract-method
     ''' Removes any html that isn't allowed_tagsed from a block '''
 
     def __init__(self):
         HTMLParser.__init__(self)
-        self.allowed_tags = ['p', 'b', 'i', 'pre', 'a', 'span']
+        self.allowed_tags = [
+            'p', 'br',
+            'b', 'i', 'strong', 'em', 'pre',
+            'a', 'span', 'ul', 'ol', 'li'
+        ]
         self.tag_stack = []
         self.output = []
         # if the html appears invalid, we just won't allow any at all

bookwyrm/settings.py

@@ -3,8 +3,11 @@ import os
 
 from environs import Env
 
+import requests
+
 env = Env()
 DOMAIN = env('DOMAIN')
+VERSION = '0.0.1'
 
 PAGE_LENGTH = env('PAGE_LENGTH', 15)
 
@@ -15,6 +18,13 @@ CELERY_ACCEPT_CONTENT = ['application/json']
 CELERY_TASK_SERIALIZER = 'json'
 CELERY_RESULT_SERIALIZER = 'json'
 
+# email
+EMAIL_HOST = env('EMAIL_HOST')
+EMAIL_PORT = env('EMAIL_PORT', 587)
+EMAIL_HOST_USER = env('EMAIL_HOST_USER')
+EMAIL_HOST_PASSWORD = env('EMAIL_HOST_PASSWORD')
+EMAIL_USE_TLS = env('EMAIL_USE_TLS', True)
+
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -68,6 +78,7 @@ TEMPLATES = [
                 'django.template.context_processors.request',
                 'django.contrib.auth.context_processors.auth',
                 'django.contrib.messages.context_processors.messages',
+                'bookwyrm.context_processors.site_settings',
             ],
         },
     },
@@ -91,10 +102,6 @@ BOOKWYRM_DBS = {
         'HOST': env('POSTGRES_HOST', ''),
         'PORT': 5432
     },
-    'sqlite': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'fedireads.db')
-    }
 }
 
 DATABASES = {
@@ -146,3 +153,6 @@ STATIC_URL = '/static/'
 STATIC_ROOT = os.path.join(BASE_DIR, env('STATIC_ROOT', 'static'))
 MEDIA_URL = '/images/'
 MEDIA_ROOT = os.path.join(BASE_DIR, env('MEDIA_ROOT', 'images'))
+
+USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % (
+    requests.utils.default_user_agent(), VERSION, DOMAIN)

bookwyrm/signatures.py

@@ -31,7 +31,7 @@ def make_signature(sender, destination, date, digest):
         'digest: %s' % digest,
     ]
     message_to_sign = '\n'.join(signature_headers)
-    signer = pkcs1_15.new(RSA.import_key(sender.private_key))
+    signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key))
     signed_message = signer.sign(SHA256.new(message_to_sign.encode('utf8')))
     signature = {
         'keyId': '%s#main-key' % sender.remote_id,

bookwyrm/static/css/format.css

@@ -5,6 +5,11 @@
 .navbar .logo {
     max-height: 50px;
 }
+
+.card {
+    overflow: visible;
+}
+
 /* --- TOGGLES --- */
 input.toggle-control {
     display: none;
@@ -65,6 +70,15 @@ input.toggle-control:checked ~ .modal.toggle-content {
 .cover-container {
     height: 250px;
     width: max-content;
+    max-width: 250px;
+}
+.cover-container.is-large {
+    height: max-content;
+    max-width: 330px;
+}
+.cover-container.is-large img {
+    max-height: 500px;
+    height: auto;
 }
 .cover-container.is-medium {
     height: 150px;
@@ -116,6 +130,9 @@ input.toggle-control:checked ~ .modal.toggle-content {
     vertical-align: middle;
     display: inline;
 }
+.navbar .avatar {
+    max-height: none;
+}
 
 
 /* --- QUOTES --- */
@@ -136,8 +153,3 @@ input.toggle-control:checked ~ .modal.toggle-content {
     content: "\e904";
     right: 0;
 }
-
-/* --- BLOCKQUOTE --- */
-blockquote {
-    white-space: pre-line;
-}

bookwyrm/status.py

@@ -1,7 +1,7 @@
 ''' Handle user activity '''
 from django.utils import timezone
 
-from bookwyrm import activitypub, books_manager, models
+from bookwyrm import models
 from bookwyrm.sanitize_html import InputHtmlParser
 
 
@@ -12,37 +12,6 @@ def delete_status(status):
     status.save()
 
 
-def create_status(activity):
-    ''' unfortunately, it's not QUITE as simple as deserializing it '''
-    # render the json into an activity object
-    serializer = activitypub.activity_objects[activity['type']]
-    activity = serializer(**activity)
-    try:
-        model = models.activity_models[activity.type]
-    except KeyError:
-        # not a type of status we are prepared to deserialize
-        return None
-
-    # ignore notes that aren't replies to known statuses
-    if activity.type == 'Note':
-        reply = models.Status.objects.filter(
-            remote_id=activity.inReplyTo
-        ).first()
-        if not reply:
-            return None
-
-    # look up books
-    book_urls = []
-    if hasattr(activity, 'inReplyToBook'):
-        book_urls.append(activity.inReplyToBook)
-    if hasattr(activity, 'tag'):
-        book_urls += [t['href'] for t in activity.tag if t['type'] == 'Book']
-    for remote_id in book_urls:
-        books_manager.get_or_create_book(remote_id)
-
-    return activity.to_model(model)
-
-
 def create_generated_note(user, content, mention_books=None, privacy='public'):
     ''' a note created by the app about user activity '''
     # sanitize input html

bookwyrm/templates/about.html

@@ -3,13 +3,13 @@
 
 <div class="columns">
     <div class="column block">
-        {% include 'snippets/about.html' with site_settings=site_settings %}
+        {% include 'snippets/about.html' %}
     </div>
 
     <div class="column block">
         <h2 class="title">Code of Conduct</h2>
         <div class="content">
-            {{ site_settings.code_of_conduct | safe }}
+            {{ site.code_of_conduct | safe }}
         </div>
     </div>
 </div>

bookwyrm/templates/author.html

@@ -1,18 +1,36 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
-    <h1 class="title">{{ author.display_name }}</h1>
+    <div class="columns">
+        <div class="column">
+            <h1 class="title">{{ author.name }}</h1>
+        </div>
+        {% if request.user.is_authenticated and perms.bookwyrm.edit_book %}
+        <div class="column is-narrow">
+            <a href="{{ author.local_path }}/edit">
+                <span class="icon icon-pencil">
+                    <span class="is-sr-only">Edit Author</span>
+                </span>
+            </a>
+        </div>
+        {% endif %}
+    </div>
+</div>
 
+<div class="block">
     {% if author.bio %}
     <p>
-        {{ author.bio }}
+        {{ author.bio | to_markdown | safe }}
     </p>
     {% endif %}
+    {% if author.wikipedia_link %}
+    <p><a href="{{ author.wikipedia_link }}" rel=”noopener” target="_blank">Wikipedia</a></p>
+    {% endif %}
 </div>
 
 <div class="block">
-    <h3 class="title is-4">Books by {{ author.display_name }}</h3>
+    <h3 class="title is-4">Books by {{ author.name }}</h3>
     {% include 'snippets/book_tiles.html' with books=books %}
 </div>
 {% endblock %}

bookwyrm/templates/book.html

@@ -1,16 +1,27 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% load humanize %}
 {% block content %}
 
 <div class="block">
-    <div class="level">
+    <div class="columns">
-        <h1 class="title level-left">
+        <div class="column">
-            <span>{% include 'snippets/book_titleby.html' with book=book %}</span>
+            <h1 class="title">
-        </h1>
+                {{ book.title }}{% if book.subtitle %}:
+                <small>{{ book.subtitle }}</small>{% endif %}
+                {% if book.series %}
+                <small class="has-text-grey-dark">({{ book.series }}{% if book.series_number %} #{{ book.series_number }}{% endif %})</small><br>
+                {% endif %}
+            </h1>
+            {% if book.authors %}
+            <h2 class="subtitle">
+                by {% include 'snippets/authors.html' with book=book %}
+            </h2>
+            {% endif %}
+        </div>
 
         {% if request.user.is_authenticated and perms.bookwyrm.edit_book %}
-        <div class="level-right">
+        <div class="column is-narrow">
             <a href="{{ book.id }}/edit">
                 <span class="icon icon-pencil">
                     <span class="is-sr-only">Edit Book</span>
@@ -41,19 +52,44 @@
             </div>
             {% endif %}
 
-            <dl class="content">
+            <section class="content">
-            {% for field in info_fields %}
+                <dl>
-                {% if field.value %}
+                    {% if book.isbn_13 %}
-                <dt>{{ field.name }}:</dt>
+                    <div class="is-flex is-justify-content-space-between">
-                <dd>{{ field.value }}</dd>
+                        <dt>ISBN:</dt>
+                        <dd>{{ book.isbn_13 }}</dd>
+                    </div>
+                    {% endif %}
+
+                    {% if book.oclc_number %}
+                    <div class="is-flex is-justify-content-space-between">
+                        <dt>OCLC Number:</dt>
+                        <dd>{{ book.oclc_number }}</dd>
+                    </div>
+                    {% endif %}
+
+                    {% if book.asin %}
+                    <div class="is-flex is-justify-content-space-between">
+                        <dt>ASIN:</dt>
+                        <dd>{{ book.asin }}</dd>
+                    </div>
+                    {% endif %}
+                </dl>
+
+                <p>
+                {% if book.physical_format %}{{ book.physical_format | title }}{% if book.pages %},<br>{% endif %}{% endif %}
+                {% if book.pages %}{{ book.pages }} pages{% endif %}
+                </p>
+
+                {% if book.openlibrary_key %}
+                <p><a href="https://openlibrary.org/books/{{ book.openlibrary_key }}" target="_blank" rel="noopener">View on OpenLibrary</a></p>
                 {% endif %}
-            {% endfor %}
+            </section>
-            </dl>
         </div>
 
         <div class="column">
             <div class="block">
-                <h3 class="field is-grouped">{% include 'snippets/stars.html' with rating=rating %} ({{ reviews|length }} review{{ reviews|length|pluralize }})</h3>
+                <h3 class="field is-grouped">{% include 'snippets/stars.html' with rating=rating %} ({{ review_count }} review{{ reviews|length|pluralize }})</h3>
 
                 {% include 'snippets/trimmed_text.html' with full=book|book_description %}
 
@@ -86,143 +122,56 @@
                 {% endif %}
 
 
-                {% if book.parent_work.edition_set.count > 1 %}
+                {% if book.parent_work.editions.count > 1 %}
-                <p><a href="/book/{{ book.parent_work.id }}/editions">{{ book.parent_work.edition_set.count }} editions</a></p>
+                <p><a href="/book/{{ book.parent_work.id }}/editions">{{ book.parent_work.editions.count }} editions</a></p>
                 {% endif %}
             </div>
 
-            {% for readthrough in readthroughs %}
+            {# user's relationship to the book #}
-            <div class="content block">
+            <div class="block">
-                <input class="toggle-control" type="radio" name="show-edit-readthrough" id="show-readthrough-{{ readthrough.id }}" checked>
+                {% for shelf in user_shelves %}
-                <div class="toggle-content hidden">
+                <p>
-                    <dl>
+                    This edition is on your <a href="/user/{{ user.localname }}/shelf/{{ shelf.shelf.identifier }}">{{ shelf.shelf.name }}</a> shelf.
-                        {% if readthrough.start_date %}
+                    {% include 'snippets/shelf_selector.html' with current=shelf.shelf %}
-                        <dt>Started reading:</dt>
+                </p>
-                        <dd>{{ readthrough.start_date | naturalday }}</dd>
+                {% endfor %}
-                        {% endif %}
+                {% for shelf in other_edition_shelves %}
-                        {% if readthrough.finish_date %}
+                <p>
-                        <dt>Finished reading:</dt>
+                    A <a href="/book/{{ shelf.book.id }}">different edition</a> of this book is on your <a href="/user/{{ user.localname }}/shelf/{{ shelf.shelf.identifier }}">{{ shelf.shelf.name }}</a> shelf.
-                        <dd>{{ readthrough.finish_date | naturalday }}</dd>
+                    {% include 'snippets/switch_edition_button.html' with edition=book %}
-                        {% elif readthrough.progress %}
+                </p>
-                        <dt>Progress:</dt>
+                {% endfor %}
-                        {% if readthrough.progress_mode == 'PG' %}
-                        <dd>on page {{ readthrough.progress }} of {{ book.pages }}</dd>
-                        {% else %}
-                        <dd>{{ readthrough.progress }}%</dd>
-                        {% endif %}
-                        {% endif %}
-                    </dl>
-                    <div class="field is-grouped">
-                        <label class="button is-small" for="edit-readthrough-{{ readthrough.id }}" role="button" tabindex="0">
-                            <span class="icon icon-pencil">
-                                <span class="is-sr-only">Edit read-through dates</span>
-                            </span>
-                        </label>
-                        <label class="button is-small" for="delete-readthrough-{{ readthrough.id }}" role="button" tabindex="0">
-                            <span class="icon icon-x">
-                                <span class="is-sr-only">Delete this read-through</span>
-                            </span>
-                        </label>
-                    </div>
-                    {% if show_progress %}
-                    Progress Updates:
-                    <ul>
-                        {% if readthrough.finish_date %}
-                        <li>{{ readthrough.start_date | naturalday }}: finished</li>
-                        {% endif %}
-                        {% for progress_update in readthrough.progress_updates %}
-                        <li>
-                            {{ progress_update.created_date | naturalday }}:
-                            {% if progress_update.mode == 'PG' %}
-                            page {{ progress_update.progress }} of {{ book.pages }}
-                            {% else %}
-                            {{ progress_update.progress }}%
-                            {% endif %}
-                        </li>
-                        {% endfor %}
-                        <li>{{ readthrough.start_date | naturalday }}: started</li>
-                    </ul>
-                    {% elif readthrough.progress_updates|length %}
-                    <a href="?showprogress">Show {{ readthrough.progress_updates|length }} Progress Updates</a>
-                    {% endif %}
-                </div>
             </div>
 
+            {% if readthroughs.exists %}
+            <div>
+                <h2 class="title is-5">Your reading activity</h2>
+                {% for readthrough in readthroughs %}
+                    {% include 'snippets/readthrough.html' with readthrough=readthrough %}
+                {% endfor %}
+            </div>
+            {% endif %}
             <div class="block">
-                <input class="toggle-control" type="radio" name="show-edit-readthrough" id="edit-readthrough-{{ readthrough.id }}">
+                <div>
-                <div class="toggle-content hidden">
+                    <input type="radio" class="toggle-control" name="add-readthrough-form" id="hide-create-readthrough" checked>
-                    <div class="box">
+                    <div class="toggle-content hidden">
-                        <form name="edit-readthrough" action="/edit-readthrough" method="post">
+                        <label class="button" for="add-readthrough" class="button" role="button" tabindex="0">Add read dates</label>
-                            {% csrf_token %}
+                    </div>
-                            <input type="hidden" name="id" value="{{ readthrough.id }}">
+                </div>
-                            <div class="field">
+                <div>
-                                <label class="label">
+                    <input type="radio" class="toggle-control" id="add-readthrough" name="add-readthrough-form">
-                                    Started reading
+                    <div class="toggle-content hidden box">
-                                    <input type="date" name="start_date" class="input" id="id_start_date-{{ readthrough.id }}" value="{{ readthrough.start_date | date:"Y-m-d" }}">
+                        <form name="add-readthrough" action="/create-readthrough" method="post">
-                                </label>
+                            {% include 'snippets/readthrough_form.html' with readthrough=None %}
-                            </div>
-                            <div class="columns">
-                                <div class="column">
-                                    <div class="field">
-                                        <label class="label">
-                                            Progress
-                                            <input type="number" name="progress" class="input" id="id_progress-{{ readthrough.id }}" value="{{ readthrough.progress }}">
-                                        </label>
-                                    </div>
-                                </div>
-                                <div class="column">
-                                    <div class="control mt-5">
-                                        <label class="radio">
-                                            <input type="radio" name="progress_mode" id="id_progress_mode-{{ readthrough.id }}" value="PG" {% if readthrough.progress_mode == 'PG' %}checked{% endif %}>
-                                            pages
-                                        </label>
-                                        <label class="radio">
-                                            <input type="radio" name="progress_mode" id="id_progress_mode-{{ readthrough.id }}" value="PCT" {% if readthrough.progress_mode == 'PCT' %}checked{% endif %}>
-                                            percent
-                                        </label>
-                                    </div>
-                                </div>
-                            </div>
-                            <div class="field">
-                                <label class="label">
-                                    Finished reading
-                                    <input type="date" name="finish_date" class="input" id="id_finish_date-{{ readthrough.id }}" value="{{ readthrough.finish_date | date:"Y-m-d" }}">
-                                </label>
-                            </div>
                             <div class="field is-grouped">
-                                <button class="button is-primary" type="submit">Save</button>
+                                <button class="button is-primary" type="submit">Create</button>
-                                <label class="button" for="show-readthrough-{{ readthrough.id }}" role="button" tabindex="0">Cancel</label>
+                                <label class="button" for="hide-create-readthrough" role="button" tabindex="0">Cancel</label>
                             </div>
                         </form>
                     </div>
                 </div>
             </div>
 
-            <div>
-                <input class="toggle-control" type="checkbox" name="delete-readthrough-{{ readthrough.id }}" id="delete-readthrough-{{ readthrough.id }}">
-                <div class="modal toggle-content hidden">
-                    <div class="modal-background"></div>
-                    <div class="modal-card">
-                        <header class="modal-card-head">
-                            <p class="modal-card-title">Delete this read-though?</p>
-                            <label class="delete" for="delete-readthrough-{{ readthrough.id }}" aria-label="close"></label>
-                        </header>
-                        <footer class="modal-card-foot">
-			    <form name="delete-readthrough-{{ readthrough.id }}" action="/delete-readthrough" method="POST">
-				{% csrf_token %}
-				<input type="hidden" name="id" value="{{ readthrough.id }}">
-				<button class="button is-danger is-light" type="submit">
-				    Delete
-				</button>
-				<label for="delete-readthrough-{{ readthrough.id }}" class="button" role="button" tabindex="0">Cancel</button>
-			    </form>
-                        </footer>
-                    </div>
-                    <label class="modal-close is-large" for="delete-readthrough-{{ readthrough.id }}" aria-label="close"></label>
-                </div>
-            </div>
-            {% endfor %}
-
             {% if request.user.is_authenticated %}
             <div class="box">
                 {% include 'snippets/create_status.html' with book=book hide_cover=True %}
@@ -248,16 +197,32 @@
             </div>
 
         </div>
+        <div class="column is-narrow">
+            {% if book.subjects %}
+            <section class="content block">
+                <h2 class="title is-5">Subjects</h2>
+                <ul>
+                {% for subject in book.subjects %}
+                    <li>{{ subject }}</li>
+                {% endfor %}
+                </ul>
+            </section>
+            {% endif %}
+
+            {% if book.subject_places %}
+            <section class="content block">
+                <h2 class="title is-5">Places</h2>
+                <ul>
+                {% for place in book.subject_placess %}
+                    <li>{{ place }}</li>
+                {% endfor %}
+                </ul>
+            </section>
+            {% endif %}
+        </div>
     </div>
 </div>
 
-
-{% if not reviews %}
-<div class="block">
-    <p>No reviews yet!</p>
-</div>
-{% endif %}
-
 <div class="block">
     {% for review in reviews %}
     <div class="block">
@@ -265,14 +230,14 @@
     </div>
     {% endfor %}
 
-    <div class="block columns">
+    <div class="block is-flex is-flex-wrap-wrap">
     {% for rating in ratings %}
-    <div class="column">
+    <div class="block mr-5">
         <div class="media">
-            <div class="media-left">{% include 'snippets/avatar.html' %}</div>
+            <div class="media-left">{% include 'snippets/avatar.html' with user=rating.user %}</div>
             <div class="media-content">
                 <div>
-                    {% include 'snippets/username.html' %}
+                    {% include 'snippets/username.html' with user=rating.user %}
                 </div>
                 <div class="field is-grouped mb-0">
                     <div>rated it</div>
@@ -288,6 +253,5 @@
     </div>
 </div>
 
-
 {% endblock %}
 

bookwyrm/templates/direct_messages.html

@@ -0,0 +1,37 @@
+{% extends 'layout.html' %}
+{% block content %}
+
+<div class="block">
+    <h1 class="title">Direct Messages</h1>
+
+    {% if not activities %}
+    <p>You have no messages right now.</p>
+    {% endif %}
+    {% for activity in activities %}
+    <div class="block">
+    {% include 'snippets/status.html' with status=activity %}
+    </div>
+    {% endfor %}
+
+    <nav class="pagination" role="navigation" aria-label="pagination">
+        {% if prev %}
+        <p class="pagination-previous">
+            <a href="{{ prev }}">
+                <span class="icon icon-arrow-left"></span>
+                Previous
+            </a>
+        </p>
+        {% endif %}
+
+        {% if next %}
+        <p class="pagination-next">
+            <a href="{{ next }}">
+                Next
+                <span class="icon icon-arrow-right"></span>
+            </a>
+        </p>
+        {% endif %}
+    </nav>
+</div>
+
+{% endblock %}

bookwyrm/templates/discover.html

@@ -0,0 +1,80 @@
+{% extends 'layout.html' %}
+{% block content %}
+
+{% if not request.user.is_authenticated %}
+<div class="block">
+    <h1 class="title has-text-centered">{{ site.name }}: {{ site.instance_tagline }}</h1>
+</div>
+
+<section class="tile is-ancestor">
+    <div class="tile is-7 is-parent">
+        <div class="tile is-child box">
+            {% include 'snippets/about.html' %}
+        </div>
+    </div>
+    <div class="tile is-5 is-parent">
+        <div class="tile is-child box has-background-primary-light content">
+            {% if site.allow_registration %}
+            <h2 class="title">Join {{ site.name }}</h2>
+            <form name="register" method="post" action="/user-register">
+                {% include 'snippets/register_form.html' %}
+            </form>
+            {% else %}
+            <h2 class="title">This instance is closed</h2>
+            <p>{{ site.registration_closed_text | safe}}</p>
+            {% endif %}
+        </div>
+    </div>
+</section>
+{% else %}
+<div class="block">
+    <h1 class="title has-text-centered">Discover</h1>
+</div>
+{% endif %}
+
+<div class="block is-hidden-tablet">
+    <h2 class="title has-text-centered">Recent Books</h2>
+</div>
+
+<section class="tile is-ancestor">
+    <div class="tile is-vertical">
+        <div class="tile is-parent">
+            <div class="tile is-child box has-background-white-ter">
+                {% include 'snippets/discover/large-book.html' with book=books.0 %}
+            </div>
+        </div>
+        <div class="tile">
+            <div class="tile is-parent is-6">
+                <div class="tile is-child box has-background-white-ter">
+                    {% include 'snippets/discover/small-book.html' with book=books.1 %}
+                </div>
+            </div>
+            <div class="tile is-parent is-6">
+                <div class="tile is-child box has-background-white-ter">
+                    {% include 'snippets/discover/small-book.html' with book=books.2 %}
+                </div>
+            </div>
+        </div>
+    </div>
+    <div class="tile is-vertical">
+        <div class="tile">
+            <div class="tile is-parent is-6">
+                <div class="tile is-child box has-background-white-ter">
+                    {% include 'snippets/discover/small-book.html' with book=books.3 %}
+                </div>
+            </div>
+            <div class="tile is-parent is-6">
+                <div class="tile is-child box has-background-white-ter">
+                    {% include 'snippets/discover/small-book.html' with book=books.4 %}
+                </div>
+            </div>
+        </div>
+        <div class="tile is-parent">
+            <div class="tile is-child box has-background-white-ter">
+                {% include 'snippets/discover/large-book.html' with book=books.5 %}
+            </div>
+        </div>
+    </div>
+</section>
+
+{% endblock %}

bookwyrm/templates/edit_author.html

@@ -0,0 +1,89 @@
+{% extends 'layout.html' %}
+{% load humanize %}
+{% block content %}
+<div class="block">
+    <div class="level">
+        <h1 class="title level-left">
+            Edit "{{ author.name }}"
+        </h1>
+        <div class="level-right">
+            <a href="/author/{{ author.id }}">
+                <span class="edit-link icon icon-close">
+                    <span class="is-sr-only">Close</span>
+                </span>
+            </a>
+        </div>
+    </div>
+    <div>
+        <p>Added: {{ author.created_date | naturaltime }}</p>
+        <p>Updated: {{ author.updated_date | naturaltime }}</p>
+        <p>Last edited by: <a href="{{ author.last_edited_by.remote_id }}">{{ author.last_edited_by.display_name }}</a></p>
+    </div>
+</div>
+
+{% if form.non_field_errors %}
+<div class="block">
+    <p class="notification is-danger">{{ form.non_field_errors }}</p>
+</div>
+{% endif %}
+
+<form class="block" name="edit-author" action="/edit-author/{{ author.id }}" method="post">
+    {% csrf_token %}
+    <input type="hidden" name="last_edited_by" value="{{ request.user.id }}">
+
+    <div class="columns">
+        <div class="column">
+            <h2 class="title is-4">Metadata</h2>
+            <p><label class="label" for="id_name">Name:</label> {{ form.name }}</p>
+            {% for error in form.name.errors %}
+            <p class="help is-danger">{{ error | escape }}</p>
+            {% endfor %}
+
+            <p><label class="label" for="id_bio">Bio:</label> {{ form.bio }}</p>
+            {% for error in form.bio.errors %}
+            <p class="help is-danger">{{ error | escape }}</p>
+            {% endfor %}
+
+            <p><label class="label" for="id_wikipedia_link">Wikipedia link:</label> {{ form.wikipedia_link }}</p>
+            {% for error in form.wikipedia_link.errors %}
+            <p class="help is-danger">{{ error | escape }}</p>
+            {% endfor %}
+
+            <p><label class="label" for="id_born">Birth date:</label> {{ form.born }}</p>
+            {% for error in form.born.errors %}
+            <p class="help is-danger">{{ error | escape }}</p>
+            {% endfor %}
+
+            <p><label class="label" for="id_died">Death date:</label> {{ form.died }}</p>
+            {% for error in form.died.errors %}
+            <p class="help is-danger">{{ error | escape }}</p>
+            {% endfor %}
+        </div>
+        <div class="column">
+            <h2 class="title is-4">Author Identifiers</h2>
+            <p><label class="label" for="id_openlibrary_key">Openlibrary key:</label> {{ form.openlibrary_key }}</p>
+            {% for error in form.openlibrary_key.errors %}
+            <p class="help is-danger">{{ error | escape }}</p>
+            {% endfor %}
+
+            <p><label class="label" for="id_librarything_key">Librarything key:</label> {{ form.librarything_key }}</p>
+            {% for error in form.librarything_key.errors %}
+            <p class="help is-danger">{{ error | escape }}</p>
+            {% endfor %}
+
+            <p><label class="label" for="id_goodreads_key">Goodreads key:</label> {{ form.goodreads_key }}</p>
+            {% for error in form.goodreads_key.errors %}
+            <p class="help is-danger">{{ error | escape }}</p>
+            {% endfor %}
+
+        </div>
+    </div>
+
+    <div class="block">
+        <button class="button is-primary" type="submit">Save</button>
+        <a class="button" href="/author/{{ author.id }}">Cancel</a>
+    </div>
+</form>
+
+{% endblock %}
+

bookwyrm/templates/edit_book.html

@@ -17,63 +17,47 @@
     <div>
         <p>Added: {{ book.created_date | naturaltime }}</p>
         <p>Updated: {{ book.updated_date | naturaltime }}</p>
+        <p>Last edited by: <a href="{{ book.last_edited_by.remote_id }}">{{ book.last_edited_by.display_name }}</a></p>
     </div>
 </div>
 
-{% if login_form.non_field_errors %}
+{% if form.non_field_errors %}
 <div class="block">
-    <p class="notification is-danger">{{ login_form.non_field_errors }}</p>
+    <p class="notification is-danger">{{ form.non_field_errors }}</p>
 </div>
 {% endif %}
 
 <form class="block" name="edit-book" action="/edit-book/{{ book.id }}" method="post" enctype="multipart/form-data">
     {% csrf_token %}
-    <div class="block">
+    <input type="hidden" name="last_edited_by" value="{{ request.user.id }}">
-        <h2 class="title is-4">Data sync
-            <p class="subtitle is-6">If sync is enabled, any changes will be over-written</p>
-        </h2>
-        <div class="columns">
-            <div class="column is-narrow">
-                <label class="checkbox" for="id_sync"><input class="checkbox" type="checkbox" name="sync" id="id_sync"> Sync</label>
-            </div>
-            <div class="column is-narrow">
-                <label class="checkbox" for="id_sync_cover"><input class="checkbox" type="checkbox" name="sync_cover" id="id_sync_cover"> Sync cover</label>
-            </div>
-        </div>
-    </div>
-
     <div class="columns">
         <div class="column">
             <h2 class="title is-4">Metadata</h2>
-            <p class="fields is-grouped"><label class="label"for="id_title">Title:</label> {{ form.title }} </p>
+            <p class="fields is-grouped"><label class="label" for="id_title">Title:</label> {{ form.title }} </p>
             {% for error in form.title.errors %}
             <p class="help is-danger">{{ error | escape }}</p>
             {% endfor %}
-            <p class="fields is-grouped"><label class="label"for="id_sort_title">Sort title:</label> {{ form.sort_title }} </p>
+            <p class="fields is-grouped"><label class="label" for="id_subtitle">Subtitle:</label> {{ form.subtitle }} </p>
-            {% for error in form.sort_title.errors %}
-            <p class="help is-danger">{{ error | escape }}</p>
-            {% endfor %}
-            <p class="fields is-grouped"><label class="label"for="id_subtitle">Subtitle:</label> {{ form.subtitle }} </p>
             {% for error in form.subtitle.errors %}
             <p class="help is-danger">{{ error | escape }}</p>
             {% endfor %}
-            <p class="fields is-grouped"><label class="label"for="id_description">Description:</label> {{ form.description }} </p>
+            <p class="fields is-grouped"><label class="label" for="id_description">Description:</label> {{ form.description }} </p>
             {% for error in form.description.errors %}
             <p class="help is-danger">{{ error | escape }}</p>
             {% endfor %}
-            <p class="fields is-grouped"><label class="label"for="id_series">Series:</label> {{ form.series }} </p>
+            <p class="fields is-grouped"><label class="label" for="id_series">Series:</label> {{ form.series }} </p>
             {% for error in form.series.errors %}
             <p class="help is-danger">{{ error | escape }}</p>
             {% endfor %}
-            <p class="fields is-grouped"><label class="label"for="id_series_number">Series number:</label> {{ form.series_number }} </p>
+            <p class="fields is-grouped"><label class="label" for="id_series_number">Series number:</label> {{ form.series_number }} </p>
             {% for error in form.series_number.errors %}
             <p class="help is-danger">{{ error | escape }}</p>
             {% endfor %}
-            <p class="fields is-grouped"><label class="label"for="id_first_published_date">First published date:</label> {{ form.first_published_date }} </p>
+            <p class="fields is-grouped"><label class="label" for="id_first_published_date">First published date:</label> {{ form.first_published_date }} </p>
             {% for error in form.first_published_date.errors %}
             <p class="help is-danger">{{ error | escape }}</p>
             {% endfor %}
-            <p class="fields is-grouped"><label class="label"for="id_published_date">Published date:</label> {{ form.published_date }} </p>
+            <p class="fields is-grouped"><label class="label" for="id_published_date">Published date:</label> {{ form.published_date }} </p>
             {% for error in form.published_date.errors %}
             <p class="help is-danger">{{ error | escape }}</p>
             {% endfor %}
@@ -97,7 +81,7 @@
 
             <div class="block">
                 <h2 class="title is-4">Physical Properties</h2>
-                <p class="fields is-grouped"><label class="label"for="id_physical_format">Format:</label> {{ form.physical_format }} </p>
+                <p class="fields is-grouped"><label class="label" for="id_physical_format">Format:</label> {{ form.physical_format }} </p>
                 {% for error in form.physical_format.errors %}
                 <p class="help is-danger">{{ error | escape }}</p>
                 {% endfor %}
@@ -105,7 +89,7 @@
                 <p class="help is-danger">{{ error | escape }}</p>
                 {% endfor %}
 
-                <p class="fields is-grouped"><label class="label"for="id_pages">Pages:</label> {{ form.pages }} </p>
+                <p class="fields is-grouped"><label class="label" for="id_pages">Pages:</label> {{ form.pages }} </p>
                 {% for error in form.pages.errors %}
                 <p class="help is-danger">{{ error | escape }}</p>
                 {% endfor %}
@@ -113,24 +97,24 @@
 
             <div class="block">
                 <h2 class="title is-4">Book Identifiers</h2>
-                <p class="fields is-grouped"><label class="label"for="id_isbn_13">ISBN 13:</label> {{ form.isbn_13 }} </p>
+                <p class="fields is-grouped"><label class="label" for="id_isbn_13">ISBN 13:</label> {{ form.isbn_13 }} </p>
                 {% for error in form.isbn_13.errors %}
                 <p class="help is-danger">{{ error | escape }}</p>
                 {% endfor %}
-                <p class="fields is-grouped"><label class="label"for="id_isbn_10">ISBN 10:</label> {{ form.isbn_10 }} </p>
+                <p class="fields is-grouped"><label class="label" for="id_isbn_10">ISBN 10:</label> {{ form.isbn_10 }} </p>
                 {% for error in form.isbn_10.errors %}
                 <p class="help is-danger">{{ error | escape }}</p>
                 {% endfor %}
-                <p class="fields is-grouped"><label class="label"for="id_openlibrary_key">Openlibrary key:</label> {{ form.openlibrary_key }} </p>
+                <p class="fields is-grouped"><label class="label" for="id_openlibrary_key">Openlibrary key:</label> {{ form.openlibrary_key }} </p>
                 {% for error in form.openlibrary_key.errors %}
                 <p class="help is-danger">{{ error | escape }}</p>
                 {% endfor %}
-                <p class="fields is-grouped"><label class="label"for="id_librarything_key">Librarything key:</label> {{ form.librarything_key }} </p>
+                <p class="fields is-grouped"><label class="label" for="id_librarything_key">OCLC Number:</label> {{ form.oclc_number }} </p>
-                {% for error in form.librarything_key.errors %}
+                {% for error in form.oclc_number.errors %}
                 <p class="help is-danger">{{ error | escape }}</p>
                 {% endfor %}
-                <p class="fields is-grouped"><label class="label"for="id_goodreads_key">Goodreads key:</label> {{ form.goodreads_key }} </p>
+                <p class="fields is-grouped"><label class="label" for="id_asin">ASIN:</label> {{ form.asin }} </p>
-                {% for error in form.goodreads_key.errors %}
+                {% for error in form.ASIN.errors %}
                 <p class="help is-danger">{{ error | escape }}</p>
                 {% endfor %}
             </div>

bookwyrm/templates/editions.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">Editions of <a href="/book/{{ work.id }}">"{{ work.title }}"</a></h1>

bookwyrm/templates/feed.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 
 <div class="columns">
@@ -44,26 +44,38 @@
         <div>
             <input class="toggle-control" type="radio" name="recent-books" id="book-{{ book.id }}" {% if shelf_counter == 1 and forloop.first %}checked{% endif %}>
             <div class="toggle-content hidden" role="tabpanel" id="book-{{ book.id }}-panel">
-                <div class="block">
+                <div class="card">
-                    {% include 'snippets/book_titleby.html' with book=book %}
+                    <div class="card-header">
-                    <div class="columns is-gapless">
+                        <p class="card-header-title">
-                        <div class="column is-narrow">
+                        <span>{% include 'snippets/book_titleby.html' with book=book %}</span>
-                        {% include 'snippets/shelve_button.html' with book=book %}
+                        </>
-                        {% active_shelf book as active_shelf %}
+                        <div class="card-header-icon is-hidden-tablet">
+                            <label class="delete" for="no-book" aria-label="close" role="button"></label>
                         </div>
-                        {% if active_shelf.identifier == 'reading' and book.latest_readthrough %}
+                    </div>
-                        <div class="column">
+                    <div class="card-content">
-                        {% include 'snippets/progress_update.html' with readthrough=book.latest_readthrough %}
+					    <div class="columns is-gapless">
-                        </div>
+						    <div class="column is-narrow">
-                        {% endif %}
+	                    	{% include 'snippets/shelve_button.html' with book=book %}
+	                        {% active_shelf book as active_shelf %}
+	                        </div>
+	                        {% if active_shelf.identifier == 'reading' and book.latest_readthrough %}
+	                        <div class="column">
+	                        {% include 'snippets/progress_update.html' with readthrough=book.latest_readthrough %}
+	                        </div>
+	                        {% endif %}
+	                    </div>
+                        {% include 'snippets/create_status.html' with book=book %}
                     </div>
                 </div>
-                {% include 'snippets/create_status.html' with book=book %}
             </div>
         </div>
         {% endfor %}
         {% endwith %}
         {% endfor %}
+        <div>
+            <input class="toggle-control" type="radio" name="recent-books" id="no-book">
+        </div>
         {% endif %}
     </div>
 

bookwyrm/templates/followers.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">

bookwyrm/templates/following.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">
@@ -31,7 +31,7 @@
     </div>
     {% endfor %}
     {% if not following.count %}
-    <div>No one is following {{ user|username }}</div>
+    <div>{{ user|username }} isn't following any users</div>
     {% endif %}
 </div>
 

bookwyrm/templates/import.html

@@ -21,8 +21,6 @@
         </div>
         <button class="button is-primary" type="submit">Import</button>
     </form>
-    <p>
-    Imports are limited in size, and only the first {{ limit }} items will be imported.
 </div>
 
 <div class="content block">

bookwyrm/templates/import_status.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% load humanize %}
 {% block content %}
 <div class="block">
@@ -8,7 +8,7 @@
     <p>
         Import started: {{ job.created_date | naturaltime }}
     </p>
-    {% if task.successful %}
+    {% if job.complete %}
     <p>
         Import completed: {{ task.date_done | naturaltime }}
     </p>
@@ -18,7 +18,7 @@
 </div>
 
 <div class="block">
-    {% if not task.ready %}
+    {% if not job.complete %}
     Import still in progress.
     <p>
         (Hit reload to update!)

bookwyrm/templates/invite.html

@@ -3,7 +3,8 @@
 
 <div class="columns">
     <div class="column">
-        <div class="block login">
+        <div class="block">
+            {% if valid %}
             <h1 class="title">Create an Account</h1>
             <div>
                 <form name="register" method="post" action="/user-register">
@@ -11,11 +12,17 @@
                     {% include 'snippets/register_form.html' %}
                 </form>
             </div>
+            {% else %}
+            <div class="content">
+                <h1 class="title">Permission Denied</h1>
+                <p>Sorry! This invite code is no longer valid.</p>
+            </div>
+            {% endif %}
         </div>
     </div>
     <div class="column">
         <div class="block">
-            {% include 'snippets/about.html' with site_settings=site_settings %}
+            {% include 'snippets/about.html' %}
         </div>
     </div>
 </div>

bookwyrm/templates/layout.html

@@ -1,29 +1,30 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
-    <title>{% if title %}{{ title }} | {% endif %}BookWyrm</title>
+    <title>{% if title %}{{ title }} | {% endif %}{{ site.name }}</title>
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <link type="text/css" rel="stylesheet" href="/static/css/bulma.min.css">
     <link type="text/css" rel="stylesheet" href="/static/css/format.css">
     <link type="text/css" rel="stylesheet" href="/static/css/icons.css">
 
-    <link rel="shortcut icon" type="image/x-icon" href="/static/images/favicon.ico">
+    <link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}/images/{{ site.favicon }}{% else %}/static/images/favicon.ico{% endif %}">
 
     <meta name="twitter:card" content="summary">
-    <meta name="twitter:title" content="BookWyrm">
+    <meta name="twitter:title" content="{% if title %}{{ title }} | {% endif %}{{ site.name }}">
-    <meta name="og:title" content="BookWyrm">
+    <meta name="og:title" content="{% if title %}{{ title }} | {% endif %}{{ site.name }}">
-    <meta name="twitter:description" content="Federated Social Reading">
+    <meta name="twitter:description" content="{{ site.instance_tagline }}">
-    <meta name="og:description" content="Federated Social Reading">
+    <meta name="og:description" content="{{ site.instance_tagline }}">
-    <meta name="twitter:creator" content="@tripofmice">
+
-    <meta name="twitter:site" content="@tripofmice">
+    <meta name="twitter:image" content="{% if site.logo %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}">
+    <meta name="og:image" content="{% if site.logo %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}">
+    <meta name="twitter:image:alt" content="BookWyrm Logo">
 </head>
 <body>
-
+<nav class="navbar container" role="navigation" aria-label="main navigation">
-<nav class="navbar" role="navigation" aria-label="main navigation">
     <div class="navbar-brand">
         <a class="navbar-item" href="/">
-            <img class="image logo" src="/static/images/logo-small.png" alt="Home page">
+            <img class="image logo" src="{% if site.logo_small %}/images/{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" alt="Home page">
         </a>
         <form class="navbar-item column" action="/search/">
             <div class="field has-addons">
@@ -65,30 +66,45 @@
         <div class="navbar-end">
 	    {% if request.user.is_authenticated %}
             <div class="navbar-item has-dropdown is-hoverable">
-                <div class="navbar-link"><p>
+                <div class="navbar-link" role="button" aria-expanded=false" onclick="toggleMenu(this)" tabindex="0" aria-haspopup="true" aria-controls="navbar-dropdown"><p>
                     {% include 'snippets/avatar.html' with user=request.user %}
                     {% include 'snippets/username.html' with user=request.user %}
                 </p></div>
-                <div class="navbar-dropdown">
+                <ul class="navbar-dropdown" id="navbar-dropdown">
-                    <a href="/user/{{request.user.localname}}" class="navbar-item">
+                    <li>
-                        Profile
+                        <a href="/direct-messages" class="navbar-item">
-                    </a>
+                            Direct messages
-                    <a href="/user-edit" class="navbar-item">
+                        </a>
-                        Settings
+                    </li>
-                    </a>
+                    <li>
-                    <a href="/import" class="navbar-item">
+                        <a href="/user/{{request.user.localname}}" class="navbar-item">
-                        Import books
+                            Profile
-                    </a>
+                        </a>
+                    </li>
+                    <li>
+                        <a href="/user-edit" class="navbar-item">
+                            Settings
+                        </a>
+                    </li>
+                    <li>
+                        <a href="/import" class="navbar-item">
+                            Import books
+                        </a>
+                    </li>
                     {% if perms.bookwyrm.create_invites %}
-                    <a href="/invite" class="navbar-item">
+                    <li>
-                        Invites
+                        <a href="/invite" class="navbar-item">
-                    </a>
+                            Invites
+                        </a>
+                    </li>
                     {% endif %}
                     <hr class="navbar-divider">
-                    <a href="/logout" class="navbar-item">
+                    <li>
-                        Log out
+                        <a href="/logout" class="navbar-item">
-                    </a>
+                            Log out
-                </div>
+                        </a>
+                    </li>
+                </ul>
             </div>
             <div class="navbar-item">
                 <a href="/notifications">
@@ -104,29 +120,77 @@
                     </div>
                 </a>
             </div>
-	    {% else %}
+            {% else %}
             <div class="navbar-item">
-                <div class="buttons">
+                {% if request.path != '/login' and request.path != '/login/' and request.path != '/user-login' %}
-                    <a href="/login" class="button is-primary">
+                <div class="columns">
-                        Join
+                    <div class="column">
-                    </a>
+                        <form name="login" method="post" action="/user-login">
+                            {% csrf_token %}
+                            <div class="field is-grouped">
+                                <div class="control">
+                                    <label class="is-sr-only" for="id_localname">Username:</label>
+                                    <input type="text" name="localname" maxlength="150" class="input" required="" id="id_localname" placeholder="username">
+                                </div>
+                                <div class="control">
+                                    <label class="is-sr-only" for="id_password">Username:</label>
+                                    <input type="password" name="password" maxlength="128" class="input" required="" id="id_password" placeholder="password">
+                                    <p class="help"><a href="/password-reset">Forgot your password?</a></p>
+                                </div>
+                                <button class="button is-primary" type="submit">Log in</button>
+                            </div>
+                        </form>
+                    </div>
+                    {% if site.allow_registration and request.path != '' and request.path != '/' %}
+                    <div class="column is-narrow">
+                        <a href="/" class="button is-link">
+                            Join
+                        </a>
+                    </div>
+                    {% endif %}
                 </div>
+                {% endif %}
             </div>
-	    {% endif %}
+            {% endif %}
         </div>
     </div>
 </nav>
 
 
-<div class="section">
+<div class="section container">
     {% block content %}
     {% endblock %}
 </div>
 
+<footer class="footer">
+    <div class="container">
+        <div class="columns">
+            <div class="column">
+                <p>
+                    <a href="/about">About this server</a>
+                </p>
+                {% if site.admin_email %}
+                <p>
+                    <a href="mailto:{{ site.admin_email }}">Contact site admin</a>
+                </p>
+                {% endif %}
+            </div>
+            {% if site.support_link %}
+            <div class="column">
+                <span class="icon icon-heart"></span>
+                Support {{ site.name }} on <a href="{{ site.support_link }}" target="_blank">{{ site.support_title }}</a>
+            </div>
+            {% endif %}
+            <div class="column">
+                BookWyrm is open source software. You can contribute or report issues on <a href="https://github.com/mouse-reeve/bookwyrm">GitHub</a>.
+            </div>
+        </div>
+    </div>
+</footer>
+
 <script>
     var csrf_token = '{{ csrf_token }}';
 </script>
 <script src="/static/js/shared.js"></script>
 </body>
 </html>
-

bookwyrm/templates/login.html

@@ -11,9 +11,9 @@
             <form name="login" method="post" action="/user-login">
                 {% csrf_token %}
                 <div class="field">
-                    <label class="label" for="id_username">Username:</label>
+                    <label class="label" for="id_localname">Username:</label>
                     <div class="control">
-                        {{ login_form.username }}
+                        {{ login_form.localname }}
                     </div>
                 </div>
                 <div class="field">
@@ -30,13 +30,13 @@
                         <button class="button is-primary" type="submit">Log in</button>
                     </div>
                     <div class="control">
-                        <small><a href="/reset-password">Forgot your password?</a></small>
+                        <small><a href="/password-reset">Forgot your password?</a></small>
                     </div>
                 </div>
             </form>
         </div>
         <div class="box has-background-primary-light">
-            {% if site_settings.allow_registration %}
+            {% if site.allow_registration %}
             <h2 class="title">Create an Account</h2>
             <form name="register" method="post" action="/user-register">
                 {% include 'snippets/register_form.html' %}
@@ -50,7 +50,7 @@
 
     <div class="column">
         <div class="block">
-            {% include 'snippets/about.html' with site_settings=site_settings %}
+            {% include 'snippets/about.html' %}
 
             <p class="block">
                 <a href="/about/">More about this site</a>

bookwyrm/templates/notifications.html

@@ -1,5 +1,6 @@
 {% extends 'layout.html' %}
-{% load humanize %}l
+{% load humanize %}
+{% load bookwyrm_tags %}
 {% block content %}
 <div class="block">
     <h1 class="title">Notifications</h1>
@@ -12,50 +13,89 @@
 
 <div class="block">
     {% for notification in notifications %}
-    <div class="notification level{% if notification.id in unread %} is-primary{% endif %}">
+    {% related_status notification as related_status %}
-        <div class="level-left">
+    <div class="notification {% if notification.id in unread %} is-primary{% endif %}">
-            <p>
+        <div class="columns is-mobile">
-                {% if notification.related_user %}
+            <div class="column is-narrow is-size-3 {% if notification.id in unread%}has-text-white{% else %}has-text-grey{% endif %}">
-                {% include 'snippets/avatar.html' with user=notification.related_user %}
+                {% if notification.notification_type == 'MENTION' %}
-                {% include 'snippets/username.html' with user=notification.related_user %}
+                <span class="icon icon-comment"></span>
-                {% if notification.notification_type == 'FAVORITE' %}
-                    favorited your
-                    <a href="{{ notification.related_status.remote_id}}">status</a>
-
-                {% elif notification.notification_type == 'MENTION' %}
-                    mentioned you in a
-                    <a href="{{ notification.related_status.remote_id}}">status</a>
-
                 {% elif notification.notification_type == 'REPLY' %}
-                    <a href="{{ notification.related_status.remote_id}}">replied</a>
+                <span class="icon icon-comments"></span>
-                    to your
+                {% elif notification.notification_type == 'FOLLOW' or notification.notification_type == 'FOLLOW_REQUEST' %}
-                    <a href="{{ notification.related_status.reply_parent.remote_id}}">status</a>
+                <span class="icon icon-local"></span>
-
-                {% elif notification.notification_type == 'FOLLOW' %}
-                    followed you
-
-                {% elif notification.notification_type == 'FOLLOW_REQUEST' %}
-                    sent you a follow request
-                    <div class="row shrink">
-                        {% include 'snippets/follow_request_buttons.html' with user=notification.related_user %}
-                    </div>
-
                 {% elif notification.notification_type == 'BOOST' %}
-                    boosted your <a href="{{ notification.related_status.remote_id}}">status</a>
+                <span class="icon icon-boost"></span>
+                {% elif notification.notification_type == 'FAVORITE' %}
+                <span class="icon icon-heart"></span>
+                {% elif notification.notification_type == 'IMPORT' %}
+                <span class="icon icon-list"></span>
                 {% endif %}
-                {% else %}
+            </div>
-                    your <a href="/import-status/{{ notification.related_import.id }}">import</a> completed.
+            <div class="column">
+                <div class="block">
+                    <p>
+                        {# DESCRIPTION #}
+                        {% if notification.related_user %}
+                        {% include 'snippets/avatar.html' with user=notification.related_user %}
+                        {% include 'snippets/username.html' with user=notification.related_user %}
+                        {% if notification.notification_type == 'FAVORITE' %}
+                            favorited your
+                            <a href="{{ related_status.local_path }}">{{ related_status | status_preview_name|safe }}</a>
 
+                        {% elif notification.notification_type == 'MENTION' %}
+                            mentioned you in a
+                            <a href="{{ related_status.local_path }}">{{ related_status | status_preview_name|safe }}</a>
+
+                        {% elif notification.notification_type == 'REPLY' %}
+                            <a href="{{ related_status.local_path }}">replied</a>
+                            to your
+                            <a href="{{ related_status.reply_parent.local_path }}">{{ related_status | status_preview_name|safe }}</a>
+                        {% elif notification.notification_type == 'FOLLOW' %}
+                            followed you
+                            {% include 'snippets/follow_button.html' with user=notification.related_user %}
+                        {% elif notification.notification_type == 'FOLLOW_REQUEST' %}
+                            sent you a follow request
+                            <div class="row shrink">
+                                {% include 'snippets/follow_request_buttons.html' with user=notification.related_user %}
+                            </div>
+
+                        {% elif notification.notification_type == 'BOOST' %}
+                            boosted your <a href="{{ related_status.local_path }}">{{ related_status | status_preview_name|safe }}</a>
+                        {% endif %}
+                        {% else %}
+                            your <a href="/import-status/{{ notification.related_import.id }}">import</a> completed.
+                        {% endif %}
+                    </p>
+                </div>
+                {% if related_status %}
+                <div class="block">
+                    {# PREVIEW #}
+                    <div class="notification py-2 {% if notification.id in unread %}is-primary is-light{% else %}has-background-white{% if notification.notification_type == 'REPLY' or notification.notification_type == 'MENTION' %} has-text-black{% else %}-bis has-text-grey-dark{% endif %}{% endif %}">
+                        <div class="columns">
+                            <div class="column">
+                                {% if related_status.content %}
+                                <a href="{{ related_status.local_path }}">{{ related_status.content | safe | truncatewords_html:10 }}</a>
+                                {% elif related_status.quote %}
+                                <a href="{{ related_status.local_path }}">{{ related_status.quote | safe | truncatewords_html:10 }}</a>
+                                {% elif related_status.rating %}
+                                {% include 'snippets/stars.html' with rating=related_status.rating %}
+                                {% endif %}
+                            </div>
+                            <div class="column is-narrow {% if notification.notification_type == 'REPLY' or notification.notification_type == 'MENTION' %}has-text-black{% else %}has-text-grey-dark{% endif %}">
+                                {{ related_status.published_date | post_date }}
+                                {% include 'snippets/privacy-icons.html' with item=related_status %}
+                            </div>
+                        </div>
+                    </div>
+                </div>
                 {% endif %}
-            </p>
+            </div>
         </div>
-
-        <p class="level-right">{{ notification.created_date | naturaltime }}</p>
     </div>
     {% endfor %}
+
     {% if not notifications %}
     <p>You're all caught up!</p>
     {% endif %}
 </div>
-
 {% endblock %}

bookwyrm/templates/password_reset.html

@@ -34,7 +34,7 @@
 
     <div class="column">
         <div class="block">
-            {% include 'snippets/about.html' with site_settings=site_settings %}
+            {% include 'snippets/about.html' %}
         </div>
     </div>
 

bookwyrm/templates/shelf.html

@@ -1,5 +1,5 @@
 {% extends 'layout.html' %}
-{% load fr_display %}
+{% load bookwyrm_tags %}
 {% block content %}
 
 <div class="columns">
@@ -106,6 +106,8 @@
                 <label class="label" for="id_name">Name:</label>
                 <input type="text" name="name" maxlength="100" class="input" required="true" value="{{ shelf.name }}" id="id_name">
             </div>
+            {% else %}
+            <input type="hidden" name="name" required="true" value="{{ shelf.name }}">
             {% endif %}
 
             <label class="label">
@@ -122,7 +124,7 @@
 
 <div class="block">
     <div>
-        {% include 'snippets/shelf.html' with shelf=shelf ratings=ratings %}
+        {% include 'snippets/shelf.html' with shelf=shelf books=books ratings=ratings %}
     </div>
 </div>
 {% endblock %}

bookwyrm/templates/snippets/about.html

@@ -1,7 +1,12 @@
-<h1 class="title">About {{ site_settings.name }}</h1>
+<div class="columns">
-<div class="block">
+    <div class="column is-narrow is-hidden-mobile">
-    <img src="/static/images/logo.png" alt="BookWyrm">
+        <figure class="block">
+            <img src="{% if site.logo_small %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}" alt="BookWyrm logo">
+        </figure>
+    </div>
+    <div class="content">
+        <p class="block">
+            {{ site.instance_description | safe }}
+        </p>
+    </div>
 </div>
-<p class="block">
-    {{ site_settings.instance_description }}
-</p>

bookwyrm/templates/snippets/authors.html

@@ -1 +1 @@
-<a href="/author/{{ book.authors.first.id }}" class="author">{{ book.authors.first.display_name }}</a>
+{% for author in book.authors.all %}<a href="/author/{{ author.id }}" class="author">{{ author.name }}</a>{% if not forloop.last %}, {% endif %}{% endfor %}

bookwyrm/templates/snippets/avatar.html

@@ -1,3 +1,3 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
-<img class="avatar image {% if large %}is-96x96{% else %}is-32x32{% endif %}" src="{% if user.avatar %}/images/{{ user.avatar }}{% else %}/static/images/default_avi.jpg{% endif %}" alt="avatar for {{ user|username }}">
+<img class="avatar image {% if large %}is-96x96{% else %}is-32x32{% endif %}" src="{% if user.avatar %}/images/{{ user.avatar }}{% else %}/static/images/default_avi.jpg{% endif %}" alt="{{ user.alt_text }}">
 

bookwyrm/templates/snippets/book_cover.html

@@ -1,13 +1,13 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="cover-container is-{{ size }}">
 {% if book.cover %}
-    <img class="book-cover" src="/images/{{ book.cover }}" alt="{% include 'snippets/cover_alt.html' with book=book %}">
+<img class="book-cover" src="/images/{{ book.cover }}" alt="{{ book.alt_text }}" title="{{ book.alt_text }}">
 {% else %}
     <div class="no-cover book-cover">
         <img class="book-cover" src="/static/images/no_cover.jpg" alt="No cover">
         <div>
             <p>{{ book.title }}</p>
-            <p>({{ book|edition_info }})</p>
+            <p>({{ book.edition_info }})</p>
         </div>
     </div>
 {% endif %}

bookwyrm/templates/snippets/book_preview.html

@@ -1,4 +1,4 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
 <div class="columns">
     <div class="column is-narrow">
         <div>

bookwyrm/templates/snippets/book_tiles.html

@@ -1,16 +1,11 @@
-<div class="columns">
+<div class="columns is-multiline">
     {% for book in books %}
-    {% if forloop.counter0|divisibleby:"4" %}
-</div>
-<div class="columns">
-        {% endif %}
     <div class="column is-narrow">
         <div class="box">
             <a href="/book/{{ book.id }}">
                 {% include 'snippets/book_cover.html' with book=book %}
             </a>
-            {% include 'snippets/rate_action.html' with user=request.user book=book %}
+            {% include 'snippets/shelve_button.html' with book=book switch_mode=True %}
-            {% include 'snippets/shelve_button.html' with book=book %}
         </div>
     </div>
     {% endfor %}

bookwyrm/templates/snippets/book_titleby.html

@@ -1,9 +1,5 @@
-<span>
+<a href="/book/{{ book.id }}">{{ book.title }}</a>
-    <a href="/book/{{ book.id }}">{{ book.title }}</a>
-</span>
 {% if book.authors %}
-<span>
+by {% include 'snippets/authors.html' with book=book %}
-    by {% include 'snippets/authors.html' with book=book %}
-</span>
 {% endif %}
 

bookwyrm/templates/snippets/boost_button.html

@@ -1,8 +1,9 @@
-{% load fr_display %}
+{% load bookwyrm_tags %}
+
 {% with status.id|uuid as uuid %}
 <form name="boost" action="/boost/{{ status.id }}" method="post" onsubmit="return interact(event)" class="boost-{{ status.id }}-{{ uuid }} {% if request.user|boosted:status  %}hidden{% endif %}" data-id="boost-{{ status.id }}-{{ uuid }}">
     {% csrf_token %}
-    <button class="button is-small" type="submit">
+    <button class="button is-small" type="submit" {% if not status.boostable %}disabled{% endif %}>
         <span class="icon icon-boost">
             <span class="is-sr-only">Boost status</span>
         </span>

bookwyrm/templates/snippets/content_warning_field.html

@@ -0,0 +1,20 @@
+{% load bookwyrm_tags %}
+{% with 0|uuid as uuid %}
+<div class="control">
+    <div>
+        <input type="radio" class="toggle-control" name="sensitive" value="false" id="hide-spoilers-{{ uuid }}" {% if not parent_status.content_warning %}checked{% endif %}>
+        <div class="toggle-content hidden">
+            <label class="button is-small" role="button" tabindex="0" for="include-spoilers-{{ uuid }}">Add spoilers/content warning</label>
+        </div>
+    </div>
+
+    <div>
+        <input type="radio" class="toggle-control" id="include-spoilers-{{ uuid }}" name="sensitive" value="true" {% if parent_status.content_warning %}checked{% endif %}>
+        <div class="toggle-content hidden">
+            <label class="button is-small" role="button" tabindex="0" for="hide-spoilers-{{ uuid }}">Remove spoilers/content warning</label>
+            <label class="is-sr-only" for="id_content_warning_{{ uuid }}">Spoilers/content warning:</label>
+            <input type="text" name="content_warning" maxlength="255" class="input" id="id_content_warning_{{ uuid }}" placeholder="Spoilers ahead!"{% if parent_status.content_warning %} value="{{ parent_status.content_warning }}"{% endif %}>
+        </div>
+    </div>
+</div>
+{% endwith %}