Merge pull request #2148 from hughrun/quotes

add page numbers to comment and quote statuses

.env.dev.example

@@ -1,75 +0,0 @@
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG=true
-USE_HTTPS=false
-
-DOMAIN=your.domain.here
-#EMAIL=your@email.here
-
-# Used for deciding which editions to prefer
-DEFAULT_LANGUAGE="English"
-
-## Leave unset to allow all hosts
-# ALLOWED_HOSTS="localhost,127.0.0.1,[::1]"
-
-MEDIA_ROOT=images/
-
-PGPORT=5432
-POSTGRES_PASSWORD=securedbypassword123
-POSTGRES_USER=fedireads
-POSTGRES_DB=fedireads
-POSTGRES_HOST=db
-
-# Redis activity stream manager
-MAX_STREAM_LENGTH=200
-REDIS_ACTIVITY_HOST=redis_activity
-REDIS_ACTIVITY_PORT=6379
-#REDIS_ACTIVITY_PASSWORD=redispassword345
-
-# Redis as celery broker
-REDIS_BROKER_PORT=6379
-#REDIS_BROKER_PASSWORD=redispassword123
-
-FLOWER_PORT=8888
-#FLOWER_USER=mouse
-#FLOWER_PASSWORD=changeme
-
-EMAIL_HOST="smtp.mailgun.org"
-EMAIL_PORT=587
-EMAIL_HOST_USER=mail@your.domain.here
-EMAIL_HOST_PASSWORD=emailpassword123
-EMAIL_USE_TLS=true
-EMAIL_USE_SSL=false
-
-# Thumbnails Generation
-ENABLE_THUMBNAIL_GENERATION=false
-
-# S3 configuration
-USE_S3=false
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_ACCESS_KEY=
-
-# Commented are example values if you use a non-AWS, S3-compatible service
-# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
-# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
-# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
-
-# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
-# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
-# AWS_S3_REGION_NAME=None         # "fr-par"
-# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
-
-
-# Preview image generation can be computing and storage intensive
-# ENABLE_PREVIEW_IMAGES=True
-
-# Specify RGB tuple or RGB hex strings,
-# or use_dominant_color_light / use_dominant_color_dark
-PREVIEW_BG_COLOR=use_dominant_color_light
-# Change to #FFF if you use use_dominant_color_dark
-PREVIEW_TEXT_COLOR="#363636"
-PREVIEW_IMG_WIDTH=1200
-PREVIEW_IMG_HEIGHT=630
-PREVIEW_DEFAULT_COVER_COLOR="#002549"

.env.example

@@ -8,6 +8,8 @@ USE_HTTPS=true
 DOMAIN=your.domain.here
 EMAIL=your@email.here
 
+# Instance defualt language (see options at bookwyrm/settings.py "LANGUAGES"
+LANGUAGE_CODE="en-us"
 # Used for deciding which editions to prefer
 DEFAULT_LANGUAGE="English"
 
@@ -16,6 +18,7 @@ DEFAULT_LANGUAGE="English"
 
 MEDIA_ROOT=images/
 
+# Database configuration
 PGPORT=5432
 POSTGRES_PASSWORD=securedbypassword123
 POSTGRES_USER=fedireads
@@ -27,21 +30,34 @@ MAX_STREAM_LENGTH=200
 REDIS_ACTIVITY_HOST=redis_activity
 REDIS_ACTIVITY_PORT=6379
 REDIS_ACTIVITY_PASSWORD=redispassword345
+# Optional, use a different redis database (defaults to 0)
+# REDIS_ACTIVITY_DB_INDEX=0
 
 # Redis as celery broker
 REDIS_BROKER_PORT=6379
 REDIS_BROKER_PASSWORD=redispassword123
+# Optional, use a different redis database (defaults to 0)
+# REDIS_BROKER_DB_INDEX=0
 
+# Monitoring for celery
 FLOWER_PORT=8888
-FLOWER_USER=mouse
+FLOWER_USER=admin
 FLOWER_PASSWORD=changeme
 
-EMAIL_HOST="smtp.mailgun.org"
+# Email config
+EMAIL_HOST=smtp.mailgun.org
 EMAIL_PORT=587
 EMAIL_HOST_USER=mail@your.domain.here
 EMAIL_HOST_PASSWORD=emailpassword123
 EMAIL_USE_TLS=true
 EMAIL_USE_SSL=false
+EMAIL_SENDER_NAME=admin
+# defaults to DOMAIN
+EMAIL_SENDER_DOMAIN=
+
+# Query timeouts
+SEARCH_TIMEOUT=15
+QUERY_TIMEOUT=5
 
 # Thumbnails Generation
 ENABLE_THUMBNAIL_GENERATION=false
@@ -69,7 +85,26 @@ AWS_SECRET_ACCESS_KEY=
 # or use_dominant_color_light / use_dominant_color_dark
 PREVIEW_BG_COLOR=use_dominant_color_light
 # Change to #FFF if you use use_dominant_color_dark
-PREVIEW_TEXT_COLOR="#363636"
+PREVIEW_TEXT_COLOR=#363636
 PREVIEW_IMG_WIDTH=1200
 PREVIEW_IMG_HEIGHT=630
-PREVIEW_DEFAULT_COVER_COLOR="#002549"
+PREVIEW_DEFAULT_COVER_COLOR=#002549
+
+# Below are example keys if you want to enable automatically
+# sending telemetry to an OTLP-compatible service. Many of
+# the main monitoring apps have OLTP collectors, including
+# NewRelic, DataDog, and Honeycomb.io - consult their
+# documentation for setup instructions, and what exactly to
+# put below!
+#
+# Service name is an arbitrary tag that is attached to any
+# data sent, used to distinguish different sources. Useful
+# for sending prod and dev metrics to the same place and
+# keeping them separate, for instance!
+
+# API endpoint for your provider
+OTEL_EXPORTER_OTLP_ENDPOINT=
+# Any headers required, usually authentication info
+OTEL_EXPORTER_OTLP_HEADERS=
+# Service name to identify your app
+OTEL_SERVICE_NAME=

.github/workflows/django-tests.yml

@@ -46,6 +46,8 @@ jobs:
         POSTGRES_HOST: 127.0.0.1
         CELERY_BROKER: ""
         REDIS_BROKER_PORT: 6379
+        REDIS_BROKER_PASSWORD: beep
+        USE_DUMMY_CACHE: true
         FLOWER_PORT: 8888
         EMAIL_HOST: "smtp.mailgun.org"
         EMAIL_PORT: 587

.github/workflows/lint-frontend.yaml

@@ -1,5 +1,5 @@
 # @url https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions
-name: Lint Frontend
+name: Lint Frontend (run `./bw-dev stylelint` to fix css errors)
 
 on:
   push:
@@ -8,7 +8,7 @@ on:
       - '.github/workflows/**'
       - 'static/**'
       - '.eslintrc'
-      - '.stylelintrc'
+      - '.stylelintrc.js'
   pull_request:
     branches: [ main, ci, frontend ]
 
@@ -22,17 +22,16 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Install modules
-        run: yarn
+        run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
 
       # See .stylelintignore for files that are not linted.
       - name: Run stylelint
         run: >
-          yarn stylelint bookwyrm/static/**/*.css \
-            --report-needless-disables \
-            --report-invalid-scope-disables
+          npx stylelint bookwyrm/static/css/*.scss bookwyrm/static/css/bookwyrm/**/*.scss \
+            --config dev-tools/.stylelintrc.js
 
       # See .eslintignore for files that are not linted.
       - name: Run ESLint
         run: >
-          yarn eslint bookwyrm/static \
+          npx eslint bookwyrm/static \
             --ext .js,.jsx,.ts,.tsx

.github/workflows/lint-global.yaml

@@ -1,21 +0,0 @@
-# @url https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions
-name: Lint project globally
-
-on:
-  push:
-    branches: [ main, ci ]
-  pull_request:
-    branches: [ main, ci ]
-
-jobs:
-  lint:
-    name: Lint with EditorConfig.
-    runs-on: ubuntu-20.04
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
-
-      - name: EditorConfig
-        uses: greut/eclint-action@v0

.github/workflows/prettier.yaml

@@ -0,0 +1,23 @@
+# @url https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions
+name: JavaScript Prettier (run ./bw-dev prettier to fix)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  lint:
+    name: Lint with Prettier
+    runs-on: ubuntu-20.04
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+      - uses: actions/checkout@v2
+
+      - name: Install modules
+        run: npm install prettier
+
+      - name: Run Prettier
+        run: npx prettier --check bookwyrm/static/js/*.js

.github/workflows/pylint.yml

@@ -21,8 +21,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install pylint
     - name: Analysing the code with pylint
       run: |
-        pylint bookwyrm/ --ignore=migrations,tests --disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801
+        pylint bookwyrm/
 

.gitignore

@@ -4,6 +4,7 @@
 *.swp
 **/__pycache__
 .local
+/nginx/nginx.conf
 
 # VSCode
 /.vscode
@@ -15,6 +16,9 @@
 # BookWyrm
 .env
 /images/
+bookwyrm/static/css/bookwyrm.css
+bookwyrm/static/css/themes/
+!bookwyrm/static/css/themes/bookwyrm-*.scss
 
 # Testing
 .coverage
@@ -23,7 +27,9 @@
 .idea
 
 #Node tools
-/node_modules/
+node_modules/
+package-lock.json
+yarn.lock
 
 #nginx
 nginx/default.conf

.prettierignore

@@ -0,0 +1 @@
+**/vendor/*

.pylintrc

@@ -0,0 +1,6 @@
+[MAIN]
+ignore=migrations
+load-plugins=pylint.extensions.no_self_use
+
+[MESSAGES CONTROL]
+disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801,C3001

.stylelintrc.js

@@ -1,17 +0,0 @@
-/* global module */
-
-module.exports = {
-    "extends": "stylelint-config-standard",
-
-    "plugins": [
-        "stylelint-order"
-    ],
-
-    "rules": {
-        "order/order": [
-            "custom-properties",
-            "declarations"
-        ],
-        "indentation": 4
-    }
-};

Dockerfile

@@ -6,6 +6,7 @@ RUN mkdir /app /app/static /app/images
 
 WORKDIR /app
 
+RUN apt-get update && apt-get install -y gettext libgettextpo-dev tidy && apt-get clean
+
 COPY requirements.txt /app/
 RUN pip install -r requirements.txt --no-cache-dir
-RUN apt-get update && apt-get install -y gettext libgettextpo-dev tidy && apt-get clean

README.md

@@ -9,21 +9,18 @@ Social reading and reviewing, decentralized with ActivityPub
     - [What it is and isn't](#what-it-is-and-isnt)
     - [The role of federation](#the-role-of-federation)
     - [Features](#features)
-- [Book data](#book-data)
-- [Set up Bookwyrm](#set-up-bookwyrm)
+- [Set up BookWyrm](#set-up-bookwyrm)
 
 ## Joining BookWyrm
-BookWyrm is still a young piece of software, and isn't at the level of stability and feature-richness that you'd find in a production-ready application. But it does what it says on the box! If you'd like to join an instance, you can check out the [instances](https://docs.joinbookwyrm.com/instances.html) list.
-
-You can request an invite by entering your email address at https://bookwyrm.social.
+If you'd like to join an instance, you can check out the [instances](https://joinbookwyrm.com/instances/) list.
 
 
 ## Contributing
-See [contributing](https://docs.joinbookwyrm.com/how-to-contribute.html) for code, translation or monetary contributions.
+See [contributing](https://docs.joinbookwyrm.com/contributing.html) for code, translation or monetary contributions.
 
 ## About BookWyrm
 ### What it is and isn't
-BookWyrm is a platform for social reading! You can use it to track what you're reading, review books, and follow your friends. It isn't primarily meant for cataloguing or as a data-source for books, but it does do both of those things to some degree.
+BookWyrm is a platform for social reading. You can use it to track what you're reading, review books, and follow your friends. It isn't primarily meant for cataloguing or as a data-source for books, but it does do both of those things to some degree.
 
 ### The role of federation
 BookWyrm is built on [ActivityPub](http://activitypub.rocks/). With ActivityPub, it inter-operates with different instances of BookWyrm, and other ActivityPub compliant services, like Mastodon. This means you can run an instance for your book club, and still follow your friend who posts on a server devoted to 20th century Russian speculative fiction. It also means that your friend on mastodon can read and comment on a book review that you post on your BookWyrm instance.
@@ -78,8 +75,5 @@ Deployment
 - [Nginx](https://nginx.org/en/) HTTP server
 
 
-## Book data
-The application is set up to share book and author data between instances, and get book data from arbitrary outside sources. Right now, the only connector is to OpenLibrary, but other connectors could be written.
-
-## Set up Bookwyrm
-The [documentation website](https://docs.joinbookwyrm.com/) has instruction on how to set up Bookwyrm in a [developer environment](https://docs.joinbookwyrm.com/developer-environment.html) or [production](https://docs.joinbookwyrm.com/installing-in-production.html).
+## Set up BookWyrm
+The [documentation website](https://docs.joinbookwyrm.com/) has instruction on how to set up BookWyrm in a [developer environment](https://docs.joinbookwyrm.com/install-dev.html) or [production](https://docs.joinbookwyrm.com/install-prod.html).

bookwyrm/activitypub/base_activity.py

@@ -1,6 +1,7 @@
 """ basics for an activitypub serializer """
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
+import logging
 
 from django.apps import apps
 from django.db import IntegrityError, transaction
@@ -8,6 +9,8 @@ from django.db import IntegrityError, transaction
 from bookwyrm.connectors import ConnectorException, get_data
 from bookwyrm.tasks import app
 
+logger = logging.getLogger(__name__)
+
 
 class ActivitySerializerError(ValueError):
     """routine problems serializing activitypub json"""
@@ -20,22 +23,6 @@ class ActivityEncoder(JSONEncoder):
         return o.__dict__
 
 
-@dataclass
-class Link:
-    """for tagging a book in a status"""
-
-    href: str
-    name: str
-    type: str = "Link"
-
-
-@dataclass
-class Mention(Link):
-    """a subtype of Link for mentioning an actor"""
-
-    type: str = "Mention"
-
-
 @dataclass
 # pylint: disable=invalid-name
 class Signature:
@@ -55,12 +42,12 @@ def naive_parse(activity_objects, activity_json, serializer=None):
             activity_json["type"] = "PublicKey"
 
         activity_type = activity_json.get("type")
+        if activity_type in ["Question", "Article"]:
+            return None
         try:
             serializer = activity_objects[activity_type]
         except KeyError as err:
             # we know this exists and that we can't handle it
-            if activity_type in ["Question"]:
-                return None
             raise ActivitySerializerError(err)
 
     return serializer(activity_objects=activity_objects, **activity_json)
@@ -81,7 +68,7 @@ class ActivityObject:
             try:
                 value = kwargs[field.name]
                 if value in (None, MISSING, {}):
-                    raise KeyError()
+                    raise KeyError("Missing required field", field.name)
                 try:
                     is_subclass = issubclass(field.type, ActivityObject)
                 except TypeError:
@@ -198,8 +185,9 @@ class ActivityObject:
                 )
         return instance
 
-    def serialize(self):
+    def serialize(self, **kwargs):
         """convert to dictionary with context attr"""
+        omit = kwargs.get("omit", ())
         data = self.__dict__.copy()
         # recursively serialize
         for (k, v) in data.items():
@@ -208,8 +196,9 @@ class ActivityObject:
                     data[k] = v.serialize()
             except TypeError:
                 pass
-        data = {k: v for (k, v) in data.items() if v is not None}
-        data["@context"] = "https://www.w3.org/ns/activitystreams"
+        data = {k: v for (k, v) in data.items() if v is not None and k not in omit}
+        if "@context" not in omit:
+            data["@context"] = "https://www.w3.org/ns/activitystreams"
         return data
 
 
@@ -222,35 +211,32 @@ def set_related_field(
     model = apps.get_model(f"bookwyrm.{model_name}", require_ready=True)
     origin_model = apps.get_model(f"bookwyrm.{origin_model_name}", require_ready=True)
 
-    with transaction.atomic():
-        if isinstance(data, str):
-            existing = model.find_existing_by_remote_id(data)
-            if existing:
-                data = existing.to_activity()
-            else:
-                data = get_data(data)
-        activity = model.activity_serializer(**data)
+    if isinstance(data, str):
+        existing = model.find_existing_by_remote_id(data)
+        if existing:
+            data = existing.to_activity()
+        else:
+            data = get_data(data)
+    activity = model.activity_serializer(**data)
 
-        # this must exist because it's the object that triggered this function
-        instance = origin_model.find_existing_by_remote_id(related_remote_id)
-        if not instance:
-            raise ValueError(f"Invalid related remote id: {related_remote_id}")
+    # this must exist because it's the object that triggered this function
+    instance = origin_model.find_existing_by_remote_id(related_remote_id)
+    if not instance:
+        raise ValueError(f"Invalid related remote id: {related_remote_id}")
 
-        # set the origin's remote id on the activity so it will be there when
-        # the model instance is created
-        # edition.parentWork = instance, for example
-        model_field = getattr(model, related_field_name)
-        if hasattr(model_field, "activitypub_field"):
-            setattr(
-                activity, getattr(model_field, "activitypub_field"), instance.remote_id
-            )
-        item = activity.to_model()
+    # set the origin's remote id on the activity so it will be there when
+    # the model instance is created
+    # edition.parentWork = instance, for example
+    model_field = getattr(model, related_field_name)
+    if hasattr(model_field, "activitypub_field"):
+        setattr(activity, getattr(model_field, "activitypub_field"), instance.remote_id)
+    item = activity.to_model(model=model)
 
-        # if the related field isn't serialized (attachments on Status), then
-        # we have to set it post-creation
-        if not hasattr(model_field, "activitypub_field"):
-            setattr(item, related_field_name, instance)
-            item.save()
+    # if the related field isn't serialized (attachments on Status), then
+    # we have to set it post-creation
+    if not hasattr(model_field, "activitypub_field"):
+        setattr(item, related_field_name, instance)
+        item.save()
 
 
 def get_model_from_type(activity_type):
@@ -285,9 +271,9 @@ def resolve_remote_id(
     try:
         data = get_data(remote_id)
     except ConnectorException:
-        raise ActivitySerializerError(
-            f"Could not connect to host for remote_id: {remote_id}"
-        )
+        logger.exception("Could not connect to host for remote_id: %s", remote_id)
+        return None
+
     # determine the model implicitly, if not provided
     # or if it's a model with subclasses like Status, check again
     if not model or hasattr(model.objects, "select_subclasses"):
@@ -304,3 +290,28 @@ def resolve_remote_id(
 
     # if we're refreshing, "result" will be set and we'll update it
     return item.to_model(model=model, instance=result, save=save)
+
+
+@dataclass(init=False)
+class Link(ActivityObject):
+    """for tagging a book in a status"""
+
+    href: str
+    name: str = None
+    mediaType: str = None
+    id: str = None
+    attributedTo: str = None
+    availability: str = None
+    type: str = "Link"
+
+    def serialize(self, **kwargs):
+        """remove fields"""
+        omit = ("id", "type", "@context")
+        return super().serialize(omit=omit)
+
+
+@dataclass(init=False)
+class Mention(Link):
+    """a subtype of Link for mentioning an actor"""
+
+    type: str = "Mention"

bookwyrm/activitypub/book.py

@@ -16,7 +16,12 @@ class BookData(ActivityObject):
     librarythingKey: str = None
     goodreadsKey: str = None
     bnfId: str = None
+    viaf: str = None
+    wikidata: str = None
+    asin: str = None
     lastEditedBy: str = None
+    links: List[str] = field(default_factory=lambda: [])
+    fileLinks: List[str] = field(default_factory=lambda: [])
 
 
 # pylint: disable=invalid-name
@@ -25,8 +30,8 @@ class Book(BookData):
     """serializes an edition or work, abstract"""
 
     title: str
-    sortTitle: str = ""
-    subtitle: str = ""
+    sortTitle: str = None
+    subtitle: str = None
     description: str = ""
     languages: List[str] = field(default_factory=lambda: [])
     series: str = ""
@@ -51,7 +56,6 @@ class Edition(Book):
     isbn10: str = ""
     isbn13: str = ""
     oclcNumber: str = ""
-    asin: str = ""
     pages: int = None
     physicalFormat: str = ""
     physicalFormatDetail: str = ""

bookwyrm/activitypub/note.py

@@ -35,6 +35,7 @@ class Note(ActivityObject):
     tag: List[Link] = field(default_factory=lambda: [])
     attachment: List[Document] = field(default_factory=lambda: [])
     sensitive: bool = False
+    updated: str = None
     type: str = "Note"
 
 

bookwyrm/activitypub/person.py

@@ -15,6 +15,11 @@ class PublicKey(ActivityObject):
     publicKeyPem: str
     type: str = "PublicKey"
 
+    def serialize(self, **kwargs):
+        """remove fields"""
+        omit = ("type", "@context")
+        return super().serialize(omit=omit)
+
 
 # pylint: disable=invalid-name
 @dataclass(init=False)
@@ -34,4 +39,5 @@ class Person(ActivityObject):
     bookwyrmUser: bool = False
     manuallyApprovesFollowers: str = False
     discoverable: str = False
+    hideFollows: str = False
     type: str = "Person"

bookwyrm/activitypub/verbs.py

@@ -38,7 +38,7 @@ class Create(Verb):
 class Delete(Verb):
     """Create activity"""
 
-    to: List[str]
+    to: List[str] = field(default_factory=lambda: [])
     cc: List[str] = field(default_factory=lambda: [])
     type: str = "Delete"
 
@@ -69,8 +69,9 @@ class Update(Verb):
 
     def action(self):
         """update a model instance from the dataclass"""
-        if self.object:
-            self.object.to_model(allow_create=False)
+        if not self.object:
+            return
+        self.object.to_model(allow_create=False)
 
 
 @dataclass(init=False)
@@ -136,8 +137,8 @@ class Accept(Verb):
     type: str = "Accept"
 
     def action(self):
-        """find and remove the activity object"""
-        obj = self.object.to_model(save=False, allow_create=False)
+        """accept a request"""
+        obj = self.object.to_model(save=False, allow_create=True)
         obj.accept()
 
 
@@ -149,7 +150,7 @@ class Reject(Verb):
     type: str = "Reject"
 
     def action(self):
-        """find and remove the activity object"""
+        """reject a follow request"""
         obj = self.object.to_model(save=False, allow_create=False)
         obj.reject()
 

bookwyrm/activitystreams.py

@@ -7,7 +7,7 @@ from django.utils import timezone
 
 from bookwyrm import models
 from bookwyrm.redis_store import RedisStore, r
-from bookwyrm.tasks import app
+from bookwyrm.tasks import app, LOW, MEDIUM, HIGH
 
 
 class ActivityStream(RedisStore):
@@ -22,6 +22,11 @@ class ActivityStream(RedisStore):
         stream_id = self.stream_id(user)
         return f"{stream_id}-unread"
 
+    def unread_by_status_type_id(self, user):
+        """the redis key for this user's unread count for this stream"""
+        stream_id = self.stream_id(user)
+        return f"{stream_id}-unread-by-type"
+
     def get_rank(self, obj):  # pylint: disable=no-self-use
         """statuses are sorted by date published"""
         return obj.published_date.timestamp()
@@ -35,6 +40,10 @@ class ActivityStream(RedisStore):
             for user in self.get_audience(status):
                 # add to the unread status count
                 pipeline.incr(self.unread_id(user))
+                # add to the unread status count for status type
+                pipeline.hincrby(
+                    self.unread_by_status_type_id(user), get_status_type(status), 1
+                )
 
         # and go!
         pipeline.execute()
@@ -55,6 +64,7 @@ class ActivityStream(RedisStore):
         """load the statuses to be displayed"""
         # clear unreads for this feed
         r.set(self.unread_id(user), 0)
+        r.delete(self.unread_by_status_type_id(user))
 
         statuses = self.get_store(self.stream_id(user))
         return (
@@ -75,6 +85,14 @@ class ActivityStream(RedisStore):
         """get the unread status count for this user's feed"""
         return int(r.get(self.unread_id(user)) or 0)
 
+    def get_unread_count_by_status_type(self, user):
+        """get the unread status count for this user's feed's status types"""
+        status_types = r.hgetall(self.unread_by_status_type_id(user))
+        return {
+            str(key.decode("utf-8")): int(value) or 0
+            for key, value in status_types.items()
+        }
+
     def populate_streams(self, user):
         """go from zero to a timeline"""
         self.populate_store(self.stream_id(user))
@@ -277,7 +295,19 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
 
 def add_status_on_create_command(sender, instance, created):
     """runs this code only after the database commit completes"""
-    add_status_task.delay(instance.id, increment_unread=created)
+    priority = HIGH
+    # check if this is an old status, de-prioritize if so
+    # (this will happen if federation is very slow, or, more expectedly, on csv import)
+    if instance.published_date < timezone.now() - timedelta(
+        days=1
+    ) or instance.created_date < instance.published_date - timedelta(days=1):
+        priority = LOW
+
+    add_status_task.apply_async(
+        args=(instance.id,),
+        kwargs={"increment_unread": created},
+        queue=priority,
+    )
 
     if sender == models.Boost:
         handle_boost_task.delay(instance.id)
@@ -368,9 +398,15 @@ def populate_streams_on_account_create(sender, instance, created, *args, **kwarg
     """build a user's feeds when they join"""
     if not created or not instance.local:
         return
+    transaction.on_commit(
+        lambda: populate_streams_on_account_create_command(instance.id)
+    )
 
+
+def populate_streams_on_account_create_command(instance_id):
+    """wait for the transaction to complete"""
     for stream in streams:
-        populate_stream_task.delay(stream, instance.id)
+        populate_stream_task.delay(stream, instance_id)
 
 
 @receiver(signals.pre_save, sender=models.ShelfBook)
@@ -409,7 +445,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs):
 # ---- TASKS
 
 
-@app.task(queue="low_priority")
+@app.task(queue=LOW)
 def add_book_statuses_task(user_id, book_id):
     """add statuses related to a book on shelve"""
     user = models.User.objects.get(id=user_id)
@@ -417,7 +453,7 @@ def add_book_statuses_task(user_id, book_id):
     BooksStream().add_book_statuses(user, book)
 
 
-@app.task(queue="low_priority")
+@app.task(queue=LOW)
 def remove_book_statuses_task(user_id, book_id):
     """remove statuses about a book from a user's books feed"""
     user = models.User.objects.get(id=user_id)
@@ -425,7 +461,7 @@ def remove_book_statuses_task(user_id, book_id):
     BooksStream().remove_book_statuses(user, book)
 
 
-@app.task(queue="medium_priority")
+@app.task(queue=MEDIUM)
 def populate_stream_task(stream, user_id):
     """background task for populating an empty activitystream"""
     user = models.User.objects.get(id=user_id)
@@ -433,7 +469,7 @@ def populate_stream_task(stream, user_id):
     stream.populate_streams(user)
 
 
-@app.task(queue="medium_priority")
+@app.task(queue=MEDIUM)
 def remove_status_task(status_ids):
     """remove a status from any stream it might be in"""
     # this can take an id or a list of ids
@@ -446,10 +482,10 @@ def remove_status_task(status_ids):
             stream.remove_object_from_related_stores(status)
 
 
-@app.task(queue="high_priority")
+@app.task(queue=HIGH)
 def add_status_task(status_id, increment_unread=False):
     """add a status to any stream it should be in"""
-    status = models.Status.objects.get(id=status_id)
+    status = models.Status.objects.select_subclasses().get(id=status_id)
     # we don't want to tick the unread count for csv import statuses, idk how better
     # to check than just to see if the states is more than a few days old
     if status.created_date < timezone.now() - timedelta(days=2):
@@ -458,7 +494,7 @@ def add_status_task(status_id, increment_unread=False):
         stream.add_status(status, increment_unread=increment_unread)
 
 
-@app.task(queue="medium_priority")
+@app.task(queue=MEDIUM)
 def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
     """remove all statuses by a user from a viewer's stream"""
     stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -468,7 +504,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
         stream.remove_user_statuses(viewer, user)
 
 
-@app.task(queue="medium_priority")
+@app.task(queue=MEDIUM)
 def add_user_statuses_task(viewer_id, user_id, stream_list=None):
     """add all statuses by a user to a viewer's stream"""
     stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -478,7 +514,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None):
         stream.add_user_statuses(viewer, user)
 
 
-@app.task(queue="medium_priority")
+@app.task(queue=MEDIUM)
 def handle_boost_task(boost_id):
     """remove the original post and other, earlier boosts"""
     instance = models.Status.objects.get(id=boost_id)
@@ -496,3 +532,20 @@ def handle_boost_task(boost_id):
         stream.remove_object_from_related_stores(boosted, stores=audience)
         for status in old_versions:
             stream.remove_object_from_related_stores(status, stores=audience)
+
+
+def get_status_type(status):
+    """return status type even for boosted statuses"""
+    status_type = status.status_type.lower()
+
+    # Check if current status is a boost
+    if hasattr(status, "boost"):
+        # Act in accordance of your findings
+        if hasattr(status.boost.boosted_status, "review"):
+            status_type = "review"
+        if hasattr(status.boost.boosted_status, "comment"):
+            status_type = "comment"
+        if hasattr(status.boost.boosted_status, "quotation"):
+            status_type = "quotation"
+
+    return status_type

bookwyrm/apps.py

@@ -0,0 +1,54 @@
+"""Do further startup configuration and initialization"""
+import os
+import urllib
+import logging
+
+from django.apps import AppConfig
+
+from bookwyrm import settings
+
+logger = logging.getLogger(__name__)
+
+
+def download_file(url, destination):
+    """Downloads a file to the given path"""
+    try:
+        # Ensure our destination directory exists
+        os.makedirs(os.path.dirname(destination))
+        with urllib.request.urlopen(url) as stream:
+            with open(destination, "b+w") as outfile:
+                outfile.write(stream.read())
+    except (urllib.error.HTTPError, urllib.error.URLError):
+        logger.info("Failed to download file %s", url)
+    except OSError:
+        logger.info("Couldn't open font file %s for writing", destination)
+    except:  # pylint: disable=bare-except
+        logger.info("Unknown error in file download")
+
+
+class BookwyrmConfig(AppConfig):
+    """Handles additional configuration"""
+
+    name = "bookwyrm"
+    verbose_name = "BookWyrm"
+
+    # pylint: disable=no-self-use
+    def ready(self):
+        """set up OTLP and preview image files, if desired"""
+        if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+            # pylint: disable=import-outside-toplevel
+            from bookwyrm.telemetry import open_telemetry
+
+            open_telemetry.instrumentDjango()
+
+        if settings.ENABLE_PREVIEW_IMAGES and settings.FONTS:
+            # Download any fonts that we don't have yet
+            logger.debug("Downloading fonts..")
+            for name, config in settings.FONTS.items():
+                font_path = os.path.join(
+                    settings.FONT_DIR, config["directory"], config["filename"]
+                )
+
+                if "url" in config and not os.path.exists(font_path):
+                    logger.info("Just a sec, downloading %s", name)
+                    download_file(config["url"], font_path)

bookwyrm/book_search.py

@@ -82,6 +82,8 @@ def search_identifiers(query, *filters, return_first=False):
         *filters, reduce(operator.or_, (Q(**f) for f in or_filters))
     ).distinct()
     if results.count() <= 1:
+        if return_first:
+            return results.first()
         return results
 
     # when there are multiple editions of the same work, pick the default.
@@ -124,6 +126,7 @@ def search_title_author(query, min_confidence, *filters, return_first=False):
             result = default
         else:
             result = editions.first()
+
         if return_first:
             return result
         list_results.append(result)
@@ -145,8 +148,8 @@ class SearchResult:
 
     def __repr__(self):
         # pylint: disable=consider-using-f-string
-        return "<SearchResult key={!r} title={!r} author={!r}>".format(
-            self.key, self.title, self.author
+        return "<SearchResult key={!r} title={!r} author={!r} confidence={!r}>".format(
+            self.key, self.title, self.author, self.confidence
         )
 
     def json(self):

bookwyrm/connectors/abstract_connector.py

@@ -1,13 +1,16 @@
 """ functionality outline for a book data connector """
 from abc import ABC, abstractmethod
+import imghdr
 import logging
+import re
 
+from django.core.files.base import ContentFile
 from django.db import transaction
 import requests
 from requests.exceptions import RequestException
 
 from bookwyrm import activitypub, models, settings
-from .connector_manager import load_more_data, ConnectorException
+from .connector_manager import load_more_data, ConnectorException, raise_not_valid_url
 from .format_mappings import format_mappings
 
 
@@ -35,61 +38,34 @@ class AbstractMinimalConnector(ABC):
         for field in self_fields:
             setattr(self, field, getattr(info, field))
 
-    def search(self, query, min_confidence=None, timeout=5):
-        """free text search"""
-        params = {}
-        if min_confidence:
-            params["min_confidence"] = min_confidence
+    def get_search_url(self, query):
+        """format the query url"""
+        # Check if the query resembles an ISBN
+        if maybe_isbn(query) and self.isbn_search_url and self.isbn_search_url != "":
+            return f"{self.isbn_search_url}{query}"
 
-        data = self.get_search_data(
-            f"{self.search_url}{query}",
-            params=params,
-            timeout=timeout,
-        )
-        results = []
+        # NOTE: previously, we tried searching isbn and if that produces no results,
+        # searched as free text. This, instead, only searches isbn if it's isbn-y
+        return f"{self.search_url}{query}"
 
-        for doc in self.parse_search_data(data)[:10]:
-            results.append(self.format_search_result(doc))
-        return results
-
-    def isbn_search(self, query):
-        """isbn search"""
-        params = {}
-        data = self.get_search_data(
-            f"{self.isbn_search_url}{query}",
-            params=params,
-        )
-        results = []
-
-        # this shouldn't be returning mutliple results, but just in case
-        for doc in self.parse_isbn_search_data(data)[:10]:
-            results.append(self.format_isbn_search_result(doc))
-        return results
-
-    def get_search_data(self, remote_id, **kwargs):  # pylint: disable=no-self-use
-        """this allows connectors to override the default behavior"""
-        return get_data(remote_id, **kwargs)
+    def process_search_response(self, query, data, min_confidence):
+        """Format the search results based on the formt of the query"""
+        if maybe_isbn(query):
+            return list(self.parse_isbn_search_data(data))[:10]
+        return list(self.parse_search_data(data, min_confidence))[:10]
 
     @abstractmethod
     def get_or_create_book(self, remote_id):
         """pull up a book record by whatever means possible"""
 
     @abstractmethod
-    def parse_search_data(self, data):
+    def parse_search_data(self, data, min_confidence):
         """turn the result json from a search into a list"""
 
-    @abstractmethod
-    def format_search_result(self, search_result):
-        """create a SearchResult obj from json"""
-
     @abstractmethod
     def parse_isbn_search_data(self, data):
         """turn the result json from a search into a list"""
 
-    @abstractmethod
-    def format_isbn_search_result(self, search_result):
-        """create a SearchResult obj from json"""
-
 
 class AbstractConnector(AbstractMinimalConnector):
     """generic book data connector"""
@@ -111,7 +87,7 @@ class AbstractConnector(AbstractMinimalConnector):
                 return existing.default_edition
             return existing
 
-        # load the json
+        # load the json data from the remote data source
         data = self.get_book_data(remote_id)
         if self.is_work_data(data):
             try:
@@ -126,7 +102,7 @@ class AbstractConnector(AbstractMinimalConnector):
             try:
                 work_data = self.get_work_from_edition_data(data)
             except (KeyError, ConnectorException) as err:
-                logger.exception(err)
+                logger.info(err)
                 work_data = data
 
         if not work_data or not edition_data:
@@ -150,27 +126,37 @@ class AbstractConnector(AbstractMinimalConnector):
         """this allows connectors to override the default behavior"""
         return get_data(remote_id)
 
-    def create_edition_from_data(self, work, edition_data):
+    def create_edition_from_data(self, work, edition_data, instance=None):
         """if we already have the work, we're ready"""
         mapped_data = dict_from_mappings(edition_data, self.book_mappings)
         mapped_data["work"] = work.remote_id
         edition_activity = activitypub.Edition(**mapped_data)
-        edition = edition_activity.to_model(model=models.Edition, overwrite=False)
-        edition.connector = self.connector
-        edition.save()
+        edition = edition_activity.to_model(
+            model=models.Edition, overwrite=False, instance=instance
+        )
+
+        # if we're updating an existing instance, we don't need to load authors
+        if instance:
+            return edition
+
+        if not edition.connector:
+            edition.connector = self.connector
+            edition.save(broadcast=False, update_fields=["connector"])
 
         for author in self.get_authors_from_data(edition_data):
             edition.authors.add(author)
+        # use the authors from the work if none are found for the edition
         if not edition.authors.exists() and work.authors.exists():
             edition.authors.set(work.authors.all())
 
         return edition
 
-    def get_or_create_author(self, remote_id):
+    def get_or_create_author(self, remote_id, instance=None):
         """load that author"""
-        existing = models.Author.find_existing_by_remote_id(remote_id)
-        if existing:
-            return existing
+        if not instance:
+            existing = models.Author.find_existing_by_remote_id(remote_id)
+            if existing:
+                return existing
 
         data = self.get_book_data(remote_id)
 
@@ -181,7 +167,24 @@ class AbstractConnector(AbstractMinimalConnector):
             return None
 
         # this will dedupe
-        return activity.to_model(model=models.Author, overwrite=False)
+        return activity.to_model(
+            model=models.Author, overwrite=False, instance=instance
+        )
+
+    def get_remote_id_from_model(self, obj):
+        """given the data stored, how can we look this up"""
+        return getattr(obj, getattr(self, "generated_remote_link_field"))
+
+    def update_author_from_remote(self, obj):
+        """load the remote data from this connector and add it to an existing author"""
+        remote_id = self.get_remote_id_from_model(obj)
+        return self.get_or_create_author(remote_id, instance=obj)
+
+    def update_book_from_remote(self, obj):
+        """load the remote data from this connector and add it to an existing book"""
+        remote_id = self.get_remote_id_from_model(obj)
+        data = self.get_book_data(remote_id)
+        return self.create_edition_from_data(obj.parent_work, data, instance=obj)
 
     @abstractmethod
     def is_work_data(self, data):
@@ -220,36 +223,38 @@ def dict_from_mappings(data, mappings):
 def get_data(url, params=None, timeout=10):
     """wrapper for request.get"""
     # check if the url is blocked
-    if models.FederatedServer.is_blocked(url):
-        raise ConnectorException(f"Attempting to load data from blocked url: {url}")
+    raise_not_valid_url(url)
 
     try:
         resp = requests.get(
             url,
             params=params,
-            headers={
-                "Accept": "application/json; charset=utf-8",
+            headers={  # pylint: disable=line-too-long
+                "Accept": (
+                    'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'
+                ),
                 "User-Agent": settings.USER_AGENT,
             },
             timeout=timeout,
         )
     except RequestException as err:
-        logger.exception(err)
-        raise ConnectorException()
+        logger.info(err)
+        raise ConnectorException(err)
 
     if not resp.ok:
         raise ConnectorException()
     try:
         data = resp.json()
     except ValueError as err:
-        logger.exception(err)
-        raise ConnectorException()
+        logger.info(err)
+        raise ConnectorException(err)
 
     return data
 
 
 def get_image(url, timeout=10):
     """wrapper for requesting an image"""
+    raise_not_valid_url(url)
     try:
         resp = requests.get(
             url,
@@ -259,11 +264,19 @@ def get_image(url, timeout=10):
             timeout=timeout,
         )
     except RequestException as err:
-        logger.exception(err)
-        return None
+        logger.info(err)
+        return None, None
+
     if not resp.ok:
-        return None
-    return resp
+        return None, None
+
+    image_content = ContentFile(resp.content)
+    extension = imghdr.what(None, image_content.read())
+    if not extension:
+        logger.info("File requested was not an image: %s", url)
+        return None, None
+
+    return image_content, extension
 
 
 class Mapping:
@@ -307,3 +320,9 @@ def unique_physical_format(format_text):
         # try a direct match, so saving this would be redundant
         return None
     return format_text
+
+
+def maybe_isbn(query):
+    """check if a query looks like an isbn"""
+    isbn = re.sub(r"[\W_]", "", query)  # removes filler characters
+    return len(isbn) in [10, 13]  # ISBN10 or ISBN13

bookwyrm/connectors/bookwyrm_connector.py

@@ -10,15 +10,12 @@ class Connector(AbstractMinimalConnector):
     def get_or_create_book(self, remote_id):
         return activitypub.resolve_remote_id(remote_id, model=models.Edition)
 
-    def parse_search_data(self, data):
-        return data
-
-    def format_search_result(self, search_result):
-        search_result["connector"] = self
-        return SearchResult(**search_result)
+    def parse_search_data(self, data, min_confidence):
+        for search_result in data:
+            search_result["connector"] = self
+            yield SearchResult(**search_result)
 
     def parse_isbn_search_data(self, data):
-        return data
-
-    def format_isbn_search_result(self, search_result):
-        return self.format_search_result(search_result)
+        for search_result in data:
+            search_result["connector"] = self
+            yield SearchResult(**search_result)

bookwyrm/connectors/connector_manager.py

@@ -1,16 +1,18 @@
 """ interface with whatever connectors the app has """
-from datetime import datetime
+import asyncio
 import importlib
+import ipaddress
 import logging
-import re
 from urllib.parse import urlparse
 
+import aiohttp
 from django.dispatch import receiver
 from django.db.models import signals
 
 from requests import HTTPError
 
 from bookwyrm import book_search, models
+from bookwyrm.settings import SEARCH_TIMEOUT, USER_AGENT
 from bookwyrm.tasks import app
 
 logger = logging.getLogger(__name__)
@@ -20,54 +22,85 @@ class ConnectorException(HTTPError):
     """when the connector can't do what was asked"""
 
 
+async def get_results(session, url, min_confidence, query, connector):
+    """try this specific connector"""
+    # pylint: disable=line-too-long
+    headers = {
+        "Accept": (
+            'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'
+        ),
+        "User-Agent": USER_AGENT,
+    }
+    params = {"min_confidence": min_confidence}
+    try:
+        async with session.get(url, headers=headers, params=params) as response:
+            if not response.ok:
+                logger.info("Unable to connect to %s: %s", url, response.reason)
+                return
+
+            try:
+                raw_data = await response.json()
+            except aiohttp.client_exceptions.ContentTypeError as err:
+                logger.exception(err)
+                return
+
+            return {
+                "connector": connector,
+                "results": connector.process_search_response(
+                    query, raw_data, min_confidence
+                ),
+            }
+    except asyncio.TimeoutError:
+        logger.info("Connection timed out for url: %s", url)
+    except aiohttp.ClientError as err:
+        logger.exception(err)
+
+
+async def async_connector_search(query, items, min_confidence):
+    """Try a number of requests simultaneously"""
+    timeout = aiohttp.ClientTimeout(total=SEARCH_TIMEOUT)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        tasks = []
+        for url, connector in items:
+            tasks.append(
+                asyncio.ensure_future(
+                    get_results(session, url, min_confidence, query, connector)
+                )
+            )
+
+        results = await asyncio.gather(*tasks)
+        return results
+
+
 def search(query, min_confidence=0.1, return_first=False):
     """find books based on arbitary keywords"""
     if not query:
         return []
     results = []
 
-    # Have we got a ISBN ?
-    isbn = re.sub(r"[\W_]", "", query)
-    maybe_isbn = len(isbn) in [10, 13]  # ISBN10 or ISBN13
-
-    timeout = 15
-    start_time = datetime.now()
+    items = []
     for connector in get_connectors():
-        result_set = None
-        if maybe_isbn and connector.isbn_search_url and connector.isbn_search_url != "":
-            # Search on ISBN
-            try:
-                result_set = connector.isbn_search(isbn)
-            except Exception as err:  # pylint: disable=broad-except
-                logger.exception(err)
-                # if this fails, we can still try regular search
+        # get the search url from the connector before sending
+        url = connector.get_search_url(query)
+        try:
+            raise_not_valid_url(url)
+        except ConnectorException:
+            # if this URL is invalid we should skip it and move on
+            logger.info("Request denied to blocked domain: %s", url)
+            continue
+        items.append((url, connector))
 
-        # if no isbn search results, we fallback to generic search
-        if not result_set:
-            try:
-                result_set = connector.search(query, min_confidence=min_confidence)
-            except Exception as err:  # pylint: disable=broad-except
-                # we don't want *any* error to crash the whole search page
-                logger.exception(err)
-                continue
-
-        if return_first and result_set:
-            # if we found anything, return it
-            return result_set[0]
-
-        if result_set:
-            results.append(
-                {
-                    "connector": connector,
-                    "results": result_set,
-                }
-            )
-        if (datetime.now() - start_time).seconds >= timeout:
-            break
+    # load as many results as we can
+    results = asyncio.run(async_connector_search(query, items, min_confidence))
+    results = [r for r in results if r]
 
     if return_first:
-        return None
+        # find the best result from all the responses and return that
+        all_results = [r for con in results for r in con["results"]]
+        all_results = sorted(all_results, key=lambda r: r.confidence, reverse=True)
+        return all_results[0] if all_results else None
 
+    # failed requests will return None, so filter those out
     return results
 
 
@@ -133,3 +166,20 @@ def create_connector(sender, instance, created, *args, **kwargs):
     """create a connector to an external bookwyrm server"""
     if instance.application_type == "bookwyrm":
         get_or_create_connector(f"https://{instance.server_name}")
+
+
+def raise_not_valid_url(url):
+    """do some basic reality checks on the url"""
+    parsed = urlparse(url)
+    if not parsed.scheme in ["http", "https"]:
+        raise ConnectorException("Invalid scheme: ", url)
+
+    try:
+        ipaddress.ip_address(parsed.netloc)
+        raise ConnectorException("Provided url is an IP address: ", url)
+    except ValueError:
+        # it's not an IP address, which is good
+        pass
+
+    if models.FederatedServer.is_blocked(url):
+        raise ConnectorException(f"Attempting to load data from blocked url: {url}")

bookwyrm/connectors/inventaire.py

@@ -11,6 +11,8 @@ from .connector_manager import ConnectorException
 class Connector(AbstractConnector):
     """instantiate a connector for inventaire"""
 
+    generated_remote_link_field = "inventaire_id"
+
     def __init__(self, identifier):
         super().__init__(identifier)
 
@@ -67,7 +69,7 @@ class Connector(AbstractConnector):
         extracted = list(data.get("entities").values())
         try:
             data = extracted[0]
-        except KeyError:
+        except (KeyError, IndexError):
             raise ConnectorException("Invalid book data")
         # flatten the data so that images, uri, and claims are on the same level
         return {
@@ -75,59 +77,49 @@ class Connector(AbstractConnector):
             **{k: data.get(k) for k in ["uri", "image", "labels", "sitelinks", "type"]},
         }
 
-    def search(self, query, min_confidence=None):  # pylint: disable=arguments-differ
-        """overrides default search function with confidence ranking"""
-        results = super().search(query)
-        if min_confidence:
-            # filter the search results after the fact
-            return [r for r in results if r.confidence >= min_confidence]
-        return results
-
-    def parse_search_data(self, data):
-        return data.get("results")
-
-    def format_search_result(self, search_result):
-        images = search_result.get("image")
-        cover = f"{self.covers_url}/img/entities/{images[0]}" if images else None
-        # a deeply messy translation of inventaire's scores
-        confidence = float(search_result.get("_score", 0.1))
-        confidence = 0.1 if confidence < 150 else 0.999
-        return SearchResult(
-            title=search_result.get("label"),
-            key=self.get_remote_id(search_result.get("uri")),
-            author=search_result.get("description"),
-            view_link=f"{self.base_url}/entity/{search_result.get('uri')}",
-            cover=cover,
-            confidence=confidence,
-            connector=self,
-        )
+    def parse_search_data(self, data, min_confidence):
+        for search_result in data.get("results", []):
+            images = search_result.get("image")
+            cover = f"{self.covers_url}/img/entities/{images[0]}" if images else None
+            # a deeply messy translation of inventaire's scores
+            confidence = float(search_result.get("_score", 0.1))
+            confidence = 0.1 if confidence < 150 else 0.999
+            if confidence < min_confidence:
+                continue
+            yield SearchResult(
+                title=search_result.get("label"),
+                key=self.get_remote_id(search_result.get("uri")),
+                author=search_result.get("description"),
+                view_link=f"{self.base_url}/entity/{search_result.get('uri')}",
+                cover=cover,
+                confidence=confidence,
+                connector=self,
+            )
 
     def parse_isbn_search_data(self, data):
         """got some daaaata"""
         results = data.get("entities")
         if not results:
-            return []
-        return list(results.values())
-
-    def format_isbn_search_result(self, search_result):
-        """totally different format than a regular search result"""
-        title = search_result.get("claims", {}).get("wdt:P1476", [])
-        if not title:
-            return None
-        return SearchResult(
-            title=title[0],
-            key=self.get_remote_id(search_result.get("uri")),
-            author=search_result.get("description"),
-            view_link=f"{self.base_url}/entity/{search_result.get('uri')}",
-            cover=self.get_cover_url(search_result.get("image")),
-            connector=self,
-        )
+            return
+        for search_result in list(results.values()):
+            title = search_result.get("claims", {}).get("wdt:P1476", [])
+            if not title:
+                continue
+            yield SearchResult(
+                title=title[0],
+                key=self.get_remote_id(search_result.get("uri")),
+                author=search_result.get("description"),
+                view_link=f"{self.base_url}/entity/{search_result.get('uri')}",
+                cover=self.get_cover_url(search_result.get("image")),
+                connector=self,
+            )
 
     def is_work_data(self, data):
         return data.get("type") == "work"
 
     def load_edition_data(self, work_uri):
         """get a list of editions for a work"""
+        # pylint: disable=line-too-long
         url = f"{self.books_url}?action=reverse-claims&property=wdt:P629&value={work_uri}&sort=true"
         return get_data(url)
 
@@ -209,6 +201,11 @@ class Connector(AbstractConnector):
             return ""
         return data.get("extract")
 
+    def get_remote_id_from_model(self, obj):
+        """use get_remote_id to figure out the link from a model obj"""
+        remote_id_value = obj.inventaire_id
+        return self.get_remote_id(remote_id_value)
+
 
 def get_language_code(options, code="en"):
     """when there are a bunch of translation but we need a single field"""

bookwyrm/connectors/openlibrary.py

@@ -12,6 +12,8 @@ from .openlibrary_languages import languages
 class Connector(AbstractConnector):
     """instantiate a connector for OL"""
 
+    generated_remote_link_field = "openlibrary_link"
+
     def __init__(self, identifier):
         super().__init__(identifier)
 
@@ -66,6 +68,30 @@ class Connector(AbstractConnector):
             Mapping("born", remote_field="birth_date"),
             Mapping("died", remote_field="death_date"),
             Mapping("bio", formatter=get_description),
+            Mapping(
+                "isni",
+                remote_field="remote_ids",
+                formatter=lambda b: get_dict_field(b, "isni"),
+            ),
+            Mapping(
+                "asin",
+                remote_field="remote_ids",
+                formatter=lambda b: get_dict_field(b, "amazon"),
+            ),
+            Mapping(
+                "viaf",
+                remote_field="remote_ids",
+                formatter=lambda b: get_dict_field(b, "viaf"),
+            ),
+            Mapping(
+                "wikidata",
+                remote_field="remote_ids",
+                formatter=lambda b: get_dict_field(b, "wikidata"),
+            ),
+            Mapping(
+                "wikipedia_link", remote_field="links", formatter=get_wikipedia_link
+            ),
+            Mapping("inventaire_id", remote_field="links", formatter=get_inventaire_id),
         ]
 
     def get_book_data(self, remote_id):
@@ -126,39 +152,41 @@ class Connector(AbstractConnector):
         image_name = f"{cover_id}-{size}.jpg"
         return f"{self.covers_url}/b/id/{image_name}"
 
-    def parse_search_data(self, data):
-        return data.get("docs")
+    def parse_search_data(self, data, min_confidence):
+        for idx, search_result in enumerate(data.get("docs")):
+            # build the remote id from the openlibrary key
+            key = self.books_url + search_result["key"]
+            author = search_result.get("author_name") or ["Unknown"]
+            cover_blob = search_result.get("cover_i")
+            cover = self.get_cover_url([cover_blob], size="M") if cover_blob else None
 
-    def format_search_result(self, search_result):
-        # build the remote id from the openlibrary key
-        key = self.books_url + search_result["key"]
-        author = search_result.get("author_name") or ["Unknown"]
-        cover_blob = search_result.get("cover_i")
-        cover = self.get_cover_url([cover_blob], size="M") if cover_blob else None
-        return SearchResult(
-            title=search_result.get("title"),
-            key=key,
-            author=", ".join(author),
-            connector=self,
-            year=search_result.get("first_publish_year"),
-            cover=cover,
-        )
+            # OL doesn't provide confidence, but it does sort by an internal ranking, so
+            # this confidence value is relative to the list position
+            confidence = 1 / (idx + 1)
+
+            yield SearchResult(
+                title=search_result.get("title"),
+                key=key,
+                author=", ".join(author),
+                connector=self,
+                year=search_result.get("first_publish_year"),
+                cover=cover,
+                confidence=confidence,
+            )
 
     def parse_isbn_search_data(self, data):
-        return list(data.values())
-
-    def format_isbn_search_result(self, search_result):
-        # build the remote id from the openlibrary key
-        key = self.books_url + search_result["key"]
-        authors = search_result.get("authors") or [{"name": "Unknown"}]
-        author_names = [author.get("name") for author in authors]
-        return SearchResult(
-            title=search_result.get("title"),
-            key=key,
-            author=", ".join(author_names),
-            connector=self,
-            year=search_result.get("publish_date"),
-        )
+        for search_result in list(data.values()):
+            # build the remote id from the openlibrary key
+            key = self.books_url + search_result["key"]
+            authors = search_result.get("authors") or [{"name": "Unknown"}]
+            author_names = [author.get("name") for author in authors]
+            yield SearchResult(
+                title=search_result.get("title"),
+                key=key,
+                author=", ".join(author_names),
+                connector=self,
+                year=search_result.get("publish_date"),
+            )
 
     def load_edition_data(self, olkey):
         """query openlibrary for editions of a work"""
@@ -224,6 +252,40 @@ def get_languages(language_blob):
     return langs
 
 
+def get_dict_field(blob, field_name):
+    """extract the isni from the remote id data for the author"""
+    if not blob or not isinstance(blob, dict):
+        return None
+    return blob.get(field_name)
+
+
+def get_wikipedia_link(links):
+    """extract wikipedia links"""
+    if not isinstance(links, list):
+        return None
+
+    for link in links:
+        if not isinstance(link, dict):
+            continue
+        if link.get("title") == "wikipedia":
+            return link.get("url")
+    return None
+
+
+def get_inventaire_id(links):
+    """extract and format inventaire ids"""
+    if not isinstance(links, list):
+        return None
+
+    for link in links:
+        if not isinstance(link, dict):
+            continue
+        if link.get("title") == "inventaire.io":
+            iv_link = link.get("url")
+            return iv_link.split("/")[-1]
+    return None
+
+
 def pick_default_edition(options):
     """favor physical copies with covers in english"""
     if not options:

bookwyrm/context_processors.py

@@ -8,8 +8,20 @@ def site_settings(request):  # pylint: disable=unused-argument
     if not request.is_secure():
         request_protocol = "http://"
 
+    site = models.SiteSettings.objects.get()
+    theme = "css/themes/bookwyrm-light.scss"
+    if (
+        hasattr(request, "user")
+        and request.user.is_authenticated
+        and request.user.theme
+    ):
+        theme = request.user.theme.path
+    elif site.default_theme:
+        theme = site.default_theme.path
+
     return {
-        "site": models.SiteSettings.objects.get(),
+        "site": site,
+        "site_theme": theme,
         "active_announcements": models.Announcement.active_announcements(),
         "thumbnail_generation_enabled": settings.ENABLE_THUMBNAIL_GENERATION,
         "media_full_url": settings.MEDIA_FULL_URL,

bookwyrm/emailing.py

@@ -10,14 +10,9 @@ from bookwyrm.settings import DOMAIN
 def email_data():
     """fields every email needs"""
     site = models.SiteSettings.objects.get()
-    if site.logo_small:
-        logo_path = f"/images/{site.logo_small.url}"
-    else:
-        logo_path = "/static/images/logo-small.png"
-
     return {
         "site_name": site.name,
-        "logo": logo_path,
+        "logo": site.logo_small_url,
         "domain": DOMAIN,
         "user": None,
     }
@@ -46,6 +41,20 @@ def password_reset_email(reset_code):
     send_email.delay(reset_code.user.email, *format_email("password_reset", data))
 
 
+def moderation_report_email(report):
+    """a report was created"""
+    data = email_data()
+    data["reporter"] = report.reporter.localname or report.reporter.username
+    data["reportee"] = report.user.localname or report.user.username
+    data["report_link"] = report.remote_id
+
+    for admin in models.User.objects.filter(
+        groups__name__in=["admin", "moderator"]
+    ).distinct():
+        data["user"] = admin.display_name
+        send_email.delay(admin.email, *format_email("moderation_report", data))
+
+
 def format_email(email_name, data):
     """render the email templates"""
     subject = get_template(f"email/{email_name}/subject.html").render(data).strip()
@@ -62,7 +71,7 @@ def format_email(email_name, data):
 def send_email(recipient, subject, html_content, text_content):
     """use a task to send the email"""
     email = EmailMultiAlternatives(
-        subject, text_content, settings.DEFAULT_FROM_EMAIL, [recipient]
+        subject, text_content, settings.EMAIL_SENDER, [recipient]
     )
     email.attach_alternative(html_content, "text/html")
     email.send()

bookwyrm/forms.py

@@ -1,337 +0,0 @@
-""" using django model forms """
-import datetime
-from collections import defaultdict
-
-from django import forms
-from django.forms import ModelForm, PasswordInput, widgets, ChoiceField
-from django.forms.widgets import Textarea
-from django.utils import timezone
-from django.utils.translation import gettext_lazy as _
-
-from bookwyrm import models
-
-
-class CustomForm(ModelForm):
-    """add css classes to the forms"""
-
-    def __init__(self, *args, **kwargs):
-        css_classes = defaultdict(lambda: "")
-        css_classes["text"] = "input"
-        css_classes["password"] = "input"
-        css_classes["email"] = "input"
-        css_classes["number"] = "input"
-        css_classes["checkbox"] = "checkbox"
-        css_classes["textarea"] = "textarea"
-        # pylint: disable=super-with-arguments
-        super(CustomForm, self).__init__(*args, **kwargs)
-        for visible in self.visible_fields():
-            if hasattr(visible.field.widget, "input_type"):
-                input_type = visible.field.widget.input_type
-            if isinstance(visible.field.widget, Textarea):
-                input_type = "textarea"
-                visible.field.widget.attrs["rows"] = 5
-            visible.field.widget.attrs["class"] = css_classes[input_type]
-
-
-# pylint: disable=missing-class-docstring
-class LoginForm(CustomForm):
-    class Meta:
-        model = models.User
-        fields = ["localname", "password"]
-        help_texts = {f: None for f in fields}
-        widgets = {
-            "password": PasswordInput(),
-        }
-
-
-class RegisterForm(CustomForm):
-    class Meta:
-        model = models.User
-        fields = ["localname", "email", "password"]
-        help_texts = {f: None for f in fields}
-        widgets = {"password": PasswordInput()}
-
-
-class RatingForm(CustomForm):
-    class Meta:
-        model = models.ReviewRating
-        fields = ["user", "book", "rating", "privacy"]
-
-
-class ReviewForm(CustomForm):
-    class Meta:
-        model = models.Review
-        fields = [
-            "user",
-            "book",
-            "name",
-            "content",
-            "rating",
-            "content_warning",
-            "sensitive",
-            "privacy",
-        ]
-
-
-class CommentForm(CustomForm):
-    class Meta:
-        model = models.Comment
-        fields = [
-            "user",
-            "book",
-            "content",
-            "content_warning",
-            "sensitive",
-            "privacy",
-            "progress",
-            "progress_mode",
-            "reading_status",
-        ]
-
-
-class QuotationForm(CustomForm):
-    class Meta:
-        model = models.Quotation
-        fields = [
-            "user",
-            "book",
-            "quote",
-            "content",
-            "content_warning",
-            "sensitive",
-            "privacy",
-            "position",
-            "position_mode",
-        ]
-
-
-class ReplyForm(CustomForm):
-    class Meta:
-        model = models.Status
-        fields = [
-            "user",
-            "content",
-            "content_warning",
-            "sensitive",
-            "reply_parent",
-            "privacy",
-        ]
-
-
-class StatusForm(CustomForm):
-    class Meta:
-        model = models.Status
-        fields = ["user", "content", "content_warning", "sensitive", "privacy"]
-
-
-class DirectForm(CustomForm):
-    class Meta:
-        model = models.Status
-        fields = ["user", "content", "content_warning", "sensitive", "privacy"]
-
-
-class EditUserForm(CustomForm):
-    class Meta:
-        model = models.User
-        fields = [
-            "avatar",
-            "name",
-            "email",
-            "summary",
-            "show_goal",
-            "show_suggested_users",
-            "manually_approves_followers",
-            "default_post_privacy",
-            "discoverable",
-            "preferred_timezone",
-            "preferred_language",
-        ]
-        help_texts = {f: None for f in fields}
-
-
-class LimitedEditUserForm(CustomForm):
-    class Meta:
-        model = models.User
-        fields = [
-            "avatar",
-            "name",
-            "summary",
-            "manually_approves_followers",
-            "discoverable",
-        ]
-        help_texts = {f: None for f in fields}
-
-
-class DeleteUserForm(CustomForm):
-    class Meta:
-        model = models.User
-        fields = ["password"]
-
-
-class UserGroupForm(CustomForm):
-    class Meta:
-        model = models.User
-        fields = ["groups"]
-
-
-class CoverForm(CustomForm):
-    class Meta:
-        model = models.Book
-        fields = ["cover"]
-        help_texts = {f: None for f in fields}
-
-
-class EditionForm(CustomForm):
-    class Meta:
-        model = models.Edition
-        exclude = [
-            "remote_id",
-            "origin_id",
-            "created_date",
-            "updated_date",
-            "edition_rank",
-            "authors",
-            "parent_work",
-            "shelves",
-            "connector",
-            "search_vector",
-        ]
-
-
-class AuthorForm(CustomForm):
-    class Meta:
-        model = models.Author
-        exclude = [
-            "remote_id",
-            "origin_id",
-            "created_date",
-            "updated_date",
-            "search_vector",
-        ]
-
-
-class ImportForm(forms.Form):
-    csv_file = forms.FileField()
-
-
-class ExpiryWidget(widgets.Select):
-    def value_from_datadict(self, data, files, name):
-        """human-readable exiration time buckets"""
-        selected_string = super().value_from_datadict(data, files, name)
-
-        if selected_string == "day":
-            interval = datetime.timedelta(days=1)
-        elif selected_string == "week":
-            interval = datetime.timedelta(days=7)
-        elif selected_string == "month":
-            interval = datetime.timedelta(days=31)  # Close enough?
-        elif selected_string == "forever":
-            return None
-        else:
-            return selected_string  # This will raise
-
-        return timezone.now() + interval
-
-
-class InviteRequestForm(CustomForm):
-    def clean(self):
-        """make sure the email isn't in use by a registered user"""
-        cleaned_data = super().clean()
-        email = cleaned_data.get("email")
-        if email and models.User.objects.filter(email=email).exists():
-            self.add_error("email", _("A user with this email already exists."))
-
-    class Meta:
-        model = models.InviteRequest
-        fields = ["email"]
-
-
-class CreateInviteForm(CustomForm):
-    class Meta:
-        model = models.SiteInvite
-        exclude = ["code", "user", "times_used", "invitees"]
-        widgets = {
-            "expiry": ExpiryWidget(
-                choices=[
-                    ("day", _("One Day")),
-                    ("week", _("One Week")),
-                    ("month", _("One Month")),
-                    ("forever", _("Does Not Expire")),
-                ]
-            ),
-            "use_limit": widgets.Select(
-                choices=[(i, _(f"{i} uses")) for i in [1, 5, 10, 25, 50, 100]]
-                + [(None, _("Unlimited"))]
-            ),
-        }
-
-
-class ShelfForm(CustomForm):
-    class Meta:
-        model = models.Shelf
-        fields = ["user", "name", "privacy", "description"]
-
-
-class GoalForm(CustomForm):
-    class Meta:
-        model = models.AnnualGoal
-        fields = ["user", "year", "goal", "privacy"]
-
-
-class SiteForm(CustomForm):
-    class Meta:
-        model = models.SiteSettings
-        exclude = []
-
-
-class AnnouncementForm(CustomForm):
-    class Meta:
-        model = models.Announcement
-        exclude = ["remote_id"]
-
-
-class ListForm(CustomForm):
-    class Meta:
-        model = models.List
-        fields = ["user", "name", "description", "curation", "privacy"]
-
-
-class ReportForm(CustomForm):
-    class Meta:
-        model = models.Report
-        fields = ["user", "reporter", "statuses", "note"]
-
-
-class EmailBlocklistForm(CustomForm):
-    class Meta:
-        model = models.EmailBlocklist
-        fields = ["domain"]
-
-
-class IPBlocklistForm(CustomForm):
-    class Meta:
-        model = models.IPBlocklist
-        fields = ["address"]
-
-
-class ServerForm(CustomForm):
-    class Meta:
-        model = models.FederatedServer
-        exclude = ["remote_id"]
-
-
-class SortListForm(forms.Form):
-    sort_by = ChoiceField(
-        choices=(
-            ("order", _("List Order")),
-            ("title", _("Book Title")),
-            ("rating", _("Rating")),
-        ),
-        label=_("Sort By"),
-    )
-    direction = ChoiceField(
-        choices=(
-            ("ascending", _("Ascending")),
-            ("descending", _("Descending")),
-        ),
-    )

bookwyrm/forms/__init__.py

@@ -0,0 +1,12 @@
+""" make forms available to the app """
+# site admin
+from .admin import *
+from .author import *
+from .books import *
+from .edit_user import *
+from .forms import *
+from .groups import *
+from .landing import *
+from .links import *
+from .lists import *
+from .status import *

bookwyrm/forms/admin.py

@@ -0,0 +1,141 @@
+""" using django model forms """
+import datetime
+
+from django import forms
+from django.forms import widgets
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+from django_celery_beat.models import IntervalSchedule
+
+from bookwyrm import models
+from .custom_form import CustomForm
+
+
+# pylint: disable=missing-class-docstring
+class ExpiryWidget(widgets.Select):
+    def value_from_datadict(self, data, files, name):
+        """human-readable exiration time buckets"""
+        selected_string = super().value_from_datadict(data, files, name)
+
+        if selected_string == "day":
+            interval = datetime.timedelta(days=1)
+        elif selected_string == "week":
+            interval = datetime.timedelta(days=7)
+        elif selected_string == "month":
+            interval = datetime.timedelta(days=31)  # Close enough?
+        elif selected_string == "forever":
+            return None
+        else:
+            return selected_string  # This will raise
+
+        return timezone.now() + interval
+
+
+class CreateInviteForm(CustomForm):
+    class Meta:
+        model = models.SiteInvite
+        exclude = ["code", "user", "times_used", "invitees"]
+        widgets = {
+            "expiry": ExpiryWidget(
+                choices=[
+                    ("day", _("One Day")),
+                    ("week", _("One Week")),
+                    ("month", _("One Month")),
+                    ("forever", _("Does Not Expire")),
+                ]
+            ),
+            "use_limit": widgets.Select(
+                choices=[(i, _(f"{i} uses")) for i in [1, 5, 10, 25, 50, 100]]
+                + [(None, _("Unlimited"))]
+            ),
+        }
+
+
+class SiteForm(CustomForm):
+    class Meta:
+        model = models.SiteSettings
+        exclude = ["admin_code", "install_mode"]
+        widgets = {
+            "instance_short_description": forms.TextInput(
+                attrs={"aria-describedby": "desc_instance_short_description"}
+            ),
+            "require_confirm_email": forms.CheckboxInput(
+                attrs={"aria-describedby": "desc_require_confirm_email"}
+            ),
+            "invite_request_text": forms.Textarea(
+                attrs={"aria-describedby": "desc_invite_request_text"}
+            ),
+        }
+
+
+class ThemeForm(CustomForm):
+    class Meta:
+        model = models.Theme
+        fields = ["name", "path"]
+        widgets = {
+            "name": forms.TextInput(attrs={"aria-describedby": "desc_name"}),
+            "path": forms.TextInput(
+                attrs={
+                    "aria-describedby": "desc_path",
+                    "placeholder": "css/themes/theme-name.scss",
+                }
+            ),
+        }
+
+
+class AnnouncementForm(CustomForm):
+    class Meta:
+        model = models.Announcement
+        exclude = ["remote_id"]
+        widgets = {
+            "preview": forms.TextInput(attrs={"aria-describedby": "desc_preview"}),
+            "content": forms.Textarea(attrs={"aria-describedby": "desc_content"}),
+            "event_date": forms.SelectDateWidget(
+                attrs={"aria-describedby": "desc_event_date"}
+            ),
+            "start_date": forms.SelectDateWidget(
+                attrs={"aria-describedby": "desc_start_date"}
+            ),
+            "end_date": forms.SelectDateWidget(
+                attrs={"aria-describedby": "desc_end_date"}
+            ),
+            "active": forms.CheckboxInput(attrs={"aria-describedby": "desc_active"}),
+        }
+
+
+class EmailBlocklistForm(CustomForm):
+    class Meta:
+        model = models.EmailBlocklist
+        fields = ["domain"]
+        widgets = {
+            "avatar": forms.TextInput(attrs={"aria-describedby": "desc_domain"}),
+        }
+
+
+class IPBlocklistForm(CustomForm):
+    class Meta:
+        model = models.IPBlocklist
+        fields = ["address"]
+
+
+class ServerForm(CustomForm):
+    class Meta:
+        model = models.FederatedServer
+        exclude = ["remote_id"]
+
+
+class AutoModRuleForm(CustomForm):
+    class Meta:
+        model = models.AutoMod
+        fields = ["string_match", "flag_users", "flag_statuses", "created_by"]
+
+
+class IntervalScheduleForm(CustomForm):
+    class Meta:
+        model = IntervalSchedule
+        fields = ["every", "period"]
+
+        widgets = {
+            "every": forms.NumberInput(attrs={"aria-describedby": "desc_every"}),
+            "period": forms.Select(attrs={"aria-describedby": "desc_period"}),
+        }

bookwyrm/forms/author.py

@@ -0,0 +1,47 @@
+""" using django model forms """
+from django import forms
+
+from bookwyrm import models
+from .custom_form import CustomForm
+
+
+# pylint: disable=missing-class-docstring
+class AuthorForm(CustomForm):
+    class Meta:
+        model = models.Author
+        fields = [
+            "last_edited_by",
+            "name",
+            "aliases",
+            "bio",
+            "wikipedia_link",
+            "born",
+            "died",
+            "openlibrary_key",
+            "inventaire_id",
+            "librarything_key",
+            "goodreads_key",
+            "isni",
+        ]
+        widgets = {
+            "name": forms.TextInput(attrs={"aria-describedby": "desc_name"}),
+            "aliases": forms.TextInput(attrs={"aria-describedby": "desc_aliases"}),
+            "bio": forms.Textarea(attrs={"aria-describedby": "desc_bio"}),
+            "wikipedia_link": forms.TextInput(
+                attrs={"aria-describedby": "desc_wikipedia_link"}
+            ),
+            "born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}),
+            "died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}),
+            "oepnlibrary_key": forms.TextInput(
+                attrs={"aria-describedby": "desc_oepnlibrary_key"}
+            ),
+            "inventaire_id": forms.TextInput(
+                attrs={"aria-describedby": "desc_inventaire_id"}
+            ),
+            "librarything_key": forms.TextInput(
+                attrs={"aria-describedby": "desc_librarything_key"}
+            ),
+            "goodreads_key": forms.TextInput(
+                attrs={"aria-describedby": "desc_goodreads_key"}
+            ),
+        }

bookwyrm/forms/books.py

@@ -0,0 +1,104 @@
+""" using django model forms """
+from django import forms
+
+from bookwyrm import models
+from bookwyrm.models.fields import ClearableFileInputWithWarning
+from .custom_form import CustomForm
+from .widgets import ArrayWidget, SelectDateWidget, Select
+
+
+# pylint: disable=missing-class-docstring
+class CoverForm(CustomForm):
+    class Meta:
+        model = models.Book
+        fields = ["cover"]
+        help_texts = {f: None for f in fields}
+
+
+class EditionForm(CustomForm):
+    class Meta:
+        model = models.Edition
+        exclude = [
+            "remote_id",
+            "origin_id",
+            "created_date",
+            "updated_date",
+            "edition_rank",
+            "authors",
+            "parent_work",
+            "shelves",
+            "connector",
+            "search_vector",
+            "links",
+            "file_links",
+        ]
+        widgets = {
+            "title": forms.TextInput(attrs={"aria-describedby": "desc_title"}),
+            "subtitle": forms.TextInput(attrs={"aria-describedby": "desc_subtitle"}),
+            "description": forms.Textarea(
+                attrs={"aria-describedby": "desc_description"}
+            ),
+            "series": forms.TextInput(attrs={"aria-describedby": "desc_series"}),
+            "series_number": forms.TextInput(
+                attrs={"aria-describedby": "desc_series_number"}
+            ),
+            "subjects": ArrayWidget(),
+            "languages": forms.TextInput(
+                attrs={"aria-describedby": "desc_languages_help desc_languages"}
+            ),
+            "publishers": forms.TextInput(
+                attrs={"aria-describedby": "desc_publishers_help desc_publishers"}
+            ),
+            "first_published_date": SelectDateWidget(
+                attrs={"aria-describedby": "desc_first_published_date"}
+            ),
+            "published_date": SelectDateWidget(
+                attrs={"aria-describedby": "desc_published_date"}
+            ),
+            "cover": ClearableFileInputWithWarning(
+                attrs={"aria-describedby": "desc_cover"}
+            ),
+            "physical_format": Select(
+                attrs={"aria-describedby": "desc_physical_format"}
+            ),
+            "physical_format_detail": forms.TextInput(
+                attrs={"aria-describedby": "desc_physical_format_detail"}
+            ),
+            "pages": forms.NumberInput(attrs={"aria-describedby": "desc_pages"}),
+            "isbn_13": forms.TextInput(attrs={"aria-describedby": "desc_isbn_13"}),
+            "isbn_10": forms.TextInput(attrs={"aria-describedby": "desc_isbn_10"}),
+            "openlibrary_key": forms.TextInput(
+                attrs={"aria-describedby": "desc_openlibrary_key"}
+            ),
+            "inventaire_id": forms.TextInput(
+                attrs={"aria-describedby": "desc_inventaire_id"}
+            ),
+            "oclc_number": forms.TextInput(
+                attrs={"aria-describedby": "desc_oclc_number"}
+            ),
+            "ASIN": forms.TextInput(attrs={"aria-describedby": "desc_ASIN"}),
+        }
+
+
+class EditionFromWorkForm(CustomForm):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # make all fields hidden
+        for visible in self.visible_fields():
+            visible.field.widget = forms.HiddenInput()
+
+    class Meta:
+        model = models.Work
+        fields = [
+            "title",
+            "subtitle",
+            "authors",
+            "description",
+            "languages",
+            "series",
+            "series_number",
+            "subjects",
+            "subject_places",
+            "cover",
+            "first_published_date",
+        ]

bookwyrm/forms/custom_form.py

@@ -0,0 +1,26 @@
+""" Overrides django's default form class """
+from collections import defaultdict
+from django.forms import ModelForm
+from django.forms.widgets import Textarea
+
+
+class CustomForm(ModelForm):
+    """add css classes to the forms"""
+
+    def __init__(self, *args, **kwargs):
+        css_classes = defaultdict(lambda: "")
+        css_classes["text"] = "input"
+        css_classes["password"] = "input"
+        css_classes["email"] = "input"
+        css_classes["number"] = "input"
+        css_classes["checkbox"] = "checkbox"
+        css_classes["textarea"] = "textarea"
+        # pylint: disable=super-with-arguments
+        super(CustomForm, self).__init__(*args, **kwargs)
+        for visible in self.visible_fields():
+            if hasattr(visible.field.widget, "input_type"):
+                input_type = visible.field.widget.input_type
+            if isinstance(visible.field.widget, Textarea):
+                input_type = "textarea"
+                visible.field.widget.attrs["rows"] = 5
+            visible.field.widget.attrs["class"] = css_classes[input_type]

bookwyrm/forms/edit_user.py

@@ -0,0 +1,68 @@
+""" using django model forms """
+from django import forms
+
+from bookwyrm import models
+from bookwyrm.models.fields import ClearableFileInputWithWarning
+from .custom_form import CustomForm
+
+
+# pylint: disable=missing-class-docstring
+class EditUserForm(CustomForm):
+    class Meta:
+        model = models.User
+        fields = [
+            "avatar",
+            "name",
+            "email",
+            "summary",
+            "show_goal",
+            "show_suggested_users",
+            "manually_approves_followers",
+            "default_post_privacy",
+            "discoverable",
+            "hide_follows",
+            "preferred_timezone",
+            "preferred_language",
+            "theme",
+        ]
+        help_texts = {f: None for f in fields}
+        widgets = {
+            "avatar": ClearableFileInputWithWarning(
+                attrs={"aria-describedby": "desc_avatar"}
+            ),
+            "name": forms.TextInput(attrs={"aria-describedby": "desc_name"}),
+            "summary": forms.Textarea(attrs={"aria-describedby": "desc_summary"}),
+            "email": forms.EmailInput(attrs={"aria-describedby": "desc_email"}),
+            "discoverable": forms.CheckboxInput(
+                attrs={"aria-describedby": "desc_discoverable"}
+            ),
+        }
+
+
+class LimitedEditUserForm(CustomForm):
+    class Meta:
+        model = models.User
+        fields = [
+            "avatar",
+            "name",
+            "summary",
+            "manually_approves_followers",
+            "discoverable",
+        ]
+        help_texts = {f: None for f in fields}
+        widgets = {
+            "avatar": ClearableFileInputWithWarning(
+                attrs={"aria-describedby": "desc_avatar"}
+            ),
+            "name": forms.TextInput(attrs={"aria-describedby": "desc_name"}),
+            "summary": forms.Textarea(attrs={"aria-describedby": "desc_summary"}),
+            "discoverable": forms.CheckboxInput(
+                attrs={"aria-describedby": "desc_discoverable"}
+            ),
+        }
+
+
+class DeleteUserForm(CustomForm):
+    class Meta:
+        model = models.User
+        fields = ["password"]

bookwyrm/forms/forms.py

@@ -0,0 +1,64 @@
+""" using django model forms """
+from django import forms
+from django.forms import widgets
+from django.utils.translation import gettext_lazy as _
+
+from bookwyrm import models
+from bookwyrm.models.user import FeedFilterChoices
+from .custom_form import CustomForm
+
+
+# pylint: disable=missing-class-docstring
+class FeedStatusTypesForm(CustomForm):
+    class Meta:
+        model = models.User
+        fields = ["feed_status_types"]
+        help_texts = {f: None for f in fields}
+        widgets = {
+            "feed_status_types": widgets.CheckboxSelectMultiple(
+                choices=FeedFilterChoices,
+            ),
+        }
+
+
+class ImportForm(forms.Form):
+    csv_file = forms.FileField()
+
+
+class ShelfForm(CustomForm):
+    class Meta:
+        model = models.Shelf
+        fields = ["user", "name", "privacy", "description"]
+
+
+class GoalForm(CustomForm):
+    class Meta:
+        model = models.AnnualGoal
+        fields = ["user", "year", "goal", "privacy"]
+
+
+class ReportForm(CustomForm):
+    class Meta:
+        model = models.Report
+        fields = ["user", "reporter", "status", "links", "note"]
+
+
+class ReadThroughForm(CustomForm):
+    def clean(self):
+        """don't let readthroughs end before they start"""
+        cleaned_data = super().clean()
+        start_date = cleaned_data.get("start_date")
+        finish_date = cleaned_data.get("finish_date")
+        if start_date and finish_date and start_date > finish_date:
+            self.add_error(
+                "finish_date", _("Reading finish date cannot be before start date.")
+            )
+        stopped_date = cleaned_data.get("stopped_date")
+        if start_date and stopped_date and start_date > stopped_date:
+            self.add_error(
+                "stopped_date", _("Reading stopped date cannot be before start date.")
+            )
+
+    class Meta:
+        model = models.ReadThrough
+        fields = ["user", "book", "start_date", "finish_date", "stopped_date"]

bookwyrm/forms/groups.py

@@ -0,0 +1,16 @@
+""" using django model forms """
+from bookwyrm import models
+from .custom_form import CustomForm
+
+
+# pylint: disable=missing-class-docstring
+class UserGroupForm(CustomForm):
+    class Meta:
+        model = models.User
+        fields = ["groups"]
+
+
+class GroupForm(CustomForm):
+    class Meta:
+        model = models.Group
+        fields = ["user", "privacy", "name", "description"]

bookwyrm/forms/landing.py

@@ -0,0 +1,45 @@
+""" Forms for the landing pages """
+from django.forms import PasswordInput
+from django.utils.translation import gettext_lazy as _
+
+from bookwyrm import models
+from .custom_form import CustomForm
+
+
+# pylint: disable=missing-class-docstring
+class LoginForm(CustomForm):
+    class Meta:
+        model = models.User
+        fields = ["localname", "password"]
+        help_texts = {f: None for f in fields}
+        widgets = {
+            "password": PasswordInput(),
+        }
+
+
+class RegisterForm(CustomForm):
+    class Meta:
+        model = models.User
+        fields = ["localname", "email", "password"]
+        help_texts = {f: None for f in fields}
+        widgets = {"password": PasswordInput()}
+
+    def clean(self):
+        """Check if the username is taken"""
+        cleaned_data = super().clean()
+        localname = cleaned_data.get("localname").strip()
+        if models.User.objects.filter(localname=localname).first():
+            self.add_error("localname", _("User with this username already exists"))
+
+
+class InviteRequestForm(CustomForm):
+    def clean(self):
+        """make sure the email isn't in use by a registered user"""
+        cleaned_data = super().clean()
+        email = cleaned_data.get("email")
+        if email and models.User.objects.filter(email=email).exists():
+            self.add_error("email", _("A user with this email already exists."))
+
+    class Meta:
+        model = models.InviteRequest
+        fields = ["email", "answer"]

bookwyrm/forms/links.py

@@ -0,0 +1,48 @@
+""" using django model forms """
+from urllib.parse import urlparse
+
+from django.utils.translation import gettext_lazy as _
+
+from bookwyrm import models
+from .custom_form import CustomForm
+
+
+# pylint: disable=missing-class-docstring
+class LinkDomainForm(CustomForm):
+    class Meta:
+        model = models.LinkDomain
+        fields = ["name"]
+
+
+class FileLinkForm(CustomForm):
+    class Meta:
+        model = models.FileLink
+        fields = ["url", "filetype", "availability", "book", "added_by"]
+
+    def clean(self):
+        """make sure the domain isn't blocked or pending"""
+        cleaned_data = super().clean()
+        url = cleaned_data.get("url")
+        filetype = cleaned_data.get("filetype")
+        book = cleaned_data.get("book")
+        domain = urlparse(url).netloc
+        if models.LinkDomain.objects.filter(domain=domain).exists():
+            status = models.LinkDomain.objects.get(domain=domain).status
+            if status == "blocked":
+                # pylint: disable=line-too-long
+                self.add_error(
+                    "url",
+                    _(
+                        "This domain is blocked. Please contact your administrator if you think this is an error."
+                    ),
+                )
+            elif models.FileLink.objects.filter(
+                url=url, book=book, filetype=filetype
+            ).exists():
+                # pylint: disable=line-too-long
+                self.add_error(
+                    "url",
+                    _(
+                        "This link with file type has already been added for this book. If it is not visible, the domain is still pending."
+                    ),
+                )

bookwyrm/forms/lists.py

@@ -0,0 +1,37 @@
+""" using django model forms """
+from django import forms
+from django.forms import ChoiceField
+from django.utils.translation import gettext_lazy as _
+
+from bookwyrm import models
+from .custom_form import CustomForm
+
+
+# pylint: disable=missing-class-docstring
+class ListForm(CustomForm):
+    class Meta:
+        model = models.List
+        fields = ["user", "name", "description", "curation", "privacy", "group"]
+
+
+class ListItemForm(CustomForm):
+    class Meta:
+        model = models.ListItem
+        fields = ["user", "book", "book_list", "notes"]
+
+
+class SortListForm(forms.Form):
+    sort_by = ChoiceField(
+        choices=(
+            ("order", _("List Order")),
+            ("title", _("Book Title")),
+            ("rating", _("Rating")),
+        ),
+        label=_("Sort By"),
+    )
+    direction = ChoiceField(
+        choices=(
+            ("ascending", _("Ascending")),
+            ("descending", _("Descending")),
+        ),
+    )

bookwyrm/forms/status.py

@@ -0,0 +1,82 @@
+""" using django model forms """
+from bookwyrm import models
+from .custom_form import CustomForm
+
+
+# pylint: disable=missing-class-docstring
+class RatingForm(CustomForm):
+    class Meta:
+        model = models.ReviewRating
+        fields = ["user", "book", "rating", "privacy"]
+
+
+class ReviewForm(CustomForm):
+    class Meta:
+        model = models.Review
+        fields = [
+            "user",
+            "book",
+            "name",
+            "content",
+            "rating",
+            "content_warning",
+            "sensitive",
+            "privacy",
+        ]
+
+
+class CommentForm(CustomForm):
+    class Meta:
+        model = models.Comment
+        fields = [
+            "user",
+            "book",
+            "content",
+            "content_warning",
+            "sensitive",
+            "privacy",
+            "progress",
+            "progress_mode",
+            "reading_status",
+        ]
+
+
+class QuotationForm(CustomForm):
+    class Meta:
+        model = models.Quotation
+        fields = [
+            "user",
+            "book",
+            "quote",
+            "content",
+            "content_warning",
+            "sensitive",
+            "privacy",
+            "position",
+            "position_mode",
+        ]
+
+
+class ReplyForm(CustomForm):
+    class Meta:
+        model = models.Status
+        fields = [
+            "user",
+            "content",
+            "content_warning",
+            "sensitive",
+            "reply_parent",
+            "privacy",
+        ]
+
+
+class StatusForm(CustomForm):
+    class Meta:
+        model = models.Status
+        fields = ["user", "content", "content_warning", "sensitive", "privacy"]
+
+
+class DirectForm(CustomForm):
+    class Meta:
+        model = models.Status
+        fields = ["user", "content", "content_warning", "sensitive", "privacy"]

bookwyrm/forms/widgets.py

@@ -0,0 +1,70 @@
+""" using django model forms """
+from django import forms
+
+
+class ArrayWidget(forms.widgets.TextInput):
+    """Inputs for postgres array fields"""
+
+    # pylint: disable=unused-argument
+    # pylint: disable=no-self-use
+    def value_from_datadict(self, data, files, name):
+        """get all values for this name"""
+        return [i for i in data.getlist(name) if i]
+
+
+class Select(forms.Select):
+    """custom template for select widget"""
+
+    template_name = "widgets/select.html"
+
+
+class SelectDateWidget(forms.SelectDateWidget):
+    """
+    A widget that splits date input into two <select> boxes and a numerical year.
+    """
+
+    template_name = "widgets/addon_multiwidget.html"
+    select_widget = Select
+
+    def get_context(self, name, value, attrs):
+        """sets individual widgets"""
+        context = super().get_context(name, value, attrs)
+        date_context = {}
+        year_name = self.year_field % name
+        date_context["year"] = forms.NumberInput().get_context(
+            name=year_name,
+            value=context["widget"]["value"]["year"],
+            attrs={
+                **context["widget"]["attrs"],
+                "id": f"id_{year_name}",
+                "class": "input",
+            },
+        )
+        month_choices = list(self.months.items())
+        if not self.is_required:
+            month_choices.insert(0, self.month_none_value)
+        month_name = self.month_field % name
+        date_context["month"] = self.select_widget(
+            attrs, choices=month_choices
+        ).get_context(
+            name=month_name,
+            value=context["widget"]["value"]["month"],
+            attrs={**context["widget"]["attrs"], "id": f"id_{month_name}"},
+        )
+        day_choices = [(i, i) for i in range(1, 32)]
+        if not self.is_required:
+            day_choices.insert(0, self.day_none_value)
+        day_name = self.day_field % name
+        date_context["day"] = self.select_widget(
+            attrs,
+            choices=day_choices,
+        ).get_context(
+            name=day_name,
+            value=context["widget"]["value"]["day"],
+            attrs={**context["widget"]["attrs"], "id": f"id_{day_name}"},
+        )
+        subwidgets = []
+        for field in self._parse_date_fmt():
+            subwidgets.append(date_context[field]["widget"])
+        context["widget"]["subwidgets"] = subwidgets
+        return context

bookwyrm/importers/__init__.py

@@ -1,6 +1,8 @@
 """ import classes """
 
 from .importer import Importer
+from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter
+from .openlibrary_import import OpenLibraryImporter
 from .storygraph_import import StorygraphImporter

bookwyrm/importers/calibre_import.py

@@ -0,0 +1,28 @@
+""" handle reading a csv from calibre """
+from bookwyrm.models import Shelf
+
+from . import Importer
+
+
+class CalibreImporter(Importer):
+    """csv downloads from Calibre"""
+
+    service = "Calibre"
+
+    def __init__(self, *args, **kwargs):
+        # Add timestamp to row_mappings_guesses for date_added to avoid
+        # integrity error
+        row_mappings_guesses = []
+
+        for field, mapping in self.row_mappings_guesses:
+            if field in ("date_added",):
+                row_mappings_guesses.append((field, mapping + ["timestamp"]))
+            else:
+                row_mappings_guesses.append((field, mapping))
+
+        self.row_mappings_guesses = row_mappings_guesses
+        super().__init__(*args, **kwargs)
+
+    def get_shelf(self, normalized_row):
+        # Calibre export does not indicate which shelf to use. Go with a default one for now
+        return Shelf.TO_READ

bookwyrm/importers/goodreads_import.py

@@ -3,14 +3,7 @@ from . import Importer
 
 
 class GoodreadsImporter(Importer):
-    """GoodReads is the default importer, thus Importer follows its structure.
+    """Goodreads is the default importer, thus Importer follows its structure.
     For a more complete example of overriding see librarything_import.py"""
 
-    service = "GoodReads"
-
-    def parse_fields(self, entry):
-        """handle the specific fields in goodreads csvs"""
-        entry.update({"import_source": self.service})
-        # add missing 'Date Started' field
-        entry.update({"Date Started": None})
-        return entry
+    service = "Goodreads"

bookwyrm/importers/importer.py

@@ -1,4 +1,4 @@
-""" handle reading a csv from an external service, defaults are from GoodReads """
+""" handle reading a csv from an external service, defaults are from Goodreads """
 import csv
 import logging
 
@@ -7,7 +7,7 @@ from django.utils.translation import gettext_lazy as _
 
 from bookwyrm import models
 from bookwyrm.models import ImportJob, ImportItem
-from bookwyrm.tasks import app
+from bookwyrm.tasks import app, LOW
 
 logger = logging.getLogger(__name__)
 
@@ -15,33 +15,93 @@ logger = logging.getLogger(__name__)
 class Importer:
     """Generic class for csv data import from an outside service"""
 
-    service = "Unknown"
+    service = "Import"
     delimiter = ","
     encoding = "UTF-8"
-    mandatory_fields = ["Title", "Author"]
+
+    # these are from Goodreads
+    row_mappings_guesses = [
+        ("id", ["id", "book id"]),
+        ("title", ["title"]),
+        ("authors", ["author", "authors", "primary author"]),
+        ("isbn_10", ["isbn10", "isbn"]),
+        ("isbn_13", ["isbn13", "isbn", "isbns"]),
+        ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]),
+        ("review_name", ["review name"]),
+        ("review_body", ["my review", "review"]),
+        ("rating", ["my rating", "rating", "star rating"]),
+        ("date_added", ["date added", "entry date", "added"]),
+        ("date_started", ["date started", "started"]),
+        ("date_finished", ["date finished", "last date read", "date read", "finished"]),
+    ]
+    date_fields = ["date_added", "date_started", "date_finished"]
+    shelf_mapping_guesses = {
+        "to-read": ["to-read", "want to read"],
+        "read": ["read", "already read"],
+        "reading": ["currently-reading", "reading", "currently reading"],
+    }
 
     def create_job(self, user, csv_file, include_reviews, privacy):
         """check over a csv and creates a database entry for the job"""
+        csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
+        rows = enumerate(list(csv_reader))
         job = ImportJob.objects.create(
-            user=user, include_reviews=include_reviews, privacy=privacy
+            user=user,
+            include_reviews=include_reviews,
+            privacy=privacy,
+            mappings=self.create_row_mappings(csv_reader.fieldnames),
+            source=self.service,
         )
-        for index, entry in enumerate(
-            list(csv.DictReader(csv_file, delimiter=self.delimiter))
-        ):
-            if not all(x in entry for x in self.mandatory_fields):
-                raise ValueError("Author and title must be in data.")
-            entry = self.parse_fields(entry)
-            self.save_item(job, index, entry)
+
+        for index, entry in rows:
+            self.create_item(job, index, entry)
         return job
 
-    def save_item(self, job, index, data):  # pylint: disable=no-self-use
-        """creates and saves an import item"""
-        ImportItem(job=job, index=index, data=data).save()
+    def update_legacy_job(self, job):
+        """patch up a job that was in the old format"""
+        items = job.items
+        headers = list(items.first().data.keys())
+        job.mappings = self.create_row_mappings(headers)
+        job.updated_date = timezone.now()
+        job.save()
 
-    def parse_fields(self, entry):
-        """updates csv data with additional info"""
-        entry.update({"import_source": self.service})
-        return entry
+        for item in items.all():
+            normalized = self.normalize_row(item.data, job.mappings)
+            normalized["shelf"] = self.get_shelf(normalized)
+            item.normalized_data = normalized
+            item.save()
+
+    def create_row_mappings(self, headers):
+        """guess what the headers mean"""
+        mappings = {}
+        for (key, guesses) in self.row_mappings_guesses:
+            value = [h for h in headers if h.lower() in guesses]
+            value = value[0] if len(value) else None
+            if value:
+                headers.remove(value)
+            mappings[key] = value
+        return mappings
+
+    def create_item(self, job, index, data):
+        """creates and saves an import item"""
+        normalized = self.normalize_row(data, job.mappings)
+        normalized["shelf"] = self.get_shelf(normalized)
+        ImportItem(job=job, index=index, data=data, normalized_data=normalized).save()
+
+    def get_shelf(self, normalized_row):
+        """determine which shelf to use"""
+        shelf_name = normalized_row.get("shelf")
+        if not shelf_name:
+            return None
+        shelf_name = shelf_name.lower()
+        shelf = [
+            s for (s, gs) in self.shelf_mapping_guesses.items() if shelf_name in gs
+        ]
+        return shelf[0] if shelf else None
+
+    def normalize_row(self, entry, mappings):  # pylint: disable=no-self-use
+        """use the dataclass to create the formatted row of data"""
+        return {k: entry.get(v) for k, v in mappings.items()}
 
     def create_retry_job(self, user, original_job, items):
         """retry items that didn't import"""
@@ -49,55 +109,66 @@ class Importer:
             user=user,
             include_reviews=original_job.include_reviews,
             privacy=original_job.privacy,
+            source=original_job.source,
+            # TODO: allow users to adjust mappings
+            mappings=original_job.mappings,
             retry=True,
         )
         for item in items:
-            self.save_item(job, item.index, item.data)
+            # this will re-normalize the raw data
+            self.create_item(job, item.index, item.data)
         return job
 
-    def start_import(self, job):
+    def start_import(self, job):  # pylint: disable=no-self-use
         """initalizes a csv import job"""
-        result = import_data.delay(self.service, job.id)
+        result = start_import_task.delay(job.id)
         job.task_id = result.id
         job.save()
 
 
 @app.task(queue="low_priority")
-def import_data(source, job_id):
-    """does the actual lookup work in a celery task"""
+def start_import_task(job_id):
+    """trigger the child tasks for each row"""
     job = ImportJob.objects.get(id=job_id)
+    # these are sub-tasks so that one big task doesn't use up all the memory in celery
+    for item in job.items.values_list("id", flat=True).all():
+        import_item_task.delay(item)
+
+
+@app.task(queue="low_priority")
+def import_item_task(item_id):
+    """resolve a row into a book"""
+    item = models.ImportItem.objects.get(id=item_id)
     try:
-        for item in job.items.all():
-            try:
-                item.resolve()
-            except Exception as err:  # pylint: disable=broad-except
-                logger.exception(err)
-                item.fail_reason = _("Error loading book")
-                item.save()
-                continue
+        item.resolve()
+    except Exception as err:  # pylint: disable=broad-except
+        item.fail_reason = _("Error loading book")
+        item.save()
+        item.update_job()
+        raise err
 
-            if item.book or item.book_guess:
-                item.save()
+    if item.book:
+        # shelves book and handles reviews
+        handle_imported_book(item)
+    else:
+        item.fail_reason = _("Could not find a match for book")
 
-            if item.book:
-                # shelves book and handles reviews
-                handle_imported_book(
-                    source, job.user, item, job.include_reviews, job.privacy
-                )
-            else:
-                item.fail_reason = _("Could not find a match for book")
-                item.save()
-    finally:
-        job.complete = True
-        job.save()
+    item.save()
+    item.update_job()
 
 
-def handle_imported_book(source, user, item, include_reviews, privacy):
+def handle_imported_book(item):
     """process a csv and then post about it"""
+    job = item.job
+    user = job.user
     if isinstance(item.book, models.Work):
         item.book = item.book.default_edition
     if not item.book:
+        item.fail_reason = _("Error loading book")
+        item.save()
         return
+    if not isinstance(item.book, models.Edition):
+        item.book = item.book.edition
 
     existing_shelf = models.ShelfBook.objects.filter(book=item.book, user=user).exists()
 
@@ -105,9 +176,9 @@ def handle_imported_book(source, user, item, include_reviews, privacy):
     if item.shelf and not existing_shelf:
         desired_shelf = models.Shelf.objects.get(identifier=item.shelf, user=user)
         shelved_date = item.date_added or timezone.now()
-        models.ShelfBook.objects.create(
+        models.ShelfBook(
             book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date
-        )
+        ).save(priority=LOW)
 
     for read in item.reads:
         # check for an existing readthrough with the same dates
@@ -122,35 +193,52 @@ def handle_imported_book(source, user, item, include_reviews, privacy):
         read.user = user
         read.save()
 
-    if include_reviews and (item.rating or item.review):
+    if job.include_reviews and (item.rating or item.review) and not item.linked_review:
         # we don't know the publication date of the review,
         # but "now" is a bad guess
         published_date_guess = item.date_read or item.date_added
         if item.review:
             # pylint: disable=consider-using-f-string
-            review_title = (
-                "Review of {!r} on {!r}".format(
-                    item.book.title,
-                    source,
-                )
-                if item.review
-                else ""
+            review_title = "Review of {!r} on {!r}".format(
+                item.book.title,
+                job.source,
             )
-            models.Review.objects.create(
+            review = models.Review.objects.filter(
                 user=user,
                 book=item.book,
                 name=review_title,
-                content=item.review,
                 rating=item.rating,
                 published_date=published_date_guess,
-                privacy=privacy,
-            )
+            ).first()
+            if not review:
+                review = models.Review(
+                    user=user,
+                    book=item.book,
+                    name=review_title,
+                    content=item.review,
+                    rating=item.rating,
+                    published_date=published_date_guess,
+                    privacy=job.privacy,
+                )
+                review.save(software="bookwyrm", priority=LOW)
         else:
             # just a rating
-            models.ReviewRating.objects.create(
+            review = models.ReviewRating.objects.filter(
                 user=user,
                 book=item.book,
-                rating=item.rating,
                 published_date=published_date_guess,
-                privacy=privacy,
-            )
+                rating=item.rating,
+            ).first()
+            if not review:
+                review = models.ReviewRating(
+                    user=user,
+                    book=item.book,
+                    rating=item.rating,
+                    published_date=published_date_guess,
+                    privacy=job.privacy,
+                )
+                review.save(software="bookwyrm", priority=LOW)
+
+        # only broadcast this review to other bookwyrm instances
+        item.linked_review = review
+    item.save()

bookwyrm/importers/librarything_import.py

@@ -1,6 +1,7 @@
-""" handle reading a csv from librarything """
+""" handle reading a tsv from librarything """
 import re
-import math
+
+from bookwyrm.models import Shelf
 
 from . import Importer
 
@@ -11,32 +12,19 @@ class LibrarythingImporter(Importer):
     service = "LibraryThing"
     delimiter = "\t"
     encoding = "ISO-8859-1"
-    # mandatory_fields : fields matching the book title and author
-    mandatory_fields = ["Title", "Primary Author"]
 
-    def parse_fields(self, entry):
-        """custom parsing for librarything"""
-        data = {}
-        data["import_source"] = self.service
-        data["Book Id"] = entry["Book Id"]
-        data["Title"] = entry["Title"]
-        data["Author"] = entry["Primary Author"]
-        data["ISBN13"] = entry["ISBN"]
-        data["My Review"] = entry["Review"]
-        if entry["Rating"]:
-            data["My Rating"] = math.ceil(float(entry["Rating"]))
-        else:
-            data["My Rating"] = ""
-        data["Date Added"] = re.sub(r"\[|\]", "", entry["Entry Date"])
-        data["Date Started"] = re.sub(r"\[|\]", "", entry["Date Started"])
-        data["Date Read"] = re.sub(r"\[|\]", "", entry["Date Read"])
+    def normalize_row(self, entry, mappings):  # pylint: disable=no-self-use
+        """use the dataclass to create the formatted row of data"""
+        remove_brackets = lambda v: re.sub(r"\[|\]", "", v) if v else None
+        normalized = {k: remove_brackets(entry.get(v)) for k, v in mappings.items()}
+        isbn_13 = normalized.get("isbn_13")
+        isbn_13 = isbn_13.split(", ") if isbn_13 else []
+        normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 0 else None
+        return normalized
 
-        data["Exclusive Shelf"] = None
-        if data["Date Read"]:
-            data["Exclusive Shelf"] = "read"
-        elif data["Date Started"]:
-            data["Exclusive Shelf"] = "reading"
-        else:
-            data["Exclusive Shelf"] = "to-read"
-
-        return data
+    def get_shelf(self, normalized_row):
+        if normalized_row["date_finished"]:
+            return Shelf.READ_FINISHED
+        if normalized_row["date_started"]:
+            return Shelf.READING
+        return Shelf.TO_READ

bookwyrm/importers/openlibrary_import.py

@@ -0,0 +1,13 @@
+""" handle reading a csv from openlibrary"""
+from . import Importer
+
+
+class OpenLibraryImporter(Importer):
+    """csv downloads from OpenLibrary"""
+
+    service = "OpenLibrary"
+
+    def __init__(self, *args, **kwargs):
+        self.row_mappings_guesses.append(("openlibrary_key", ["edition id"]))
+        self.row_mappings_guesses.append(("openlibrary_work_key", ["work id"]))
+        super().__init__(*args, **kwargs)

bookwyrm/importers/storygraph_import.py

@@ -1,34 +1,8 @@
-""" handle reading a csv from librarything """
-import re
-import math
-
+""" handle reading a csv from storygraph"""
 from . import Importer
 
 
 class StorygraphImporter(Importer):
-    """csv downloads from librarything"""
+    """csv downloads from Storygraph"""
 
     service = "Storygraph"
-    # mandatory_fields : fields matching the book title and author
-    mandatory_fields = ["Title"]
-
-    def parse_fields(self, entry):
-        """custom parsing for storygraph"""
-        data = {}
-        data["import_source"] = self.service
-        data["Title"] = entry["Title"]
-        data["Author"] = entry["Authors"] if "Authors" in entry else entry["Author"]
-        data["ISBN13"] = entry["ISBN"]
-        data["My Review"] = entry["Review"]
-        if entry["Star Rating"]:
-            data["My Rating"] = math.ceil(float(entry["Star Rating"]))
-        else:
-            data["My Rating"] = ""
-
-        data["Date Added"] = re.sub(r"[/]", "-", entry["Date Added"])
-        data["Date Read"] = re.sub(r"[/]", "-", entry["Last Date Read"])
-
-        data["Exclusive Shelf"] = (
-            {"read": "read", "currently-reading": "reading", "to-read": "to-read"}
-        ).get(entry["Read Status"], None)
-        return data

bookwyrm/lists_stream.py

@@ -0,0 +1,251 @@
+""" access the list streams stored in redis """
+from django.dispatch import receiver
+from django.db import transaction
+from django.db.models import signals, Count, Q
+
+from bookwyrm import models
+from bookwyrm.redis_store import RedisStore
+from bookwyrm.tasks import app, MEDIUM, HIGH
+
+
+class ListsStream(RedisStore):
+    """all the lists you can see"""
+
+    def stream_id(self, user):  # pylint: disable=no-self-use
+        """the redis key for this user's instance of this stream"""
+        if isinstance(user, int):
+            # allows the function to take an int or an obj
+            return f"{user}-lists"
+        return f"{user.id}-lists"
+
+    def get_rank(self, obj):  # pylint: disable=no-self-use
+        """lists are sorted by updated date"""
+        return obj.updated_date.timestamp()
+
+    def add_list(self, book_list):
+        """add a list to users' feeds"""
+        # the pipeline contains all the add-to-stream activities
+        self.add_object_to_related_stores(book_list)
+
+    def add_user_lists(self, viewer, user):
+        """add a user's lists to another user's feed"""
+        # only add the lists that the viewer should be able to see
+        lists = models.List.privacy_filter(viewer).filter(user=user)
+        self.bulk_add_objects_to_store(lists, self.stream_id(viewer))
+
+    def remove_user_lists(self, viewer, user, exclude_privacy=None):
+        """remove a user's list from another user's feed"""
+        # remove all so that followers only lists are removed
+        lists = user.list_set
+        if exclude_privacy:
+            lists = lists.exclude(privacy=exclude_privacy)
+        self.bulk_remove_objects_from_store(lists.all(), self.stream_id(viewer))
+
+    def get_list_stream(self, user):
+        """load the lists to be displayed"""
+        lists = self.get_store(self.stream_id(user))
+        return (
+            models.List.objects.filter(id__in=lists)
+            .annotate(item_count=Count("listitem", filter=Q(listitem__approved=True)))
+            # hide lists with no approved books
+            .filter(item_count__gt=0)
+            .select_related("user")
+            .prefetch_related("listitem_set")
+            .order_by("-updated_date")
+            .distinct()
+        )
+
+    def populate_lists(self, user):
+        """go from zero to a timeline"""
+        self.populate_store(self.stream_id(user))
+
+    def get_audience(self, book_list):  # pylint: disable=no-self-use
+        """given a list, what users should see it"""
+        # everybody who could plausibly see this list
+        audience = models.User.objects.filter(
+            is_active=True,
+            local=True,  # we only create feeds for users of this instance
+        ).exclude(  # not blocked
+            Q(id__in=book_list.user.blocks.all()) | Q(blocks=book_list.user)
+        )
+
+        group = book_list.group
+        # only visible to the poster and mentioned users
+        if book_list.privacy == "direct":
+            if group:
+                audience = audience.filter(
+                    Q(id=book_list.user.id)  # if the user is the post's author
+                    | ~Q(groups=group.memberships)  # if the user is in the group
+                )
+            else:
+                audience = audience.filter(
+                    Q(id=book_list.user.id)  # if the user is the post's author
+                )
+        # only visible to the poster's followers and tagged users
+        elif book_list.privacy == "followers":
+            if group:
+                audience = audience.filter(
+                    Q(id=book_list.user.id)  # if the user is the list's owner
+                    | Q(following=book_list.user)  # if the user is following the pwmer
+                    # if a user is in the group
+                    | Q(memberships__group__id=book_list.group.id)
+                )
+            else:
+                audience = audience.filter(
+                    Q(id=book_list.user.id)  # if the user is the list's owner
+                    | Q(following=book_list.user)  # if the user is following the pwmer
+                )
+        return audience.distinct()
+
+    def get_stores_for_object(self, obj):
+        return [self.stream_id(u) for u in self.get_audience(obj)]
+
+    def get_lists_for_user(self, user):  # pylint: disable=no-self-use
+        """given a user, what lists should they see on this stream"""
+        return models.List.privacy_filter(
+            user,
+            privacy_levels=["public", "followers"],
+        )
+
+    def get_objects_for_store(self, store):
+        user = models.User.objects.get(id=store.split("-")[0])
+        return self.get_lists_for_user(user)
+
+
+@receiver(signals.post_save, sender=models.List)
+# pylint: disable=unused-argument
+def add_list_on_create(sender, instance, created, *args, **kwargs):
+    """add newly created lists streamsstreams"""
+    if not created:
+        return
+    # when creating new things, gotta wait on the transaction
+    transaction.on_commit(lambda: add_list_on_create_command(instance.id))
+
+
+@receiver(signals.post_delete, sender=models.List)
+# pylint: disable=unused-argument
+def remove_list_on_delete(sender, instance, *args, **kwargs):
+    """remove deleted lists to streams"""
+    remove_list_task.delay(instance.id)
+
+
+def add_list_on_create_command(instance_id):
+    """runs this code only after the database commit completes"""
+    add_list_task.delay(instance_id)
+
+
+@receiver(signals.post_save, sender=models.UserFollows)
+# pylint: disable=unused-argument
+def add_lists_on_follow(sender, instance, created, *args, **kwargs):
+    """add a newly followed user's lists to feeds"""
+    if not created or not instance.user_subject.local:
+        return
+    add_user_lists_task.delay(instance.user_subject.id, instance.user_object.id)
+
+
+@receiver(signals.post_delete, sender=models.UserFollows)
+# pylint: disable=unused-argument
+def remove_lists_on_unfollow(sender, instance, *args, **kwargs):
+    """remove lists from a feed on unfollow"""
+    if not instance.user_subject.local:
+        return
+    # remove all but public lists
+    remove_user_lists_task.delay(
+        instance.user_subject.id, instance.user_object.id, exclude_privacy="public"
+    )
+
+
+@receiver(signals.post_save, sender=models.UserBlocks)
+# pylint: disable=unused-argument
+def remove_lists_on_block(sender, instance, *args, **kwargs):
+    """remove lists from all feeds on block"""
+    # blocks apply ot all feeds
+    if instance.user_subject.local:
+        remove_user_lists_task.delay(instance.user_subject.id, instance.user_object.id)
+
+    # and in both directions
+    if instance.user_object.local:
+        remove_user_lists_task.delay(instance.user_object.id, instance.user_subject.id)
+
+
+@receiver(signals.post_delete, sender=models.UserBlocks)
+# pylint: disable=unused-argument
+def add_lists_on_unblock(sender, instance, *args, **kwargs):
+    """add lists back to all feeds on unblock"""
+    # make sure there isn't a block in the other direction
+    if models.UserBlocks.objects.filter(
+        user_subject=instance.user_object,
+        user_object=instance.user_subject,
+    ).exists():
+        return
+
+    # add lists back to streams with lists from anyone
+    if instance.user_subject.local:
+        add_user_lists_task.delay(
+            instance.user_subject.id,
+            instance.user_object.id,
+        )
+
+    # add lists back to streams with lists from anyone
+    if instance.user_object.local:
+        add_user_lists_task.delay(
+            instance.user_object.id,
+            instance.user_subject.id,
+        )
+
+
+@receiver(signals.post_save, sender=models.User)
+# pylint: disable=unused-argument
+def populate_lists_on_account_create(sender, instance, created, *args, **kwargs):
+    """build a user's feeds when they join"""
+    if not created or not instance.local:
+        return
+    transaction.on_commit(lambda: add_list_on_account_create_command(instance.id))
+
+
+def add_list_on_account_create_command(user_id):
+    """wait for the transaction to complete"""
+    populate_lists_task.delay(user_id)
+
+
+# ---- TASKS
+@app.task(queue=MEDIUM)
+def populate_lists_task(user_id):
+    """background task for populating an empty list stream"""
+    user = models.User.objects.get(id=user_id)
+    ListsStream().populate_lists(user)
+
+
+@app.task(queue=MEDIUM)
+def remove_list_task(list_id):
+    """remove a list from any stream it might be in"""
+    stores = models.User.objects.filter(local=True, is_active=True).values_list(
+        "id", flat=True
+    )
+
+    # delete for every store
+    stores = [ListsStream().stream_id(idx) for idx in stores]
+    ListsStream().remove_object_from_related_stores(list_id, stores=stores)
+
+
+@app.task(queue=HIGH)
+def add_list_task(list_id):
+    """add a list to any stream it should be in"""
+    book_list = models.List.objects.get(id=list_id)
+    ListsStream().add_list(book_list)
+
+
+@app.task(queue=MEDIUM)
+def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
+    """remove all lists by a user from a viewer's stream"""
+    viewer = models.User.objects.get(id=viewer_id)
+    user = models.User.objects.get(id=user_id)
+    ListsStream().remove_user_lists(viewer, user, exclude_privacy=exclude_privacy)
+
+
+@app.task(queue=MEDIUM)
+def add_user_lists_task(viewer_id, user_id):
+    """add all lists by a user to a viewer's stream"""
+    viewer = models.User.objects.get(id=viewer_id)
+    user = models.User.objects.get(id=user_id)
+    ListsStream().add_user_lists(viewer, user)

bookwyrm/management/commands/admin_code.py

@@ -0,0 +1,23 @@
+""" Get your admin code to allow install """
+from django.core.management.base import BaseCommand
+
+from bookwyrm import models
+
+
+def get_admin_code():
+    """get that code"""
+    return models.SiteSettings.objects.get().admin_code
+
+
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Gets admin code for configuring BookWyrm"
+
+    # pylint: disable=unused-argument
+    def handle(self, *args, **options):
+        """execute init"""
+        self.stdout.write("*******************************************")
+        self.stdout.write("Use this code to create your admin account:")
+        self.stdout.write(get_admin_code())
+        self.stdout.write("*******************************************")

bookwyrm/management/commands/erase_streams.py

@@ -5,7 +5,10 @@ import redis
 from bookwyrm import settings
 
 r = redis.Redis(
-    host=settings.REDIS_ACTIVITY_HOST, port=settings.REDIS_ACTIVITY_PORT, db=0
+    host=settings.REDIS_ACTIVITY_HOST,
+    port=settings.REDIS_ACTIVITY_PORT,
+    password=settings.REDIS_ACTIVITY_PASSWORD,
+    db=settings.REDIS_ACTIVITY_DB_INDEX,
 )
 
 

bookwyrm/management/commands/generate_preview_images.py

@@ -10,7 +10,9 @@ class Command(BaseCommand):
 
     help = "Generate preview images"
 
+    # pylint: disable=no-self-use
     def add_arguments(self, parser):
+        """options for how the command is run"""
         parser.add_argument(
             "--all",
             "-a",
@@ -38,6 +40,7 @@ class Command(BaseCommand):
         preview_images.generate_site_preview_image_task.delay()
         self.stdout.write(" OK 🖼")
 
+        # pylint: disable=consider-using-f-string
         if options["all"]:
             # Users
             users = models.User.objects.filter(
@@ -53,12 +56,17 @@ class Command(BaseCommand):
             self.stdout.write(" OK 🖼")
 
             # Books
-            books = models.Book.objects.select_subclasses().filter()
-            self.stdout.write(
-                "   → Book preview images ({}): ".format(len(books)), ending=""
+            book_ids = (
+                models.Book.objects.select_subclasses()
+                .filter()
+                .values_list("id", flat=True)
             )
-            for book in books:
-                preview_images.generate_edition_preview_image_task.delay(book.id)
+
+            self.stdout.write(
+                "   → Book preview images ({}): ".format(len(book_ids)), ending=""
+            )
+            for book_id in book_ids:
+                preview_images.generate_edition_preview_image_task.delay(book_id)
                 self.stdout.write(".", ending="")
             self.stdout.write(" OK 🖼")
 

bookwyrm/management/commands/initdb.py

@@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand
 from django.contrib.auth.models import Group, Permission
 from django.contrib.contenttypes.models import ContentType
 
-from bookwyrm.models import Connector, FederatedServer, SiteSettings, User
+from bookwyrm import models
 
 
 def init_groups():
@@ -19,9 +19,7 @@ def init_permissions():
         {
             "codename": "edit_instance_settings",
             "name": "change the instance info",
-            "groups": [
-                "admin",
-            ],
+            "groups": ["admin"],
         },
         {
             "codename": "set_user_group",
@@ -55,7 +53,7 @@ def init_permissions():
         },
     ]
 
-    content_type = ContentType.objects.get_for_model(User)
+    content_type = ContentType.objects.get_for_model(models.User)
     for permission in permissions:
         permission_obj = Permission.objects.create(
             codename=permission["codename"],
@@ -66,15 +64,12 @@ def init_permissions():
         for group_name in permission["groups"]:
             Group.objects.get(name=group_name).permissions.add(permission_obj)
 
-    # while the groups and permissions shouldn't be changed because the code
-    # depends on them, what permissions go with what groups should be editable
-
 
 def init_connectors():
     """access book data sources"""
-    Connector.objects.create(
+    models.Connector.objects.create(
         identifier="bookwyrm.social",
-        name="BookWyrm dot Social",
+        name="Bookwyrm.social",
         connector_file="bookwyrm_connector",
         base_url="https://bookwyrm.social",
         books_url="https://bookwyrm.social/book",
@@ -84,7 +79,8 @@ def init_connectors():
         priority=2,
     )
 
-    Connector.objects.create(
+    # pylint: disable=line-too-long
+    models.Connector.objects.create(
         identifier="inventaire.io",
         name="Inventaire",
         connector_file="inventaire",
@@ -93,10 +89,10 @@ def init_connectors():
         covers_url="https://inventaire.io",
         search_url="https://inventaire.io/api/search?types=works&types=works&search=",
         isbn_search_url="https://inventaire.io/api/entities?action=by-uris&uris=isbn%3A",
-        priority=3,
+        priority=1,
     )
 
-    Connector.objects.create(
+    models.Connector.objects.create(
         identifier="openlibrary.org",
         name="OpenLibrary",
         connector_file="openlibrary",
@@ -105,34 +101,71 @@ def init_connectors():
         covers_url="https://covers.openlibrary.org",
         search_url="https://openlibrary.org/search?q=",
         isbn_search_url="https://openlibrary.org/api/books?jscmd=data&format=json&bibkeys=ISBN:",
-        priority=3,
+        priority=1,
     )
 
 
-def init_federated_servers():
-    """big no to nazis"""
-    built_in_blocks = ["gab.ai", "gab.com"]
-    for server in built_in_blocks:
-        FederatedServer.objects.create(
-            server_name=server,
-            status="blocked",
-        )
-
-
 def init_settings():
     """info about the instance"""
-    SiteSettings.objects.create(
+    models.SiteSettings.objects.create(
         support_link="https://www.patreon.com/bookwyrm",
         support_title="Patreon",
+        install_mode=True,
     )
 
 
+def init_link_domains():
+    """safe book links"""
+    domains = [
+        ("standardebooks.org", "Standard EBooks"),
+        ("www.gutenberg.org", "Project Gutenberg"),
+        ("archive.org", "Internet Archive"),
+        ("openlibrary.org", "Open Library"),
+        ("theanarchistlibrary.org", "The Anarchist Library"),
+    ]
+    for domain, name in domains:
+        models.LinkDomain.objects.create(
+            domain=domain,
+            name=name,
+            status="approved",
+        )
+
+
+# pylint: disable=no-self-use
+# pylint: disable=unused-argument
 class Command(BaseCommand):
+    """command-line options"""
+
     help = "Initializes the database with starter data"
 
+    def add_arguments(self, parser):
+        """specify which function to run"""
+        parser.add_argument(
+            "--limit",
+            default=None,
+            help="Limit init to specific table",
+        )
+
     def handle(self, *args, **options):
-        init_groups()
-        init_permissions()
-        init_connectors()
-        init_federated_servers()
-        init_settings()
+        """execute init"""
+        limit = options.get("limit")
+        tables = [
+            "group",
+            "permission",
+            "connector",
+            "settings",
+            "linkdomain",
+        ]
+        if limit and limit not in tables:
+            raise Exception("Invalid table limit:", limit)
+
+        if not limit or limit == "group":
+            init_groups()
+        if not limit or limit == "permission":
+            init_permissions()
+        if not limit or limit == "connector":
+            init_connectors()
+        if not limit or limit == "settings":
+            init_settings()
+        if not limit or limit == "linkdomain":
+            init_link_domains()

bookwyrm/management/commands/instance_version.py

@@ -0,0 +1,54 @@
+""" Get your admin code to allow install """
+from django.core.management.base import BaseCommand
+
+from bookwyrm import models
+from bookwyrm.settings import VERSION
+
+
+# pylint: disable=no-self-use
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "What version is this?"
+
+    def add_arguments(self, parser):
+        """specify which function to run"""
+        parser.add_argument(
+            "--current",
+            action="store_true",
+            help="Version stored in database",
+        )
+        parser.add_argument(
+            "--target",
+            action="store_true",
+            help="Version stored in settings",
+        )
+        parser.add_argument(
+            "--update",
+            action="store_true",
+            help="Update database version",
+        )
+
+    # pylint: disable=unused-argument
+    def handle(self, *args, **options):
+        """execute init"""
+        site = models.SiteSettings.objects.get()
+        current = site.version or "0.0.1"
+        target = VERSION
+        if options.get("current"):
+            print(current)
+            return
+
+        if options.get("target"):
+            print(target)
+            return
+
+        if options.get("update"):
+            site.version = target
+            site.save()
+            return
+
+        if current != target:
+            print(f"{current}/{target}")
+        else:
+            print(current)

bookwyrm/management/commands/populate_lists_streams.py

@@ -0,0 +1,28 @@
+""" Re-create list streams """
+from django.core.management.base import BaseCommand
+from bookwyrm import lists_stream, models
+
+
+def populate_lists_streams():
+    """build all the lists streams for all the users"""
+    print("Populating lists streams")
+    users = models.User.objects.filter(
+        local=True,
+        is_active=True,
+    ).order_by("-last_active_date")
+    print("This may take a long time! Please be patient.")
+    for user in users:
+        print(".", end="")
+        lists_stream.populate_lists_task.delay(user.id)
+    print("\nAll done, thank you for your patience!")
+
+
+class Command(BaseCommand):
+    """start all over with lists streams"""
+
+    help = "Populate list streams for all users"
+
+    # pylint: disable=no-self-use,unused-argument
+    def handle(self, *args, **options):
+        """run feed builder"""
+        populate_lists_streams()

bookwyrm/management/commands/populate_streams.py

@@ -1,18 +1,20 @@
 """ Re-create user streams """
 from django.core.management.base import BaseCommand
-from bookwyrm import activitystreams, models
+from bookwyrm import activitystreams, lists_stream, models
 
 
 def populate_streams(stream=None):
     """build all the streams for all the users"""
     streams = [stream] if stream else activitystreams.streams.keys()
-    print("Populations streams", streams)
+    print("Populating streams", streams)
     users = models.User.objects.filter(
         local=True,
         is_active=True,
     ).order_by("-last_active_date")
     print("This may take a long time! Please be patient.")
     for user in users:
+        print(".", end="")
+        lists_stream.populate_lists_task.delay(user.id)
         for stream_key in streams:
             print(".", end="")
             activitystreams.populate_stream_task.delay(stream_key, user.id)

bookwyrm/migrations/0107_alter_user_preferred_language.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.2.5 on 2021-10-11 16:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0106_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es", "Español (Spanish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("pt-br", "Português - Brasil (Brazilian Portugues)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0107_auto_20211016_0639.py

@@ -0,0 +1,871 @@
+# Generated by Django 3.2.5 on 2021-10-16 06:39
+
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0106_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Group",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                ("updated_date", models.DateTimeField(auto_now=True)),
+                (
+                    "remote_id",
+                    bookwyrm.models.fields.RemoteIdField(
+                        max_length=255,
+                        null=True,
+                        validators=[bookwyrm.models.fields.validate_remote_id],
+                    ),
+                ),
+                ("name", bookwyrm.models.fields.CharField(max_length=100)),
+                (
+                    "description",
+                    bookwyrm.models.fields.TextField(blank=True, null=True),
+                ),
+                (
+                    "privacy",
+                    bookwyrm.models.fields.PrivacyField(
+                        choices=[
+                            ("public", "Public"),
+                            ("unlisted", "Unlisted"),
+                            ("followers", "Followers"),
+                            ("direct", "Direct"),
+                        ],
+                        default="public",
+                        max_length=255,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="GroupMember",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                ("updated_date", models.DateTimeField(auto_now=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="GroupMemberInvitation",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+            ],
+        ),
+        migrations.RemoveConstraint(
+            model_name="notification",
+            name="notification_type_valid",
+        ),
+        migrations.AlterField(
+            model_name="list",
+            name="curation",
+            field=bookwyrm.models.fields.CharField(
+                choices=[
+                    ("closed", "Closed"),
+                    ("open", "Open"),
+                    ("curated", "Curated"),
+                    ("group", "Group"),
+                ],
+                default="closed",
+                max_length=255,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("BOOST", "Boost"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                ],
+                max_length=255,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es", "Español (Spanish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_timezone",
+            field=models.CharField(
+                choices=[
+                    ("Africa/Abidjan", "Africa/Abidjan"),
+                    ("Africa/Accra", "Africa/Accra"),
+                    ("Africa/Addis_Ababa", "Africa/Addis_Ababa"),
+                    ("Africa/Algiers", "Africa/Algiers"),
+                    ("Africa/Asmara", "Africa/Asmara"),
+                    ("Africa/Asmera", "Africa/Asmera"),
+                    ("Africa/Bamako", "Africa/Bamako"),
+                    ("Africa/Bangui", "Africa/Bangui"),
+                    ("Africa/Banjul", "Africa/Banjul"),
+                    ("Africa/Bissau", "Africa/Bissau"),
+                    ("Africa/Blantyre", "Africa/Blantyre"),
+                    ("Africa/Brazzaville", "Africa/Brazzaville"),
+                    ("Africa/Bujumbura", "Africa/Bujumbura"),
+                    ("Africa/Cairo", "Africa/Cairo"),
+                    ("Africa/Casablanca", "Africa/Casablanca"),
+                    ("Africa/Ceuta", "Africa/Ceuta"),
+                    ("Africa/Conakry", "Africa/Conakry"),
+                    ("Africa/Dakar", "Africa/Dakar"),
+                    ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"),
+                    ("Africa/Djibouti", "Africa/Djibouti"),
+                    ("Africa/Douala", "Africa/Douala"),
+                    ("Africa/El_Aaiun", "Africa/El_Aaiun"),
+                    ("Africa/Freetown", "Africa/Freetown"),
+                    ("Africa/Gaborone", "Africa/Gaborone"),
+                    ("Africa/Harare", "Africa/Harare"),
+                    ("Africa/Johannesburg", "Africa/Johannesburg"),
+                    ("Africa/Juba", "Africa/Juba"),
+                    ("Africa/Kampala", "Africa/Kampala"),
+                    ("Africa/Khartoum", "Africa/Khartoum"),
+                    ("Africa/Kigali", "Africa/Kigali"),
+                    ("Africa/Kinshasa", "Africa/Kinshasa"),
+                    ("Africa/Lagos", "Africa/Lagos"),
+                    ("Africa/Libreville", "Africa/Libreville"),
+                    ("Africa/Lome", "Africa/Lome"),
+                    ("Africa/Luanda", "Africa/Luanda"),
+                    ("Africa/Lubumbashi", "Africa/Lubumbashi"),
+                    ("Africa/Lusaka", "Africa/Lusaka"),
+                    ("Africa/Malabo", "Africa/Malabo"),
+                    ("Africa/Maputo", "Africa/Maputo"),
+                    ("Africa/Maseru", "Africa/Maseru"),
+                    ("Africa/Mbabane", "Africa/Mbabane"),
+                    ("Africa/Mogadishu", "Africa/Mogadishu"),
+                    ("Africa/Monrovia", "Africa/Monrovia"),
+                    ("Africa/Nairobi", "Africa/Nairobi"),
+                    ("Africa/Ndjamena", "Africa/Ndjamena"),
+                    ("Africa/Niamey", "Africa/Niamey"),
+                    ("Africa/Nouakchott", "Africa/Nouakchott"),
+                    ("Africa/Ouagadougou", "Africa/Ouagadougou"),
+                    ("Africa/Porto-Novo", "Africa/Porto-Novo"),
+                    ("Africa/Sao_Tome", "Africa/Sao_Tome"),
+                    ("Africa/Timbuktu", "Africa/Timbuktu"),
+                    ("Africa/Tripoli", "Africa/Tripoli"),
+                    ("Africa/Tunis", "Africa/Tunis"),
+                    ("Africa/Windhoek", "Africa/Windhoek"),
+                    ("America/Adak", "America/Adak"),
+                    ("America/Anchorage", "America/Anchorage"),
+                    ("America/Anguilla", "America/Anguilla"),
+                    ("America/Antigua", "America/Antigua"),
+                    ("America/Araguaina", "America/Araguaina"),
+                    (
+                        "America/Argentina/Buenos_Aires",
+                        "America/Argentina/Buenos_Aires",
+                    ),
+                    ("America/Argentina/Catamarca", "America/Argentina/Catamarca"),
+                    (
+                        "America/Argentina/ComodRivadavia",
+                        "America/Argentina/ComodRivadavia",
+                    ),
+                    ("America/Argentina/Cordoba", "America/Argentina/Cordoba"),
+                    ("America/Argentina/Jujuy", "America/Argentina/Jujuy"),
+                    ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"),
+                    ("America/Argentina/Mendoza", "America/Argentina/Mendoza"),
+                    (
+                        "America/Argentina/Rio_Gallegos",
+                        "America/Argentina/Rio_Gallegos",
+                    ),
+                    ("America/Argentina/Salta", "America/Argentina/Salta"),
+                    ("America/Argentina/San_Juan", "America/Argentina/San_Juan"),
+                    ("America/Argentina/San_Luis", "America/Argentina/San_Luis"),
+                    ("America/Argentina/Tucuman", "America/Argentina/Tucuman"),
+                    ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"),
+                    ("America/Aruba", "America/Aruba"),
+                    ("America/Asuncion", "America/Asuncion"),
+                    ("America/Atikokan", "America/Atikokan"),
+                    ("America/Atka", "America/Atka"),
+                    ("America/Bahia", "America/Bahia"),
+                    ("America/Bahia_Banderas", "America/Bahia_Banderas"),
+                    ("America/Barbados", "America/Barbados"),
+                    ("America/Belem", "America/Belem"),
+                    ("America/Belize", "America/Belize"),
+                    ("America/Blanc-Sablon", "America/Blanc-Sablon"),
+                    ("America/Boa_Vista", "America/Boa_Vista"),
+                    ("America/Bogota", "America/Bogota"),
+                    ("America/Boise", "America/Boise"),
+                    ("America/Buenos_Aires", "America/Buenos_Aires"),
+                    ("America/Cambridge_Bay", "America/Cambridge_Bay"),
+                    ("America/Campo_Grande", "America/Campo_Grande"),
+                    ("America/Cancun", "America/Cancun"),
+                    ("America/Caracas", "America/Caracas"),
+                    ("America/Catamarca", "America/Catamarca"),
+                    ("America/Cayenne", "America/Cayenne"),
+                    ("America/Cayman", "America/Cayman"),
+                    ("America/Chicago", "America/Chicago"),
+                    ("America/Chihuahua", "America/Chihuahua"),
+                    ("America/Coral_Harbour", "America/Coral_Harbour"),
+                    ("America/Cordoba", "America/Cordoba"),
+                    ("America/Costa_Rica", "America/Costa_Rica"),
+                    ("America/Creston", "America/Creston"),
+                    ("America/Cuiaba", "America/Cuiaba"),
+                    ("America/Curacao", "America/Curacao"),
+                    ("America/Danmarkshavn", "America/Danmarkshavn"),
+                    ("America/Dawson", "America/Dawson"),
+                    ("America/Dawson_Creek", "America/Dawson_Creek"),
+                    ("America/Denver", "America/Denver"),
+                    ("America/Detroit", "America/Detroit"),
+                    ("America/Dominica", "America/Dominica"),
+                    ("America/Edmonton", "America/Edmonton"),
+                    ("America/Eirunepe", "America/Eirunepe"),
+                    ("America/El_Salvador", "America/El_Salvador"),
+                    ("America/Ensenada", "America/Ensenada"),
+                    ("America/Fort_Nelson", "America/Fort_Nelson"),
+                    ("America/Fort_Wayne", "America/Fort_Wayne"),
+                    ("America/Fortaleza", "America/Fortaleza"),
+                    ("America/Glace_Bay", "America/Glace_Bay"),
+                    ("America/Godthab", "America/Godthab"),
+                    ("America/Goose_Bay", "America/Goose_Bay"),
+                    ("America/Grand_Turk", "America/Grand_Turk"),
+                    ("America/Grenada", "America/Grenada"),
+                    ("America/Guadeloupe", "America/Guadeloupe"),
+                    ("America/Guatemala", "America/Guatemala"),
+                    ("America/Guayaquil", "America/Guayaquil"),
+                    ("America/Guyana", "America/Guyana"),
+                    ("America/Halifax", "America/Halifax"),
+                    ("America/Havana", "America/Havana"),
+                    ("America/Hermosillo", "America/Hermosillo"),
+                    ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"),
+                    ("America/Indiana/Knox", "America/Indiana/Knox"),
+                    ("America/Indiana/Marengo", "America/Indiana/Marengo"),
+                    ("America/Indiana/Petersburg", "America/Indiana/Petersburg"),
+                    ("America/Indiana/Tell_City", "America/Indiana/Tell_City"),
+                    ("America/Indiana/Vevay", "America/Indiana/Vevay"),
+                    ("America/Indiana/Vincennes", "America/Indiana/Vincennes"),
+                    ("America/Indiana/Winamac", "America/Indiana/Winamac"),
+                    ("America/Indianapolis", "America/Indianapolis"),
+                    ("America/Inuvik", "America/Inuvik"),
+                    ("America/Iqaluit", "America/Iqaluit"),
+                    ("America/Jamaica", "America/Jamaica"),
+                    ("America/Jujuy", "America/Jujuy"),
+                    ("America/Juneau", "America/Juneau"),
+                    ("America/Kentucky/Louisville", "America/Kentucky/Louisville"),
+                    ("America/Kentucky/Monticello", "America/Kentucky/Monticello"),
+                    ("America/Knox_IN", "America/Knox_IN"),
+                    ("America/Kralendijk", "America/Kralendijk"),
+                    ("America/La_Paz", "America/La_Paz"),
+                    ("America/Lima", "America/Lima"),
+                    ("America/Los_Angeles", "America/Los_Angeles"),
+                    ("America/Louisville", "America/Louisville"),
+                    ("America/Lower_Princes", "America/Lower_Princes"),
+                    ("America/Maceio", "America/Maceio"),
+                    ("America/Managua", "America/Managua"),
+                    ("America/Manaus", "America/Manaus"),
+                    ("America/Marigot", "America/Marigot"),
+                    ("America/Martinique", "America/Martinique"),
+                    ("America/Matamoros", "America/Matamoros"),
+                    ("America/Mazatlan", "America/Mazatlan"),
+                    ("America/Mendoza", "America/Mendoza"),
+                    ("America/Menominee", "America/Menominee"),
+                    ("America/Merida", "America/Merida"),
+                    ("America/Metlakatla", "America/Metlakatla"),
+                    ("America/Mexico_City", "America/Mexico_City"),
+                    ("America/Miquelon", "America/Miquelon"),
+                    ("America/Moncton", "America/Moncton"),
+                    ("America/Monterrey", "America/Monterrey"),
+                    ("America/Montevideo", "America/Montevideo"),
+                    ("America/Montreal", "America/Montreal"),
+                    ("America/Montserrat", "America/Montserrat"),
+                    ("America/Nassau", "America/Nassau"),
+                    ("America/New_York", "America/New_York"),
+                    ("America/Nipigon", "America/Nipigon"),
+                    ("America/Nome", "America/Nome"),
+                    ("America/Noronha", "America/Noronha"),
+                    ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"),
+                    ("America/North_Dakota/Center", "America/North_Dakota/Center"),
+                    (
+                        "America/North_Dakota/New_Salem",
+                        "America/North_Dakota/New_Salem",
+                    ),
+                    ("America/Nuuk", "America/Nuuk"),
+                    ("America/Ojinaga", "America/Ojinaga"),
+                    ("America/Panama", "America/Panama"),
+                    ("America/Pangnirtung", "America/Pangnirtung"),
+                    ("America/Paramaribo", "America/Paramaribo"),
+                    ("America/Phoenix", "America/Phoenix"),
+                    ("America/Port-au-Prince", "America/Port-au-Prince"),
+                    ("America/Port_of_Spain", "America/Port_of_Spain"),
+                    ("America/Porto_Acre", "America/Porto_Acre"),
+                    ("America/Porto_Velho", "America/Porto_Velho"),
+                    ("America/Puerto_Rico", "America/Puerto_Rico"),
+                    ("America/Punta_Arenas", "America/Punta_Arenas"),
+                    ("America/Rainy_River", "America/Rainy_River"),
+                    ("America/Rankin_Inlet", "America/Rankin_Inlet"),
+                    ("America/Recife", "America/Recife"),
+                    ("America/Regina", "America/Regina"),
+                    ("America/Resolute", "America/Resolute"),
+                    ("America/Rio_Branco", "America/Rio_Branco"),
+                    ("America/Rosario", "America/Rosario"),
+                    ("America/Santa_Isabel", "America/Santa_Isabel"),
+                    ("America/Santarem", "America/Santarem"),
+                    ("America/Santiago", "America/Santiago"),
+                    ("America/Santo_Domingo", "America/Santo_Domingo"),
+                    ("America/Sao_Paulo", "America/Sao_Paulo"),
+                    ("America/Scoresbysund", "America/Scoresbysund"),
+                    ("America/Shiprock", "America/Shiprock"),
+                    ("America/Sitka", "America/Sitka"),
+                    ("America/St_Barthelemy", "America/St_Barthelemy"),
+                    ("America/St_Johns", "America/St_Johns"),
+                    ("America/St_Kitts", "America/St_Kitts"),
+                    ("America/St_Lucia", "America/St_Lucia"),
+                    ("America/St_Thomas", "America/St_Thomas"),
+                    ("America/St_Vincent", "America/St_Vincent"),
+                    ("America/Swift_Current", "America/Swift_Current"),
+                    ("America/Tegucigalpa", "America/Tegucigalpa"),
+                    ("America/Thule", "America/Thule"),
+                    ("America/Thunder_Bay", "America/Thunder_Bay"),
+                    ("America/Tijuana", "America/Tijuana"),
+                    ("America/Toronto", "America/Toronto"),
+                    ("America/Tortola", "America/Tortola"),
+                    ("America/Vancouver", "America/Vancouver"),
+                    ("America/Virgin", "America/Virgin"),
+                    ("America/Whitehorse", "America/Whitehorse"),
+                    ("America/Winnipeg", "America/Winnipeg"),
+                    ("America/Yakutat", "America/Yakutat"),
+                    ("America/Yellowknife", "America/Yellowknife"),
+                    ("Antarctica/Casey", "Antarctica/Casey"),
+                    ("Antarctica/Davis", "Antarctica/Davis"),
+                    ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"),
+                    ("Antarctica/Macquarie", "Antarctica/Macquarie"),
+                    ("Antarctica/Mawson", "Antarctica/Mawson"),
+                    ("Antarctica/McMurdo", "Antarctica/McMurdo"),
+                    ("Antarctica/Palmer", "Antarctica/Palmer"),
+                    ("Antarctica/Rothera", "Antarctica/Rothera"),
+                    ("Antarctica/South_Pole", "Antarctica/South_Pole"),
+                    ("Antarctica/Syowa", "Antarctica/Syowa"),
+                    ("Antarctica/Troll", "Antarctica/Troll"),
+                    ("Antarctica/Vostok", "Antarctica/Vostok"),
+                    ("Arctic/Longyearbyen", "Arctic/Longyearbyen"),
+                    ("Asia/Aden", "Asia/Aden"),
+                    ("Asia/Almaty", "Asia/Almaty"),
+                    ("Asia/Amman", "Asia/Amman"),
+                    ("Asia/Anadyr", "Asia/Anadyr"),
+                    ("Asia/Aqtau", "Asia/Aqtau"),
+                    ("Asia/Aqtobe", "Asia/Aqtobe"),
+                    ("Asia/Ashgabat", "Asia/Ashgabat"),
+                    ("Asia/Ashkhabad", "Asia/Ashkhabad"),
+                    ("Asia/Atyrau", "Asia/Atyrau"),
+                    ("Asia/Baghdad", "Asia/Baghdad"),
+                    ("Asia/Bahrain", "Asia/Bahrain"),
+                    ("Asia/Baku", "Asia/Baku"),
+                    ("Asia/Bangkok", "Asia/Bangkok"),
+                    ("Asia/Barnaul", "Asia/Barnaul"),
+                    ("Asia/Beirut", "Asia/Beirut"),
+                    ("Asia/Bishkek", "Asia/Bishkek"),
+                    ("Asia/Brunei", "Asia/Brunei"),
+                    ("Asia/Calcutta", "Asia/Calcutta"),
+                    ("Asia/Chita", "Asia/Chita"),
+                    ("Asia/Choibalsan", "Asia/Choibalsan"),
+                    ("Asia/Chongqing", "Asia/Chongqing"),
+                    ("Asia/Chungking", "Asia/Chungking"),
+                    ("Asia/Colombo", "Asia/Colombo"),
+                    ("Asia/Dacca", "Asia/Dacca"),
+                    ("Asia/Damascus", "Asia/Damascus"),
+                    ("Asia/Dhaka", "Asia/Dhaka"),
+                    ("Asia/Dili", "Asia/Dili"),
+                    ("Asia/Dubai", "Asia/Dubai"),
+                    ("Asia/Dushanbe", "Asia/Dushanbe"),
+                    ("Asia/Famagusta", "Asia/Famagusta"),
+                    ("Asia/Gaza", "Asia/Gaza"),
+                    ("Asia/Harbin", "Asia/Harbin"),
+                    ("Asia/Hebron", "Asia/Hebron"),
+                    ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"),
+                    ("Asia/Hong_Kong", "Asia/Hong_Kong"),
+                    ("Asia/Hovd", "Asia/Hovd"),
+                    ("Asia/Irkutsk", "Asia/Irkutsk"),
+                    ("Asia/Istanbul", "Asia/Istanbul"),
+                    ("Asia/Jakarta", "Asia/Jakarta"),
+                    ("Asia/Jayapura", "Asia/Jayapura"),
+                    ("Asia/Jerusalem", "Asia/Jerusalem"),
+                    ("Asia/Kabul", "Asia/Kabul"),
+                    ("Asia/Kamchatka", "Asia/Kamchatka"),
+                    ("Asia/Karachi", "Asia/Karachi"),
+                    ("Asia/Kashgar", "Asia/Kashgar"),
+                    ("Asia/Kathmandu", "Asia/Kathmandu"),
+                    ("Asia/Katmandu", "Asia/Katmandu"),
+                    ("Asia/Khandyga", "Asia/Khandyga"),
+                    ("Asia/Kolkata", "Asia/Kolkata"),
+                    ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"),
+                    ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"),
+                    ("Asia/Kuching", "Asia/Kuching"),
+                    ("Asia/Kuwait", "Asia/Kuwait"),
+                    ("Asia/Macao", "Asia/Macao"),
+                    ("Asia/Macau", "Asia/Macau"),
+                    ("Asia/Magadan", "Asia/Magadan"),
+                    ("Asia/Makassar", "Asia/Makassar"),
+                    ("Asia/Manila", "Asia/Manila"),
+                    ("Asia/Muscat", "Asia/Muscat"),
+                    ("Asia/Nicosia", "Asia/Nicosia"),
+                    ("Asia/Novokuznetsk", "Asia/Novokuznetsk"),
+                    ("Asia/Novosibirsk", "Asia/Novosibirsk"),
+                    ("Asia/Omsk", "Asia/Omsk"),
+                    ("Asia/Oral", "Asia/Oral"),
+                    ("Asia/Phnom_Penh", "Asia/Phnom_Penh"),
+                    ("Asia/Pontianak", "Asia/Pontianak"),
+                    ("Asia/Pyongyang", "Asia/Pyongyang"),
+                    ("Asia/Qatar", "Asia/Qatar"),
+                    ("Asia/Qostanay", "Asia/Qostanay"),
+                    ("Asia/Qyzylorda", "Asia/Qyzylorda"),
+                    ("Asia/Rangoon", "Asia/Rangoon"),
+                    ("Asia/Riyadh", "Asia/Riyadh"),
+                    ("Asia/Saigon", "Asia/Saigon"),
+                    ("Asia/Sakhalin", "Asia/Sakhalin"),
+                    ("Asia/Samarkand", "Asia/Samarkand"),
+                    ("Asia/Seoul", "Asia/Seoul"),
+                    ("Asia/Shanghai", "Asia/Shanghai"),
+                    ("Asia/Singapore", "Asia/Singapore"),
+                    ("Asia/Srednekolymsk", "Asia/Srednekolymsk"),
+                    ("Asia/Taipei", "Asia/Taipei"),
+                    ("Asia/Tashkent", "Asia/Tashkent"),
+                    ("Asia/Tbilisi", "Asia/Tbilisi"),
+                    ("Asia/Tehran", "Asia/Tehran"),
+                    ("Asia/Tel_Aviv", "Asia/Tel_Aviv"),
+                    ("Asia/Thimbu", "Asia/Thimbu"),
+                    ("Asia/Thimphu", "Asia/Thimphu"),
+                    ("Asia/Tokyo", "Asia/Tokyo"),
+                    ("Asia/Tomsk", "Asia/Tomsk"),
+                    ("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"),
+                    ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"),
+                    ("Asia/Ulan_Bator", "Asia/Ulan_Bator"),
+                    ("Asia/Urumqi", "Asia/Urumqi"),
+                    ("Asia/Ust-Nera", "Asia/Ust-Nera"),
+                    ("Asia/Vientiane", "Asia/Vientiane"),
+                    ("Asia/Vladivostok", "Asia/Vladivostok"),
+                    ("Asia/Yakutsk", "Asia/Yakutsk"),
+                    ("Asia/Yangon", "Asia/Yangon"),
+                    ("Asia/Yekaterinburg", "Asia/Yekaterinburg"),
+                    ("Asia/Yerevan", "Asia/Yerevan"),
+                    ("Atlantic/Azores", "Atlantic/Azores"),
+                    ("Atlantic/Bermuda", "Atlantic/Bermuda"),
+                    ("Atlantic/Canary", "Atlantic/Canary"),
+                    ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"),
+                    ("Atlantic/Faeroe", "Atlantic/Faeroe"),
+                    ("Atlantic/Faroe", "Atlantic/Faroe"),
+                    ("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"),
+                    ("Atlantic/Madeira", "Atlantic/Madeira"),
+                    ("Atlantic/Reykjavik", "Atlantic/Reykjavik"),
+                    ("Atlantic/South_Georgia", "Atlantic/South_Georgia"),
+                    ("Atlantic/St_Helena", "Atlantic/St_Helena"),
+                    ("Atlantic/Stanley", "Atlantic/Stanley"),
+                    ("Australia/ACT", "Australia/ACT"),
+                    ("Australia/Adelaide", "Australia/Adelaide"),
+                    ("Australia/Brisbane", "Australia/Brisbane"),
+                    ("Australia/Broken_Hill", "Australia/Broken_Hill"),
+                    ("Australia/Canberra", "Australia/Canberra"),
+                    ("Australia/Currie", "Australia/Currie"),
+                    ("Australia/Darwin", "Australia/Darwin"),
+                    ("Australia/Eucla", "Australia/Eucla"),
+                    ("Australia/Hobart", "Australia/Hobart"),
+                    ("Australia/LHI", "Australia/LHI"),
+                    ("Australia/Lindeman", "Australia/Lindeman"),
+                    ("Australia/Lord_Howe", "Australia/Lord_Howe"),
+                    ("Australia/Melbourne", "Australia/Melbourne"),
+                    ("Australia/NSW", "Australia/NSW"),
+                    ("Australia/North", "Australia/North"),
+                    ("Australia/Perth", "Australia/Perth"),
+                    ("Australia/Queensland", "Australia/Queensland"),
+                    ("Australia/South", "Australia/South"),
+                    ("Australia/Sydney", "Australia/Sydney"),
+                    ("Australia/Tasmania", "Australia/Tasmania"),
+                    ("Australia/Victoria", "Australia/Victoria"),
+                    ("Australia/West", "Australia/West"),
+                    ("Australia/Yancowinna", "Australia/Yancowinna"),
+                    ("Brazil/Acre", "Brazil/Acre"),
+                    ("Brazil/DeNoronha", "Brazil/DeNoronha"),
+                    ("Brazil/East", "Brazil/East"),
+                    ("Brazil/West", "Brazil/West"),
+                    ("CET", "CET"),
+                    ("CST6CDT", "CST6CDT"),
+                    ("Canada/Atlantic", "Canada/Atlantic"),
+                    ("Canada/Central", "Canada/Central"),
+                    ("Canada/Eastern", "Canada/Eastern"),
+                    ("Canada/Mountain", "Canada/Mountain"),
+                    ("Canada/Newfoundland", "Canada/Newfoundland"),
+                    ("Canada/Pacific", "Canada/Pacific"),
+                    ("Canada/Saskatchewan", "Canada/Saskatchewan"),
+                    ("Canada/Yukon", "Canada/Yukon"),
+                    ("Chile/Continental", "Chile/Continental"),
+                    ("Chile/EasterIsland", "Chile/EasterIsland"),
+                    ("Cuba", "Cuba"),
+                    ("EET", "EET"),
+                    ("EST", "EST"),
+                    ("EST5EDT", "EST5EDT"),
+                    ("Egypt", "Egypt"),
+                    ("Eire", "Eire"),
+                    ("Etc/GMT", "Etc/GMT"),
+                    ("Etc/GMT+0", "Etc/GMT+0"),
+                    ("Etc/GMT+1", "Etc/GMT+1"),
+                    ("Etc/GMT+10", "Etc/GMT+10"),
+                    ("Etc/GMT+11", "Etc/GMT+11"),
+                    ("Etc/GMT+12", "Etc/GMT+12"),
+                    ("Etc/GMT+2", "Etc/GMT+2"),
+                    ("Etc/GMT+3", "Etc/GMT+3"),
+                    ("Etc/GMT+4", "Etc/GMT+4"),
+                    ("Etc/GMT+5", "Etc/GMT+5"),
+                    ("Etc/GMT+6", "Etc/GMT+6"),
+                    ("Etc/GMT+7", "Etc/GMT+7"),
+                    ("Etc/GMT+8", "Etc/GMT+8"),
+                    ("Etc/GMT+9", "Etc/GMT+9"),
+                    ("Etc/GMT-0", "Etc/GMT-0"),
+                    ("Etc/GMT-1", "Etc/GMT-1"),
+                    ("Etc/GMT-10", "Etc/GMT-10"),
+                    ("Etc/GMT-11", "Etc/GMT-11"),
+                    ("Etc/GMT-12", "Etc/GMT-12"),
+                    ("Etc/GMT-13", "Etc/GMT-13"),
+                    ("Etc/GMT-14", "Etc/GMT-14"),
+                    ("Etc/GMT-2", "Etc/GMT-2"),
+                    ("Etc/GMT-3", "Etc/GMT-3"),
+                    ("Etc/GMT-4", "Etc/GMT-4"),
+                    ("Etc/GMT-5", "Etc/GMT-5"),
+                    ("Etc/GMT-6", "Etc/GMT-6"),
+                    ("Etc/GMT-7", "Etc/GMT-7"),
+                    ("Etc/GMT-8", "Etc/GMT-8"),
+                    ("Etc/GMT-9", "Etc/GMT-9"),
+                    ("Etc/GMT0", "Etc/GMT0"),
+                    ("Etc/Greenwich", "Etc/Greenwich"),
+                    ("Etc/UCT", "Etc/UCT"),
+                    ("Etc/UTC", "Etc/UTC"),
+                    ("Etc/Universal", "Etc/Universal"),
+                    ("Etc/Zulu", "Etc/Zulu"),
+                    ("Europe/Amsterdam", "Europe/Amsterdam"),
+                    ("Europe/Andorra", "Europe/Andorra"),
+                    ("Europe/Astrakhan", "Europe/Astrakhan"),
+                    ("Europe/Athens", "Europe/Athens"),
+                    ("Europe/Belfast", "Europe/Belfast"),
+                    ("Europe/Belgrade", "Europe/Belgrade"),
+                    ("Europe/Berlin", "Europe/Berlin"),
+                    ("Europe/Bratislava", "Europe/Bratislava"),
+                    ("Europe/Brussels", "Europe/Brussels"),
+                    ("Europe/Bucharest", "Europe/Bucharest"),
+                    ("Europe/Budapest", "Europe/Budapest"),
+                    ("Europe/Busingen", "Europe/Busingen"),
+                    ("Europe/Chisinau", "Europe/Chisinau"),
+                    ("Europe/Copenhagen", "Europe/Copenhagen"),
+                    ("Europe/Dublin", "Europe/Dublin"),
+                    ("Europe/Gibraltar", "Europe/Gibraltar"),
+                    ("Europe/Guernsey", "Europe/Guernsey"),
+                    ("Europe/Helsinki", "Europe/Helsinki"),
+                    ("Europe/Isle_of_Man", "Europe/Isle_of_Man"),
+                    ("Europe/Istanbul", "Europe/Istanbul"),
+                    ("Europe/Jersey", "Europe/Jersey"),
+                    ("Europe/Kaliningrad", "Europe/Kaliningrad"),
+                    ("Europe/Kiev", "Europe/Kiev"),
+                    ("Europe/Kirov", "Europe/Kirov"),
+                    ("Europe/Lisbon", "Europe/Lisbon"),
+                    ("Europe/Ljubljana", "Europe/Ljubljana"),
+                    ("Europe/London", "Europe/London"),
+                    ("Europe/Luxembourg", "Europe/Luxembourg"),
+                    ("Europe/Madrid", "Europe/Madrid"),
+                    ("Europe/Malta", "Europe/Malta"),
+                    ("Europe/Mariehamn", "Europe/Mariehamn"),
+                    ("Europe/Minsk", "Europe/Minsk"),
+                    ("Europe/Monaco", "Europe/Monaco"),
+                    ("Europe/Moscow", "Europe/Moscow"),
+                    ("Europe/Nicosia", "Europe/Nicosia"),
+                    ("Europe/Oslo", "Europe/Oslo"),
+                    ("Europe/Paris", "Europe/Paris"),
+                    ("Europe/Podgorica", "Europe/Podgorica"),
+                    ("Europe/Prague", "Europe/Prague"),
+                    ("Europe/Riga", "Europe/Riga"),
+                    ("Europe/Rome", "Europe/Rome"),
+                    ("Europe/Samara", "Europe/Samara"),
+                    ("Europe/San_Marino", "Europe/San_Marino"),
+                    ("Europe/Sarajevo", "Europe/Sarajevo"),
+                    ("Europe/Saratov", "Europe/Saratov"),
+                    ("Europe/Simferopol", "Europe/Simferopol"),
+                    ("Europe/Skopje", "Europe/Skopje"),
+                    ("Europe/Sofia", "Europe/Sofia"),
+                    ("Europe/Stockholm", "Europe/Stockholm"),
+                    ("Europe/Tallinn", "Europe/Tallinn"),
+                    ("Europe/Tirane", "Europe/Tirane"),
+                    ("Europe/Tiraspol", "Europe/Tiraspol"),
+                    ("Europe/Ulyanovsk", "Europe/Ulyanovsk"),
+                    ("Europe/Uzhgorod", "Europe/Uzhgorod"),
+                    ("Europe/Vaduz", "Europe/Vaduz"),
+                    ("Europe/Vatican", "Europe/Vatican"),
+                    ("Europe/Vienna", "Europe/Vienna"),
+                    ("Europe/Vilnius", "Europe/Vilnius"),
+                    ("Europe/Volgograd", "Europe/Volgograd"),
+                    ("Europe/Warsaw", "Europe/Warsaw"),
+                    ("Europe/Zagreb", "Europe/Zagreb"),
+                    ("Europe/Zaporozhye", "Europe/Zaporozhye"),
+                    ("Europe/Zurich", "Europe/Zurich"),
+                    ("GB", "GB"),
+                    ("GB-Eire", "GB-Eire"),
+                    ("GMT", "GMT"),
+                    ("GMT+0", "GMT+0"),
+                    ("GMT-0", "GMT-0"),
+                    ("GMT0", "GMT0"),
+                    ("Greenwich", "Greenwich"),
+                    ("HST", "HST"),
+                    ("Hongkong", "Hongkong"),
+                    ("Iceland", "Iceland"),
+                    ("Indian/Antananarivo", "Indian/Antananarivo"),
+                    ("Indian/Chagos", "Indian/Chagos"),
+                    ("Indian/Christmas", "Indian/Christmas"),
+                    ("Indian/Cocos", "Indian/Cocos"),
+                    ("Indian/Comoro", "Indian/Comoro"),
+                    ("Indian/Kerguelen", "Indian/Kerguelen"),
+                    ("Indian/Mahe", "Indian/Mahe"),
+                    ("Indian/Maldives", "Indian/Maldives"),
+                    ("Indian/Mauritius", "Indian/Mauritius"),
+                    ("Indian/Mayotte", "Indian/Mayotte"),
+                    ("Indian/Reunion", "Indian/Reunion"),
+                    ("Iran", "Iran"),
+                    ("Israel", "Israel"),
+                    ("Jamaica", "Jamaica"),
+                    ("Japan", "Japan"),
+                    ("Kwajalein", "Kwajalein"),
+                    ("Libya", "Libya"),
+                    ("MET", "MET"),
+                    ("MST", "MST"),
+                    ("MST7MDT", "MST7MDT"),
+                    ("Mexico/BajaNorte", "Mexico/BajaNorte"),
+                    ("Mexico/BajaSur", "Mexico/BajaSur"),
+                    ("Mexico/General", "Mexico/General"),
+                    ("NZ", "NZ"),
+                    ("NZ-CHAT", "NZ-CHAT"),
+                    ("Navajo", "Navajo"),
+                    ("PRC", "PRC"),
+                    ("PST8PDT", "PST8PDT"),
+                    ("Pacific/Apia", "Pacific/Apia"),
+                    ("Pacific/Auckland", "Pacific/Auckland"),
+                    ("Pacific/Bougainville", "Pacific/Bougainville"),
+                    ("Pacific/Chatham", "Pacific/Chatham"),
+                    ("Pacific/Chuuk", "Pacific/Chuuk"),
+                    ("Pacific/Easter", "Pacific/Easter"),
+                    ("Pacific/Efate", "Pacific/Efate"),
+                    ("Pacific/Enderbury", "Pacific/Enderbury"),
+                    ("Pacific/Fakaofo", "Pacific/Fakaofo"),
+                    ("Pacific/Fiji", "Pacific/Fiji"),
+                    ("Pacific/Funafuti", "Pacific/Funafuti"),
+                    ("Pacific/Galapagos", "Pacific/Galapagos"),
+                    ("Pacific/Gambier", "Pacific/Gambier"),
+                    ("Pacific/Guadalcanal", "Pacific/Guadalcanal"),
+                    ("Pacific/Guam", "Pacific/Guam"),
+                    ("Pacific/Honolulu", "Pacific/Honolulu"),
+                    ("Pacific/Johnston", "Pacific/Johnston"),
+                    ("Pacific/Kanton", "Pacific/Kanton"),
+                    ("Pacific/Kiritimati", "Pacific/Kiritimati"),
+                    ("Pacific/Kosrae", "Pacific/Kosrae"),
+                    ("Pacific/Kwajalein", "Pacific/Kwajalein"),
+                    ("Pacific/Majuro", "Pacific/Majuro"),
+                    ("Pacific/Marquesas", "Pacific/Marquesas"),
+                    ("Pacific/Midway", "Pacific/Midway"),
+                    ("Pacific/Nauru", "Pacific/Nauru"),
+                    ("Pacific/Niue", "Pacific/Niue"),
+                    ("Pacific/Norfolk", "Pacific/Norfolk"),
+                    ("Pacific/Noumea", "Pacific/Noumea"),
+                    ("Pacific/Pago_Pago", "Pacific/Pago_Pago"),
+                    ("Pacific/Palau", "Pacific/Palau"),
+                    ("Pacific/Pitcairn", "Pacific/Pitcairn"),
+                    ("Pacific/Pohnpei", "Pacific/Pohnpei"),
+                    ("Pacific/Ponape", "Pacific/Ponape"),
+                    ("Pacific/Port_Moresby", "Pacific/Port_Moresby"),
+                    ("Pacific/Rarotonga", "Pacific/Rarotonga"),
+                    ("Pacific/Saipan", "Pacific/Saipan"),
+                    ("Pacific/Samoa", "Pacific/Samoa"),
+                    ("Pacific/Tahiti", "Pacific/Tahiti"),
+                    ("Pacific/Tarawa", "Pacific/Tarawa"),
+                    ("Pacific/Tongatapu", "Pacific/Tongatapu"),
+                    ("Pacific/Truk", "Pacific/Truk"),
+                    ("Pacific/Wake", "Pacific/Wake"),
+                    ("Pacific/Wallis", "Pacific/Wallis"),
+                    ("Pacific/Yap", "Pacific/Yap"),
+                    ("Poland", "Poland"),
+                    ("Portugal", "Portugal"),
+                    ("ROC", "ROC"),
+                    ("ROK", "ROK"),
+                    ("Singapore", "Singapore"),
+                    ("Turkey", "Turkey"),
+                    ("UCT", "UCT"),
+                    ("US/Alaska", "US/Alaska"),
+                    ("US/Aleutian", "US/Aleutian"),
+                    ("US/Arizona", "US/Arizona"),
+                    ("US/Central", "US/Central"),
+                    ("US/East-Indiana", "US/East-Indiana"),
+                    ("US/Eastern", "US/Eastern"),
+                    ("US/Hawaii", "US/Hawaii"),
+                    ("US/Indiana-Starke", "US/Indiana-Starke"),
+                    ("US/Michigan", "US/Michigan"),
+                    ("US/Mountain", "US/Mountain"),
+                    ("US/Pacific", "US/Pacific"),
+                    ("US/Samoa", "US/Samoa"),
+                    ("UTC", "UTC"),
+                    ("Universal", "Universal"),
+                    ("W-SU", "W-SU"),
+                    ("WET", "WET"),
+                    ("Zulu", "Zulu"),
+                ],
+                default="UTC",
+                max_length=255,
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="notification",
+            constraint=models.CheckConstraint(
+                check=models.Q(
+                    (
+                        "notification_type__in",
+                        [
+                            "FAVORITE",
+                            "REPLY",
+                            "MENTION",
+                            "TAG",
+                            "FOLLOW",
+                            "FOLLOW_REQUEST",
+                            "BOOST",
+                            "IMPORT",
+                            "ADD",
+                            "REPORT",
+                            "INVITE",
+                            "ACCEPT",
+                            "JOIN",
+                            "LEAVE",
+                            "REMOVE",
+                        ],
+                    )
+                ),
+                name="notification_type_valid",
+            ),
+        ),
+        migrations.AddField(
+            model_name="groupmemberinvitation",
+            name="group",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="user_invitations",
+                to="bookwyrm.group",
+            ),
+        ),
+        migrations.AddField(
+            model_name="groupmemberinvitation",
+            name="user",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="group_invitations",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AddField(
+            model_name="groupmember",
+            name="group",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="memberships",
+                to="bookwyrm.group",
+            ),
+        ),
+        migrations.AddField(
+            model_name="groupmember",
+            name="user",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="memberships",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AddField(
+            model_name="group",
+            name="user",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
+            ),
+        ),
+        migrations.AddField(
+            model_name="list",
+            name="group",
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="bookwyrm.group",
+            ),
+        ),
+        migrations.AddField(
+            model_name="notification",
+            name="related_group",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="notifications",
+                to="bookwyrm.group",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="groupmemberinvitation",
+            constraint=models.UniqueConstraint(
+                fields=("group", "user"), name="unique_invitation"
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="groupmember",
+            constraint=models.UniqueConstraint(
+                fields=("group", "user"), name="unique_membership"
+            ),
+        ),
+    ]

bookwyrm/migrations/0108_alter_user_preferred_language.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.2.5 on 2021-10-11 17:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0107_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("pt-br", "Português - Brasil (Brazilian Portuguese)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0109_status_edited_date.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.5 on 2021-10-15 15:54
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0108_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="status",
+            name="edited_date",
+            field=bookwyrm.models.fields.DateTimeField(blank=True, null=True),
+        ),
+    ]

bookwyrm/migrations/0110_auto_20211015_1734.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.5 on 2021-10-15 17:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0109_status_edited_date"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="quotation",
+            name="raw_quote",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="status",
+            name="raw_content",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

bookwyrm/migrations/0111_merge_0107_auto_20211016_0639_0110_auto_20211015_1734.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.5 on 2021-10-16 19:30
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0107_auto_20211016_0639"),
+        ("bookwyrm", "0110_auto_20211015_1734"),
+    ]
+
+    operations = []

bookwyrm/migrations/0112_auto_20211022_0844.py

@@ -0,0 +1,93 @@
+# Generated by Django 3.2.5 on 2021-10-22 08:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0111_merge_0107_auto_20211016_0639_0110_auto_20211015_1734"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="notification",
+            name="notification_type_valid",
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("BOOST", "Boost"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                ],
+                max_length=255,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("pt-br", "Português - Brasil (Brazilian Portuguese)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="notification",
+            constraint=models.CheckConstraint(
+                check=models.Q(
+                    (
+                        "notification_type__in",
+                        [
+                            "FAVORITE",
+                            "REPLY",
+                            "MENTION",
+                            "TAG",
+                            "FOLLOW",
+                            "FOLLOW_REQUEST",
+                            "BOOST",
+                            "IMPORT",
+                            "ADD",
+                            "REPORT",
+                            "INVITE",
+                            "ACCEPT",
+                            "JOIN",
+                            "LEAVE",
+                            "REMOVE",
+                            "GROUP_PRIVACY",
+                            "GROUP_NAME",
+                            "GROUP_DESCRIPTION",
+                        ],
+                    )
+                ),
+                name="notification_type_valid",
+            ),
+        ),
+    ]

bookwyrm/migrations/0113_auto_20211110_2104.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.2.5 on 2021-11-10 21:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0112_auto_20211022_0844"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importitem",
+            name="normalized_data",
+            field=models.JSONField(default={}),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="importjob",
+            name="mappings",
+            field=models.JSONField(default={}),
+            preserve_default=False,
+        ),
+    ]

bookwyrm/migrations/0114_importjob_source.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.5 on 2021-11-13 00:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0113_auto_20211110_2104"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importjob",
+            name="source",
+            field=models.CharField(default="Import", max_length=100),
+            preserve_default=False,
+        ),
+    ]

bookwyrm/migrations/0115_importitem_linked_review.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.5 on 2021-11-13 19:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0114_importjob_source"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importitem",
+            name="linked_review",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="bookwyrm.review",
+            ),
+        ),
+    ]

bookwyrm/migrations/0116_auto_20211114_1734.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.5 on 2021-11-14 17:34
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0115_importitem_linked_review"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="importjob",
+            name="task_id",
+        ),
+        migrations.AddField(
+            model_name="importjob",
+            name="updated_date",
+            field=models.DateTimeField(default=django.utils.timezone.now),
+        ),
+    ]

bookwyrm/migrations/0117_alter_user_preferred_language.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.2.5 on 2021-11-15 18:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0116_auto_20211114_1734"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "lietuvių (Lithuanian)"),
+                    ("pt-br", "Português - Brasil (Brazilian Portuguese)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0118_alter_user_preferred_language.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.2.5 on 2021-11-17 18:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0117_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("pt-br", "Português - Brasil (Brazilian Portuguese)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0119_user_feed_status_types.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.2.5 on 2021-11-24 10:15
+
+import bookwyrm.models.user
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0118_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="feed_status_types",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(
+                    choices=[
+                        ("review", "Reviews"),
+                        ("comment", "Comments"),
+                        ("quotation", "Quotations"),
+                        ("everything", "Everything else"),
+                    ],
+                    max_length=10,
+                ),
+                default=bookwyrm.models.user.get_feed_filter_choices,
+                size=8,
+            ),
+        ),
+    ]

bookwyrm/migrations/0120_list_embed_key.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.5 on 2021-12-04 10:55
+
+from django.db import migrations, models
+import uuid
+
+
+def gen_uuid(apps, schema_editor):
+    """sets an unique UUID for embed_key"""
+    book_lists = apps.get_model("bookwyrm", "List")
+    db_alias = schema_editor.connection.alias
+    for book_list in book_lists.objects.using(db_alias).all():
+        book_list.embed_key = uuid.uuid4()
+        book_list.save(broadcast=False)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0119_user_feed_status_types"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="list",
+            name="embed_key",
+            field=models.UUIDField(editable=False, null=True, unique=True),
+        ),
+        migrations.RunPython(gen_uuid, reverse_code=migrations.RunPython.noop),
+    ]

bookwyrm/migrations/0121_user_summary_keys.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.5 on 2021-12-22 11:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0120_list_embed_key"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="summary_keys",
+            field=models.JSONField(null=True),
+        ),
+    ]

bookwyrm/migrations/0122_alter_annualgoal_year.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.5 on 2022-01-04 18:59
+
+import bookwyrm.models.user
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0121_user_summary_keys"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="annualgoal",
+            name="year",
+            field=models.IntegerField(default=bookwyrm.models.user.get_current_year),
+        ),
+    ]

bookwyrm/migrations/0123_alter_user_preferred_language.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.5 on 2022-01-04 22:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0122_alter_annualgoal_year"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0124_auto_20220106_1759.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.2.10 on 2022-01-06 17:59
+
+from django.contrib.auth.models import AbstractUser
+from django.db import migrations
+
+
+def get_admins(apps, schema_editor):
+    """add any superusers to the "admin" group"""
+
+    db_alias = schema_editor.connection.alias
+    groups = apps.get_model("auth", "Group")
+    try:
+        group = groups.objects.using(db_alias).get(name="admin")
+    except groups.DoesNotExist:
+        # for tests
+        return
+
+    users = apps.get_model("bookwyrm", "User")
+    admins = users.objects.using(db_alias).filter(is_superuser=True)
+
+    for admin in admins:
+        admin.groups.add(group)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0123_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.RunPython(get_admins, reverse_code=migrations.RunPython.noop),
+    ]

bookwyrm/migrations/0125_alter_user_preferred_language.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.2.10 on 2022-01-09 01:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0124_auto_20220106_1759"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0126_auto_20220112_2315.py

@@ -0,0 +1,55 @@
+# Generated by Django 3.2.10 on 2022-01-12 23:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0125_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="annualgoal",
+            name="privacy",
+            field=models.CharField(
+                choices=[
+                    ("public", "Public"),
+                    ("unlisted", "Unlisted"),
+                    ("followers", "Followers"),
+                    ("direct", "Private"),
+                ],
+                default="public",
+                max_length=255,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="importjob",
+            name="privacy",
+            field=models.CharField(
+                choices=[
+                    ("public", "Public"),
+                    ("unlisted", "Unlisted"),
+                    ("followers", "Followers"),
+                    ("direct", "Private"),
+                ],
+                default="public",
+                max_length=255,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="user",
+            name="default_post_privacy",
+            field=models.CharField(
+                choices=[
+                    ("public", "Public"),
+                    ("unlisted", "Unlisted"),
+                    ("followers", "Followers"),
+                    ("direct", "Private"),
+                ],
+                default="public",
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0126_filelink_link_linkdomain.py

@@ -0,0 +1,144 @@
+# Generated by Django 3.2.10 on 2022-01-10 21:20
+
+import bookwyrm.models.activitypub_mixin
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0125_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="LinkDomain",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                ("updated_date", models.DateTimeField(auto_now=True)),
+                (
+                    "remote_id",
+                    bookwyrm.models.fields.RemoteIdField(
+                        max_length=255,
+                        null=True,
+                        validators=[bookwyrm.models.fields.validate_remote_id],
+                    ),
+                ),
+                ("domain", models.CharField(max_length=255, unique=True)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("approved", "Approved"),
+                            ("blocked", "Blocked"),
+                            ("pending", "Pending"),
+                        ],
+                        default="pending",
+                        max_length=50,
+                    ),
+                ),
+                ("name", models.CharField(max_length=100)),
+                (
+                    "reported_by",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="Link",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                ("updated_date", models.DateTimeField(auto_now=True)),
+                (
+                    "remote_id",
+                    bookwyrm.models.fields.RemoteIdField(
+                        max_length=255,
+                        null=True,
+                        validators=[bookwyrm.models.fields.validate_remote_id],
+                    ),
+                ),
+                ("url", bookwyrm.models.fields.URLField(max_length=255)),
+                (
+                    "added_by",
+                    bookwyrm.models.fields.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+                (
+                    "domain",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="links",
+                        to="bookwyrm.linkdomain",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=(bookwyrm.models.activitypub_mixin.ActivitypubMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name="FileLink",
+            fields=[
+                (
+                    "link_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.link",
+                    ),
+                ),
+                ("filetype", bookwyrm.models.fields.CharField(max_length=5)),
+                (
+                    "book",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="file_links",
+                        to="bookwyrm.book",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.link",),
+        ),
+    ]

bookwyrm/migrations/0127_auto_20220110_2211.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.10 on 2022-01-10 22:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0126_filelink_link_linkdomain"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="report",
+            name="self_report",
+        ),
+        migrations.AddField(
+            model_name="report",
+            name="links",
+            field=models.ManyToManyField(blank=True, to="bookwyrm.Link"),
+        ),
+    ]

bookwyrm/migrations/0128_merge_0126_auto_20220112_2315_0127_auto_20220110_2211.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.10 on 2022-01-13 01:14
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0126_auto_20220112_2315"),
+        ("bookwyrm", "0127_auto_20220110_2211"),
+    ]
+
+    operations = []

bookwyrm/migrations/0129_auto_20220117_1716.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.2.10 on 2022-01-17 17:16
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0128_merge_0126_auto_20220112_2315_0127_auto_20220110_2211"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="filelink",
+            name="availability",
+            field=bookwyrm.models.fields.CharField(
+                choices=[
+                    ("free", "Free"),
+                    ("purchase", "Purchasable"),
+                    ("loan", "Available for loan"),
+                ],
+                default="free",
+                max_length=100,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="filelink",
+            name="filetype",
+            field=bookwyrm.models.fields.CharField(max_length=50),
+        ),
+    ]

bookwyrm/migrations/0130_alter_listitem_notes.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.10 on 2022-01-24 20:01
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0129_auto_20220117_1716"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="listitem",
+            name="notes",
+            field=bookwyrm.models.fields.TextField(
+                blank=True, max_length=300, null=True
+            ),
+        ),
+    ]

bookwyrm/migrations/0130_alter_user_preferred_language.py

@@ -0,0 +1,37 @@
+# Generated by Django 3.2.10 on 2022-01-24 17:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0129_auto_20220117_1716"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("sv-se", "Swedish (Svenska)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0131_merge_20220125_1644.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.10 on 2022-01-25 16:44
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0130_alter_listitem_notes"),
+        ("bookwyrm", "0130_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0132_alter_user_preferred_language.py

@@ -0,0 +1,37 @@
+# Generated by Django 3.2.10 on 2022-02-02 20:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0131_merge_20220125_1644"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0133_alter_listitem_notes.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.11 on 2022-02-04 20:06
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0132_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="listitem",
+            name="notes",
+            field=bookwyrm.models.fields.HtmlField(
+                blank=True, max_length=300, null=True
+            ),
+        ),
+    ]

bookwyrm/migrations/0134_announcement_display_type.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.11 on 2022-02-11 18:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0133_alter_listitem_notes"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="announcement",
+            name="display_type",
+            field=models.CharField(
+                choices=[
+                    ("white-ter", "None"),
+                    ("primary-light", "Primary"),
+                    ("success-light", "Success"),
+                    ("link-light", "Link"),
+                    ("warning-light", "Warning"),
+                    ("danger-light", "Danger"),
+                ],
+                default="white-ter",
+                max_length=20,
+            ),
+        ),
+    ]

bookwyrm/migrations/0135_auto_20220217_1624.py

@@ -0,0 +1,58 @@
+# Generated by Django 3.2.12 on 2022-02-17 16:24
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0134_announcement_display_type"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="author",
+            old_name="viaf_id",
+            new_name="viaf",
+        ),
+        migrations.RemoveField(
+            model_name="edition",
+            name="asin",
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="asin",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="wikidata",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="asin",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="viaf",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="wikidata",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+    ]

bookwyrm/migrations/0136_auto_20220217_1708.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.12 on 2022-02-17 17:08
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0135_auto_20220217_1624"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="admin_code",
+            field=models.CharField(default=uuid.uuid4, max_length=50),
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="install_mode",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0137_alter_sitesettings_allow_registration.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.12 on 2022-02-17 19:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0136_auto_20220217_1708"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="sitesettings",
+            name="allow_registration",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0138_automod.py

@@ -0,0 +1,39 @@
+# Generated by Django 3.2.12 on 2022-02-24 18:59
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0137_alter_sitesettings_allow_registration"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="AutoMod",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("string_match", models.CharField(max_length=200, unique=True)),
+                ("flag_users", models.BooleanField(default=True)),
+                ("flag_statuses", models.BooleanField(default=True)),
+                (
+                    "created_by",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+        ),
+    ]

bookwyrm/migrations/0139_report_status.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.12 on 2022-02-24 20:41
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def set_report_statuses(apps, schema_editor):
+    """copy over status fields"""
+    db_alias = schema_editor.connection.alias
+    report_model = apps.get_model("bookwyrm", "Report")
+    reports = report_model.objects.using(db_alias).filter(statuses__isnull=False)
+    for report in reports:
+        report.status = report.statuses.first()
+        report.save()
+
+
+def set_reverse(apps, schema_editor):
+    """copy over status fields"""
+    db_alias = schema_editor.connection.alias
+    report_model = apps.get_model("bookwyrm", "Report")
+    reports = report_model.objects.using(db_alias).filter(status__isnull=False)
+    for report in reports:
+        report.statuses.set(report.status)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0138_automod"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="report",
+            name="status",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="reports",
+                to="bookwyrm.status",
+            ),
+        ),
+        migrations.RunPython(set_report_statuses, reverse_code=set_reverse),
+    ]

bookwyrm/migrations/0140_remove_report_statuses.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.12 on 2022-02-24 20:43
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0139_report_status"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="report",
+            name="statuses",
+        ),
+    ]

bookwyrm/migrations/0141_alter_report_status.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.12 on 2022-02-24 20:50
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0140_remove_report_statuses"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="report",
+            name="status",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="bookwyrm.status",
+            ),
+        ),
+    ]

bookwyrm/migrations/0142_auto_20220227_1752.py

@@ -0,0 +1,68 @@
+# Generated by Django 3.2.12 on 2022-02-27 17:52
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def add_default_themes(apps, schema_editor):
+    """add light and dark themes"""
+    db_alias = schema_editor.connection.alias
+    theme_model = apps.get_model("bookwyrm", "Theme")
+    theme_model.objects.using(db_alias).create(
+        name="BookWyrm Light",
+        path="css/themes/bookwyrm-light.scss",
+    )
+    theme_model.objects.using(db_alias).create(
+        name="BookWyrm Dark",
+        path="css/themes/bookwyrm-dark.scss",
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0141_alter_report_status"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Theme",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                ("name", models.CharField(max_length=50, unique=True)),
+                ("path", models.CharField(max_length=50, unique=True)),
+            ],
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="default_theme",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="bookwyrm.theme",
+            ),
+        ),
+        migrations.AddField(
+            model_name="user",
+            name="theme",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="bookwyrm.theme",
+            ),
+        ),
+        migrations.RunPython(
+            add_default_themes, reverse_code=migrations.RunPython.noop
+        ),
+    ]

bookwyrm/migrations/0142_user_hide_follows.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.12 on 2022-02-28 19:44
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0141_alter_report_status"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="hide_follows",
+            field=bookwyrm.models.fields.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0143_merge_0142_auto_20220227_1752_0142_user_hide_follows.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.12 on 2022-02-28 21:28
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0142_auto_20220227_1752"),
+        ("bookwyrm", "0142_user_hide_follows"),
+    ]
+
+    operations = []

bookwyrm/migrations/0144_alter_announcement_display_type.py

@@ -0,0 +1,39 @@
+# Generated by Django 3.2.12 on 2022-03-01 18:46
+
+from django.db import migrations, models
+
+
+def remove_white(apps, schema_editor):
+    """don't hardcode white announcements"""
+    db_alias = schema_editor.connection.alias
+    announcement_model = apps.get_model("bookwyrm", "Announcement")
+    announcement_model.objects.using(db_alias).filter(display_type="white-ter").update(
+        display_type=None
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0143_merge_0142_auto_20220227_1752_0142_user_hide_follows"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="announcement",
+            name="display_type",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("primary-light", "Primary"),
+                    ("success-light", "Success"),
+                    ("link-light", "Link"),
+                    ("warning-light", "Warning"),
+                    ("danger-light", "Danger"),
+                ],
+                max_length=20,
+                null=True,
+            ),
+        ),
+        migrations.RunPython(remove_white, reverse_code=migrations.RunPython.noop),
+    ]

bookwyrm/migrations/0145_sitesettings_version.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.12 on 2022-03-16 18:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0144_alter_announcement_display_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="version",
+            field=models.CharField(blank=True, max_length=10, null=True),
+        ),
+    ]