Merge branch 'main' into debug-toolbar

.dockerignore

@@ -5,3 +5,4 @@ __pycache__
 .git
 .github
 .pytest*
+.env

.env.example

@@ -8,7 +8,7 @@ USE_HTTPS=true
 DOMAIN=your.domain.here
 EMAIL=your@email.here
 
-# Instance defualt language (see options at bookwyrm/settings.py "LANGUAGES"
+# Instance default language (see options at bookwyrm/settings.py "LANGUAGES"
 LANGUAGE_CODE="en-us"
 # Used for deciding which editions to prefer
 DEFAULT_LANGUAGE="English"
@@ -32,12 +32,17 @@ REDIS_ACTIVITY_PORT=6379
 REDIS_ACTIVITY_PASSWORD=redispassword345
 # Optional, use a different redis database (defaults to 0)
 # REDIS_ACTIVITY_DB_INDEX=0
+# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket
+# REDIS_ACTIVITY_URL=
 
 # Redis as celery broker
+REDIS_BROKER_HOST=redis_broker
 REDIS_BROKER_PORT=6379
 REDIS_BROKER_PASSWORD=redispassword123
 # Optional, use a different redis database (defaults to 0)
 # REDIS_BROKER_DB_INDEX=0
+# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket
+# REDIS_BROKER_URL=
 
 # Monitoring for celery
 FLOWER_PORT=8888
@@ -60,7 +65,7 @@ SEARCH_TIMEOUT=5
 QUERY_TIMEOUT=5
 
 # Thumbnails Generation
-ENABLE_THUMBNAIL_GENERATION=false
+ENABLE_THUMBNAIL_GENERATION=true
 
 # S3 configuration
 USE_S3=false
@@ -77,6 +82,12 @@ AWS_SECRET_ACCESS_KEY=
 # AWS_S3_REGION_NAME=None         # "fr-par"
 # AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
 
+# Commented are example values if you use Azure Blob Storage
+# USE_AZURE=true
+# AZURE_ACCOUNT_NAME=     # "example-account-name"
+# AZURE_ACCOUNT_KEY=      # "base64-encoded-access-key"
+# AZURE_CONTAINER=        # "example-blob-container-name"
+# AZURE_CUSTOM_DOMAIN=    # "example-account-name.blob.core.windows.net"
 
 # Preview image generation can be computing and storage intensive
 ENABLE_PREVIEW_IMAGES=False
@@ -108,3 +119,28 @@ OTEL_EXPORTER_OTLP_ENDPOINT=
 OTEL_EXPORTER_OTLP_HEADERS=
 # Service name to identify your app
 OTEL_SERVICE_NAME=
+
+# Set HTTP_X_FORWARDED_PROTO ONLY to true if you know what you are doing.
+# Only use it if your proxy is "swallowing" if the original request was made
+# via https. Please refer to the Django-Documentation and assess the risks
+# for your instance:
+# https://docs.djangoproject.com/en/3.2/ref/settings/#secure-proxy-ssl-header
+HTTP_X_FORWARDED_PROTO=false
+
+# TOTP settings
+# TWO_FACTOR_LOGIN_VALIDITY_WINDOW sets the number of codes either side
+# which will be accepted.
+TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2
+TWO_FACTOR_LOGIN_MAX_SECONDS=60
+
+# Additional hosts to allow in the Content-Security-Policy, "self" (should be DOMAIN)
+# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
+# Value should be a comma-separated list of host names.
+CSP_ADDITIONAL_HOSTS=
+
+# Time before being logged out (in seconds)
+# SESSION_COOKIE_AGE=2592000   # current default: 30 days
+
+# Maximum allowed memory for file uploads (increase if users are having trouble
+# uploading BookWyrm export files).
+# DATA_UPLOAD_MAX_MEMORY_MiB=100

.github/workflows/black.yml

@@ -1,15 +0,0 @@
-name: Python Formatting (run ./bw-dev black to fix)
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - uses: psf/black@21.4b2

.github/workflows/codeql-analysis.yml

@@ -36,11 +36,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,7 +51,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -65,4 +65,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v3

.github/workflows/curlylint.yaml

@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install curlylint
         run: pip install curlylint

.github/workflows/django-tests.yml

@@ -1,60 +0,0 @@
-name: Run Python Tests
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-20.04
-    services:
-      postgres:
-        image: postgres:13
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: hunter2
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Run Tests
-      env:
-        SECRET_KEY: beepbeep
-        DEBUG: false
-        USE_HTTPS: true
-        DOMAIN: your.domain.here
-        BOOKWYRM_DATABASE_BACKEND: postgres
-        MEDIA_ROOT: images/
-        POSTGRES_PASSWORD: hunter2
-        POSTGRES_USER: postgres
-        POSTGRES_DB: github_actions
-        POSTGRES_HOST: 127.0.0.1
-        CELERY_BROKER: ""
-        REDIS_BROKER_PORT: 6379
-        REDIS_BROKER_PASSWORD: beep
-        USE_DUMMY_CACHE: true
-        FLOWER_PORT: 8888
-        EMAIL_HOST: "smtp.mailgun.org"
-        EMAIL_PORT: 587
-        EMAIL_HOST_USER: ""
-        EMAIL_HOST_PASSWORD: ""
-        EMAIL_USE_TLS: true
-        ENABLE_PREVIEW_IMAGES: false
-        ENABLE_THUMBNAIL_GENERATION: true
-      run: |
-        pytest -n 3

.github/workflows/lint-frontend.yaml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install modules
         run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint

.github/workflows/prettier.yaml

@@ -14,10 +14,10 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install modules
-        run: npm install prettier
+        run: npm install prettier@2.5.1
 
       - name: Run Prettier
         run: npx prettier --check bookwyrm/static/js/*.js

.github/workflows/pylint.yml

@@ -1,27 +0,0 @@
-name: Pylint
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Analysing the code with pylint
-      run: |
-        pylint bookwyrm/
-

.github/workflows/python.yml

@@ -0,0 +1,99 @@
+name: Python
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+# overrides for .env.example
+env:
+  POSTGRES_HOST: 127.0.0.1
+  PGPORT: 5432
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: hunter2
+  POSTGRES_DB: github_actions
+  SECRET_KEY: beepbeep
+  EMAIL_HOST_USER: ""
+  EMAIL_HOST_PASSWORD: ""
+
+jobs:
+  pytest:
+    name: Tests (pytest)
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:13
+        env: # does not inherit from jobs.build.env
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: hunter2
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pytest-github-actions-annotate-failures
+    - name: Set up .env
+      run: cp .env.example .env
+    - name: Check migrations up-to-date
+      run: python ./manage.py makemigrations --check
+    - name: Run Tests
+      run: pytest -n 3
+
+  pylint:
+    name: Linting (pylint)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Analyse code with pylint
+      run: pylint bookwyrm/
+
+  mypy:
+    name: Typing (mypy)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Set up .env
+      run: cp .env.example .env
+    - name: Analyse code with mypy
+      run: mypy bookwyrm celerywyrm
+
+  black:
+    name: Formatting (black; run ./bw-dev black to fix)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+    - uses: psf/black@stable
+      with:
+        version: "22.*"

.gitignore

@@ -16,6 +16,7 @@
 # BookWyrm
 .env
 /images/
+/static/
 bookwyrm/static/css/bookwyrm.css
 bookwyrm/static/css/themes/
 !bookwyrm/static/css/themes/bookwyrm-*.scss

.prettierrc

@@ -0,0 +1 @@
+'trailingComma': 'es5'

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.11
 
 ENV PYTHONUNBUFFERED 1
 

FEDERATION.md

@@ -0,0 +1,334 @@
+# Federation
+
+BookWyrm uses the [ActivityPub](http://activitypub.rocks/) protocol to send and receive user activity between other BookWyrm instances and other services that implement ActivityPub. To handle book data, BookWyrm has a handful of extended Activity types which are not part of the standard, but are legible to other BookWyrm instances.
+
+## Activities and Objects
+
+### Users and relationships
+User relationship interactions follow the standard ActivityPub spec.
+
+- `Follow`: request to receive statuses from a user, and view their statuses that have followers-only privacy
+- `Accept`: approves a `Follow` and finalizes the relationship
+- `Reject`: denies a `Follow`
+- `Block`: prevent users from seeing one another's statuses, and prevents the blocked user from viewing the actor's profile
+- `Update`: updates a user's profile and settings
+- `Delete`: deactivates a user
+- `Undo`: reverses a `Block` or `Follow`
+
+### Activities
+- `Create/Status`: saves a new status in the database.
+- `Delete/Status`: Removes a status
+- `Like/Status`: Creates a favorite on the status
+- `Announce/Status`: Boosts the status into the actor's timeline
+- `Undo/*`,: Reverses an `Announce`, `Like`, or `Move`
+- `Move/User`: Moves a user from one ActivityPub id to another.
+
+### Collections
+User's books and lists are represented by [`OrderedCollection`](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection)
+
+### Statuses
+
+BookWyrm is focused on book reading activities - it is not a general-purpose messaging application. For this reason, BookWyrm only accepts status `Create` activities if they are:
+
+- Direct messages (i.e., `Note`s with the privacy level `direct`, which mention a local user),
+- Related to a book (of a custom status type that includes the field `inReplyToBook`),
+- Replies to existing statuses saved in the database
+
+All other statuses will be received by the instance inbox, but by design **will not be delivered to user inboxes or displayed to users**.
+
+### Custom Object types
+
+With the exception of `Note`, the following object types are used in Bookwyrm but are not currently provided with a custom JSON-LD `@context` extension IRI. This is likely to change in future to make them true deserialisable JSON-LD objects.
+
+##### Note
+
+Within BookWyrm a `Note` is constructed according to [the ActivityStreams vocabulary](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-note), however `Note`s can only be created as direct messages or as replies to other statuses. As mentioned above, this also applies to incoming `Note`s.
+
+##### Review
+
+A `Review` is a status in response to a book (indicated by the `inReplyToBook` field), which has a title, body, and numerical rating between 0 (not rated) and 5.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/user/library_lurker/review/2",
+    "type": "Review",
+    "published": "2023-06-30T21:43:46.013132+00:00",
+    "attributedTo": "https://example.net/user/library_lurker",
+    "content": "<p>This is an enjoyable book with great characters.</p>",
+    "to": ["https://example.net/user/library_lurker/followers"],
+    "cc": [],
+    "replies": {
+        "id": "https://example.net/user/library_lurker/review/2/replies",
+        "type": "OrderedCollection",
+        "totalItems": 0,
+        "first": "https://example.net/user/library_lurker/review/2/replies?page=1",
+        "last": "https://example.net/user/library_lurker/review/2/replies?page=1",
+        "@context": "https://www.w3.org/ns/activitystreams"
+        },
+        "summary": "Spoilers ahead!",
+        "tag": [],
+        "attachment": [],
+        "sensitive": true,
+    "inReplyToBook": "https://example.net/book/1",
+    "name": "What a cracking read",
+    "rating": 4.5,
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Comment
+
+A `Comment` on a book mentions a book and has a message body, reading status, and progress indicator.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/user/library_lurker/comment/9",
+    "type": "Comment",
+    "published": "2023-06-30T21:43:46.013132+00:00",
+    "attributedTo": "https://example.net/user/library_lurker",
+    "content": "<p>This is a very enjoyable book so far.</p>",
+    "to": ["https://example.net/user/library_lurker/followers"],
+    "cc": [],
+    "replies": {
+        "id": "https://example.net/user/library_lurker/comment/9/replies",
+        "type": "OrderedCollection",
+        "totalItems": 0,
+        "first": "https://example.net/user/library_lurker/comment/9/replies?page=1",
+        "last": "https://example.net/user/library_lurker/comment/9/replies?page=1",
+        "@context": "https://www.w3.org/ns/activitystreams"
+        },
+        "summary": "Spoilers ahead!",
+        "tag": [],
+        "attachment": [],
+        "sensitive": true,
+    "inReplyToBook": "https://example.net/book/1",
+    "readingStatus": "reading",
+    "progress": 25,
+    "progressMode": "PG",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Quotation
+
+A quotation (aka "quote") has a message body, an excerpt from a book including position as a page number or percentage indicator, and mentions a book.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/user/mouse/quotation/13",
+    "url": "https://example.net/user/mouse/quotation/13",
+    "inReplyTo": null,
+    "published": "2020-05-10T02:38:31.150343+00:00",
+    "attributedTo": "https://example.net/user/mouse",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+        ],
+    "cc": [
+        "https://example.net/user/mouse/followers"
+        ],
+    "sensitive": false,
+    "content": "I really like this quote",
+    "type": "Quotation",
+    "replies": {
+        "id": "https://example.net/user/mouse/quotation/13/replies",
+        "type": "Collection",
+        "first": {
+            "type": "CollectionPage",
+            "next": "https://example.net/user/mouse/quotation/13/replies?only_other_accounts=true&page=true",
+            "partOf": "https://example.net/user/mouse/quotation/13/replies",
+            "items": []
+            }
+        },
+    "inReplyToBook": "https://example.net/book/1",
+    "quote": "To be or not to be, that is the question.",
+    "position": 50,
+    "positionMode": "PCT",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+### Custom Objects
+
+##### Work
+A particular book, a "work" in the [FRBR](https://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records) sense.
+
+Example:
+
+```json
+{
+    "id": "https://bookwyrm.social/book/5988",
+    "type": "Work",
+    "authors": [
+        "https://bookwyrm.social/author/417"
+    ],
+    "first_published_date": null,
+    "published_date": null,
+    "title": "Piranesi",
+    "sort_title": null,
+    "subtitle": null,
+    "description": "**From the *New York Times* bestselling author of *Jonathan Strange & Mr. Norrell*, an intoxicating, hypnotic new novel set in a dreamlike alternative reality.",
+    "languages": [],
+    "series": null,
+    "series_number": null,
+    "subjects": [
+        "English literature"
+    ],
+    "subject_places": [],
+    "openlibrary_key": "OL20893680W",
+    "librarything_key": null,
+    "goodreads_key": null,
+    "attachment": [
+        {
+            "url": "https://bookwyrm.social/images/covers/10226290-M.jpg",
+            "type": "Image"
+        }
+    ],
+    "lccn": null,
+    "editions": [
+        "https://bookwyrm.social/book/5989"
+    ],
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Edition
+A particular _manifestation_ of a Work, in the [FRBR](https://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records) sense.
+
+Example:
+
+```json
+{
+    "id": "https://bookwyrm.social/book/5989",
+    "lastEditedBy": "https://example.net/users/rat",
+    "type": "Edition",
+    "authors": [
+        "https://bookwyrm.social/author/417"
+    ],
+    "first_published_date": null,
+    "published_date": "2020-09-15T00:00:00+00:00",
+    "title": "Piranesi",
+    "sort_title": null,
+    "subtitle": null,
+    "description": "Piranesi's house is no ordinary building; its rooms are infinite, its corridors endless, its walls are lined with thousands upon thousands of statues, each one different from all the others.",
+    "languages": [
+        "English"
+    ],
+    "series": null,
+    "series_number": null,
+    "subjects": [],
+    "subject_places": [],
+    "openlibrary_key": "OL29486417M",
+    "librarything_key": null,
+    "goodreads_key": null,
+    "isfdb": null,
+    "attachment": [
+        {
+            "url": "https://bookwyrm.social/images/covers/50202953._SX318_.jpg",
+            "type": "Image"
+        }
+    ],
+    "isbn_10": "1526622424",
+    "isbn_13": "9781526622426",
+    "oclc_number": null,
+    "asin": null,
+    "pages": 272,
+    "physical_format": null,
+    "publishers": [
+        "Bloomsbury Publishing Plc"
+    ],
+    "work": "https://bookwyrm.social/book/5988",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### Shelf
+
+A user's book collection. By default, every user has a `to-read`, `reading`, `read`, and `stopped-reading` shelf which are used to track reading progress. Users may create an unlimited number of additional shelves with their own ids.
+
+Example
+
+```json
+{
+    "id": "https://example.net/user/avid_reader/books/extraspecialbooks-5",
+    "type": "Shelf",
+    "totalItems": 0,
+    "first": "https://example.net/user/avid_reader/books/extraspecialbooks-5?page=1",
+    "last": "https://example.net/user/avid_reader/books/extraspecialbooks-5?page=1",
+    "name": "Extra special books",
+    "owner": "https://example.net/user/avid_reader",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "cc": [
+        "https://example.net/user/avid_reader/followers"
+    ],
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### List
+
+A collection of books that may have items contributed by users other than the one who created the list.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/list/1",
+    "type": "BookList",
+    "totalItems": 0,
+    "first": "https://example.net/list/1?page=1",
+    "last": "https://example.net/list/1?page=1",
+    "name": "My cool list",
+    "owner": "https://example.net/user/avid_reader",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "cc": [
+        "https://example.net/user/avid_reader/followers"
+    ],
+    "summary": "A list of books I like.",
+    "curation": "curated",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### Activities
+
+- `Create`: Adds a shelf or list to the database.
+- `Delete`: Removes a shelf or list.
+- `Add`: Adds a book to a shelf or list.
+- `Remove`: Removes a book from a shelf or list.
+
+## Alternative Serialization
+Because BookWyrm uses custom object types that aren't listed in [the standard ActivityStreams Vocabulary](https://www.w3.org/TR/activitystreams-vocabulary), some statuses are transformed into standard types when sent to or viewed by non-BookWyrm services. `Review`s are converted into `Article`s, and `Comment`s and `Quotation`s are converted into `Note`s, with a link to the book and the cover image attached.
+
+In future this may be done with [JSON-LD type arrays](https://www.w3.org/TR/json-ld/#specifying-the-type) instead.
+
+## Other extensions
+
+### Webfinger
+
+Bookwyrm uses the [Webfinger](https://datatracker.ietf.org/doc/html/rfc7033) standard to identify and disambiguate fediverse actors. The [Webfinger documentation on the Mastodon project](https://docs.joinmastodon.org/spec/webfinger/) provides a good overview of how Webfinger is used.
+
+### HTTP Signatures
+
+Bookwyrm uses and requires HTTP signatures for all `POST` requests. `GET` requests are not signed by default, but if Bookwyrm receives a `403` response to a `GET` it will re-send the request, signed by the default server user. This usually will have a user id of `https://example.net/user/bookwyrm.instance.actor`
+
+#### publicKey id
+
+In older versions of Bookwyrm the `publicKey.id` was incorrectly listed in request headers as `https://example.net/user/username#main-key`. As of v0.6.3 the id is now listed correctly, as `https://example.net/user/username/#main-key`. In most ActivityPub implementations this will make no difference as the URL will usually resolve to the same place.
+
+### NodeInfo
+
+Bookwyrm uses the [NodeInfo](http://nodeinfo.diaspora.software/) standard to provide statistics and version information for each instance.
+
+## Further Documentation
+
+See [docs.joinbookwyrm.com/](https://docs.joinbookwyrm.com/) for more documentation.

README.md

@@ -10,7 +10,6 @@ BookWyrm is a social network for tracking your reading, talking about books, wri
 ## Links
 
 [![Mastodon Follow](https://img.shields.io/mastodon/follow/000146121?domain=https%3A%2F%2Ftech.lgbt&style=social)](https://tech.lgbt/@bookwyrm)
-[![Twitter Follow](https://img.shields.io/twitter/follow/BookWyrmSocial?style=social)](https://twitter.com/BookWyrmSocial)
 
  - [Project homepage](https://joinbookwyrm.com/)
  - [Support](https://patreon.com/bookwyrm)
@@ -37,7 +36,7 @@ Keep track of what books you've read, and what books you'd like to read in the f
 Federation allows you to interact with users on other instances and services, and also shares metadata about books and authors, which collaboratively builds a decentralized database of books.
 
 ### Privacy and moderation
-Users and administrators can control who can see thier posts and what other instances to federate with.
+Users and administrators can control who can see their posts and what other instances to federate with.
 
 ## Tech Stack
 Web backend

VERSION

@@ -0,0 +1 @@
+0.7.3

bookwyrm/activitypub/__init__.py

@@ -3,8 +3,12 @@ import inspect
 import sys
 
 from .base_activity import ActivityEncoder, Signature, naive_parse
-from .base_activity import Link, Mention
+from .base_activity import Link, Mention, Hashtag
-from .base_activity import ActivitySerializerError, resolve_remote_id
+from .base_activity import (
+    ActivitySerializerError,
+    resolve_remote_id,
+    get_representative,
+)
 from .image import Document, Image
 from .note import Note, GeneratedNote, Article, Comment, Quotation
 from .note import Review, Rating
@@ -19,6 +23,7 @@ from .verbs import Create, Delete, Undo, Update
 from .verbs import Follow, Accept, Reject, Block
 from .verbs import Add, Remove
 from .verbs import Announce, Like
+from .verbs import Move
 
 # this creates a list of all the Activity types that we can serialize,
 # so when an Activity comes in from outside, we can check if it's known

bookwyrm/activitypub/base_activity.py

@@ -1,16 +1,28 @@
 """ basics for an activitypub serializer """
+from __future__ import annotations
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
 import logging
+from typing import Optional, Union, TypeVar, overload, Any
+
+import requests
 
 from django.apps import apps
 from django.db import IntegrityError, transaction
+from django.utils.http import http_date
 
+from bookwyrm import models
 from bookwyrm.connectors import ConnectorException, get_data
-from bookwyrm.tasks import app, MEDIUM
+from bookwyrm.models import base_model
+from bookwyrm.signatures import make_signature
+from bookwyrm.settings import DOMAIN, INSTANCE_ACTOR_USERNAME
+from bookwyrm.tasks import app, MISC
 
 logger = logging.getLogger(__name__)
 
+# pylint: disable=invalid-name
+TBookWyrmModel = TypeVar("TBookWyrmModel", bound=base_model.BookWyrmModel)
+
 
 class ActivitySerializerError(ValueError):
     """routine problems serializing activitypub json"""
@@ -60,7 +72,13 @@ class ActivityObject:
     id: str
     type: str
 
-    def __init__(self, activity_objects=None, **kwargs):
+    def __init__(
+        self,
+        activity_objects: Optional[
+            dict[str, Union[str, list[str], ActivityObject, base_model.BookWyrmModel]]
+        ] = None,
+        **kwargs: Any,
+    ):
         """this lets you pass in an object with fields that aren't in the
         dataclass, which it ignores. Any field in the dataclass is required or
         has a default value"""
@@ -95,16 +113,34 @@ class ActivityObject:
 
     # pylint: disable=too-many-locals,too-many-branches,too-many-arguments
     def to_model(
-        self, model=None, instance=None, allow_create=True, save=True, overwrite=True
+        self,
-    ):
+        model: Optional[type[TBookWyrmModel]] = None,
-        """convert from an activity to a model instance"""
+        instance: Optional[TBookWyrmModel] = None,
+        allow_create: bool = True,
+        save: bool = True,
+        overwrite: bool = True,
+        allow_external_connections: bool = True,
+    ) -> Optional[TBookWyrmModel]:
+        """convert from an activity to a model instance. Args:
+        model: the django model that this object is being converted to
+            (will guess if not known)
+        instance: an existing database entry that is going to be updated by
+            this activity
+        allow_create: whether a new object should be created if there is no
+            existing object is provided or found matching the remote_id
+        save: store in the database if true, return an unsaved model obj if false
+        overwrite: replace fields in the database with this activity if true,
+            only update blank fields if false
+        allow_external_connections: look up missing data if true,
+            throw an exception if false and an external connection is needed
+        """
         model = model or get_model_from_type(self.type)
 
         # only reject statuses if we're potentially creating them
         if (
             allow_create
             and hasattr(model, "ignore_activity")
-            and model.ignore_activity(self)
+            and model.ignore_activity(self, allow_external_connections)
         ):
             return None
 
@@ -122,7 +158,10 @@ class ActivityObject:
         for field in instance.simple_fields:
             try:
                 changed = field.set_field_from_activity(
-                    instance, self, overwrite=overwrite
+                    instance,
+                    self,
+                    overwrite=overwrite,
+                    allow_external_connections=allow_external_connections,
                 )
                 if changed:
                     update_fields.append(field.name)
@@ -133,7 +172,11 @@ class ActivityObject:
         # too early and jank up users
         for field in instance.image_fields:
             changed = field.set_field_from_activity(
-                instance, self, save=save, overwrite=overwrite
+                instance,
+                self,
+                save=save,
+                overwrite=overwrite,
+                allow_external_connections=allow_external_connections,
             )
             if changed:
                 update_fields.append(field.name)
@@ -156,8 +199,12 @@ class ActivityObject:
 
             # add many to many fields, which have to be set post-save
             for field in instance.many_to_many_fields:
-                # mention books/users, for example
+                # mention books/users/hashtags, for example
-                field.set_field_from_activity(instance, self)
+                field.set_field_from_activity(
+                    instance,
+                    self,
+                    allow_external_connections=allow_external_connections,
+                )
 
         # reversed relationships in the models
         for (
@@ -190,10 +237,15 @@ class ActivityObject:
         omit = kwargs.get("omit", ())
         data = self.__dict__.copy()
         # recursively serialize
-        for (k, v) in data.items():
+        for k, v in data.items():
             try:
                 if issubclass(type(v), ActivityObject):
                     data[k] = v.serialize()
+                elif isinstance(v, list):
+                    data[k] = [
+                        e.serialize() if issubclass(type(e), ActivityObject) else e
+                        for e in v
+                    ]
             except TypeError:
                 pass
         data = {k: v for (k, v) in data.items() if v is not None and k not in omit}
@@ -202,7 +254,7 @@ class ActivityObject:
         return data
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MISC)
 @transaction.atomic
 def set_related_field(
     model_name, origin_model_name, related_field_name, related_remote_id, data
@@ -241,10 +293,10 @@ def set_related_field(
 
 def get_model_from_type(activity_type):
     """given the activity, what type of model"""
-    models = apps.get_models()
+    activity_models = apps.get_models()
     model = [
         m
-        for m in models
+        for m in activity_models
         if hasattr(m, "activity_serializer")
         and hasattr(m.activity_serializer, "type")
         and m.activity_serializer.type == activity_type
@@ -256,10 +308,48 @@ def get_model_from_type(activity_type):
     return model[0]
 
 
+# pylint: disable=too-many-arguments
+@overload
 def resolve_remote_id(
-    remote_id, model=None, refresh=False, save=True, get_activity=False
+    remote_id: str,
-):
+    model: type[TBookWyrmModel],
-    """take a remote_id and return an instance, creating if necessary"""
+    refresh: bool = False,
+    save: bool = True,
+    get_activity: bool = False,
+    allow_external_connections: bool = True,
+) -> TBookWyrmModel:
+    ...
+
+
+# pylint: disable=too-many-arguments
+@overload
+def resolve_remote_id(
+    remote_id: str,
+    model: Optional[str] = None,
+    refresh: bool = False,
+    save: bool = True,
+    get_activity: bool = False,
+    allow_external_connections: bool = True,
+) -> base_model.BookWyrmModel:
+    ...
+
+
+# pylint: disable=too-many-arguments
+def resolve_remote_id(
+    remote_id: str,
+    model: Optional[Union[str, type[base_model.BookWyrmModel]]] = None,
+    refresh: bool = False,
+    save: bool = True,
+    get_activity: bool = False,
+    allow_external_connections: bool = True,
+) -> base_model.BookWyrmModel:
+    """take a remote_id and return an instance, creating if necessary. Args:
+    remote_id: the unique url for looking up the object in the db or by http
+    model: a string or object representing the model that corresponds to the object
+    save: whether to return an unsaved database entry or a saved one
+    get_activity: whether to return the activitypub object or the model object
+    allow_external_connections: whether to make http connections
+    """
     if model:  # a bonus check we can do if we already know the model
         if isinstance(model, str):
             model = apps.get_model(f"bookwyrm.{model}", require_ready=True)
@@ -267,13 +357,26 @@ def resolve_remote_id(
         if result and not refresh:
             return result if not get_activity else result.to_activity_dataclass()
 
+    # The above block will return the object if it already exists in the database.
+    # If it doesn't, an external connection would be needed, so check if that's cool
+    if not allow_external_connections:
+        raise ActivitySerializerError(
+            "Unable to serialize object without making external HTTP requests"
+        )
+
     # load the data and create the object
     try:
         data = get_data(remote_id)
-    except ConnectorException:
+    except ConnectionError:
-        logger.exception("Could not connect to host for remote_id: %s", remote_id)
+        logger.info("Could not connect to host for remote_id: %s", remote_id)
         return None
-
+    except requests.HTTPError as e:
+        if (e.response is not None) and e.response.status_code == 401:
+            # This most likely means it's a mastodon with secure fetch enabled.
+            data = get_activitypub_data(remote_id)
+        else:
+            logger.info("Could not connect to host for remote_id: %s", remote_id)
+            return None
     # determine the model implicitly, if not provided
     # or if it's a model with subclasses like Status, check again
     if not model or hasattr(model.objects, "select_subclasses"):
@@ -292,6 +395,49 @@ def resolve_remote_id(
     return item.to_model(model=model, instance=result, save=save)
 
 
+def get_representative():
+    """Get or create an actor representing the instance
+    to sign outgoing HTTP GET requests"""
+    return models.User.objects.get_or_create(
+        username=f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}",
+        defaults=dict(
+            email="bookwyrm@localhost",
+            local=True,
+            localname=INSTANCE_ACTOR_USERNAME,
+        ),
+    )[0]
+
+
+def get_activitypub_data(url):
+    """wrapper for request.get"""
+    now = http_date()
+    sender = get_representative()
+    if not sender.key_pair.private_key:
+        # this shouldn't happen. it would be bad if it happened.
+        raise ValueError("No private key found for sender")
+    try:
+        resp = requests.get(
+            url,
+            headers={
+                # pylint: disable=line-too-long
+                "Accept": 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+                "Date": now,
+                "Signature": make_signature("get", sender, url, now),
+            },
+            timeout=15,
+        )
+    except requests.RequestException:
+        raise ConnectorException()
+    if not resp.ok:
+        resp.raise_for_status()
+    try:
+        data = resp.json()
+    except ValueError:
+        raise ConnectorException()
+
+    return data
+
+
 @dataclass(init=False)
 class Link(ActivityObject):
     """for tagging a book in a status"""
@@ -306,7 +452,9 @@ class Link(ActivityObject):
 
     def serialize(self, **kwargs):
         """remove fields"""
-        omit = ("id", "type", "@context")
+        omit = ("id", "@context")
+        if self.type == "Link":
+            omit += ("type",)
         return super().serialize(omit=omit)
 
 
@@ -315,3 +463,10 @@ class Mention(Link):
     """a subtype of Link for mentioning an actor"""
 
     type: str = "Mention"
+
+
+@dataclass(init=False)
+class Hashtag(Link):
+    """a subtype of Link for mentioning a hashtag"""
+
+    type: str = "Hashtag"

bookwyrm/activitypub/book.py

@@ -1,6 +1,6 @@
 """ book and author data """
 from dataclasses import dataclass, field
-from typing import List
+from typing import Optional
 
 from .base_activity import ActivityObject
 from .image import Document
@@ -11,17 +11,17 @@ from .image import Document
 class BookData(ActivityObject):
     """shared fields for all book data and authors"""
 
-    openlibraryKey: str = None
+    openlibraryKey: Optional[str] = None
-    inventaireId: str = None
+    inventaireId: Optional[str] = None
-    librarythingKey: str = None
+    librarythingKey: Optional[str] = None
-    goodreadsKey: str = None
+    goodreadsKey: Optional[str] = None
-    bnfId: str = None
+    bnfId: Optional[str] = None
-    viaf: str = None
+    viaf: Optional[str] = None
-    wikidata: str = None
+    wikidata: Optional[str] = None
-    asin: str = None
+    asin: Optional[str] = None
-    lastEditedBy: str = None
+    aasin: Optional[str] = None
-    links: List[str] = field(default_factory=lambda: [])
+    isfdb: Optional[str] = None
-    fileLinks: List[str] = field(default_factory=lambda: [])
+    lastEditedBy: Optional[str] = None
 
 
 # pylint: disable=invalid-name
@@ -33,17 +33,19 @@ class Book(BookData):
     sortTitle: str = None
     subtitle: str = None
     description: str = ""
-    languages: List[str] = field(default_factory=lambda: [])
+    languages: list[str] = field(default_factory=list)
     series: str = ""
     seriesNumber: str = ""
-    subjects: List[str] = field(default_factory=lambda: [])
+    subjects: list[str] = field(default_factory=list)
-    subjectPlaces: List[str] = field(default_factory=lambda: [])
+    subjectPlaces: list[str] = field(default_factory=list)
 
-    authors: List[str] = field(default_factory=lambda: [])
+    authors: list[str] = field(default_factory=list)
     firstPublishedDate: str = ""
     publishedDate: str = ""
 
-    cover: Document = None
+    fileLinks: list[str] = field(default_factory=list)
+
+    cover: Optional[Document] = None
     type: str = "Book"
 
 
@@ -56,10 +58,10 @@ class Edition(Book):
     isbn10: str = ""
     isbn13: str = ""
     oclcNumber: str = ""
-    pages: int = None
+    pages: Optional[int] = None
     physicalFormat: str = ""
     physicalFormatDetail: str = ""
-    publishers: List[str] = field(default_factory=lambda: [])
+    publishers: list[str] = field(default_factory=list)
     editionRank: int = 0
 
     type: str = "Edition"
@@ -71,7 +73,7 @@ class Work(Book):
     """work instance of a book object"""
 
     lccn: str = ""
-    editions: List[str] = field(default_factory=lambda: [])
+    editions: list[str] = field(default_factory=list)
     type: str = "Work"
 
 
@@ -81,12 +83,13 @@ class Author(BookData):
     """author of a book"""
 
     name: str
-    isni: str = None
+    isni: Optional[str] = None
-    viafId: str = None
+    viafId: Optional[str] = None
-    gutenbergId: str = None
+    gutenbergId: Optional[str] = None
-    born: str = None
+    born: Optional[str] = None
-    died: str = None
+    died: Optional[str] = None
-    aliases: List[str] = field(default_factory=lambda: [])
+    aliases: list[str] = field(default_factory=list)
     bio: str = ""
     wikipediaLink: str = ""
     type: str = "Author"
+    website: str = ""

bookwyrm/activitypub/note.py

@@ -1,9 +1,12 @@
 """ note serializer and children thereof """
 from dataclasses import dataclass, field
 from typing import Dict, List
-from django.apps import apps
+import re
 
-from .base_activity import ActivityObject, Link
+from django.apps import apps
+from django.db import IntegrityError, transaction
+
+from .base_activity import ActivityObject, ActivitySerializerError, Link
 from .image import Document
 
 
@@ -38,6 +41,47 @@ class Note(ActivityObject):
     updated: str = None
     type: str = "Note"
 
+    # pylint: disable=too-many-arguments
+    def to_model(
+        self,
+        model=None,
+        instance=None,
+        allow_create=True,
+        save=True,
+        overwrite=True,
+        allow_external_connections=True,
+    ):
+        instance = super().to_model(
+            model, instance, allow_create, save, overwrite, allow_external_connections
+        )
+
+        if instance is None:
+            return instance
+
+        # Replace links to hashtags in content with local URLs
+        changed_content = False
+        for hashtag in instance.mention_hashtags.all():
+            updated_content = re.sub(
+                rf'(<a href=")[^"]*(" data-mention="hashtag">{hashtag.name}</a>)',
+                rf"\1{hashtag.remote_id}\2",
+                instance.content,
+                flags=re.IGNORECASE,
+            )
+            if instance.content != updated_content:
+                instance.content = updated_content
+                changed_content = True
+
+        if not save or not changed_content:
+            return instance
+
+        with transaction.atomic():
+            try:
+                instance.save(broadcast=False, update_fields=["content"])
+            except IntegrityError as e:
+                raise ActivitySerializerError(e)
+
+        return instance
+
 
 @dataclass(init=False)
 class Article(Note):

bookwyrm/activitypub/person.py

@@ -1,5 +1,5 @@
 """ actor serializer """
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import Dict
 
 from .base_activity import ActivityObject
@@ -35,9 +35,11 @@ class Person(ActivityObject):
     endpoints: Dict = None
     name: str = None
     summary: str = None
-    icon: Image = field(default_factory=lambda: {})
+    icon: Image = None
     bookwyrmUser: bool = False
     manuallyApprovesFollowers: str = False
     discoverable: str = False
     hideFollows: str = False
+    movedTo: str = None
+    alsoKnownAs: dict[str] = None
     type: str = "Person"

bookwyrm/activitypub/verbs.py

@@ -14,12 +14,12 @@ class Verb(ActivityObject):
     actor: str
     object: ActivityObject
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """usually we just want to update and save"""
         # self.object may return None if the object is invalid in an expected way
         # ie, Question type
         if self.object:
-            self.object.to_model()
+            self.object.to_model(allow_external_connections=allow_external_connections)
 
 
 # pylint: disable=invalid-name
@@ -42,7 +42,7 @@ class Delete(Verb):
     cc: List[str] = field(default_factory=lambda: [])
     type: str = "Delete"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """find and delete the activity object"""
         if not self.object:
             return
@@ -52,7 +52,11 @@ class Delete(Verb):
             model = apps.get_model("bookwyrm.User")
             obj = model.find_existing_by_remote_id(self.object)
         else:
-            obj = self.object.to_model(save=False, allow_create=False)
+            obj = self.object.to_model(
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            )
 
         if obj:
             obj.delete()
@@ -67,11 +71,13 @@ class Update(Verb):
     to: List[str]
     type: str = "Update"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """update a model instance from the dataclass"""
         if not self.object:
             return
-        self.object.to_model(allow_create=False)
+        self.object.to_model(
+            allow_create=False, allow_external_connections=allow_external_connections
+        )
 
 
 @dataclass(init=False)
@@ -80,10 +86,10 @@ class Undo(Verb):
 
     type: str = "Undo"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """find and remove the activity object"""
         if isinstance(self.object, str):
-            # it may be that sometihng should be done with these, but idk what
+            # it may be that something should be done with these, but idk what
             # this seems just to be coming from pleroma
             return
 
@@ -92,13 +98,28 @@ class Undo(Verb):
         model = None
         if self.object.type == "Follow":
             model = apps.get_model("bookwyrm.UserFollows")
-            obj = self.object.to_model(model=model, save=False, allow_create=False)
+            obj = self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            )
             if not obj:
-                # this could be a folloq request not a follow proper
+                # this could be a follow request not a follow proper
                 model = apps.get_model("bookwyrm.UserFollowRequest")
-                obj = self.object.to_model(model=model, save=False, allow_create=False)
+                obj = self.object.to_model(
+                    model=model,
+                    save=False,
+                    allow_create=False,
+                    allow_external_connections=allow_external_connections,
+                )
         else:
-            obj = self.object.to_model(model=model, save=False, allow_create=False)
+            obj = self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            )
         if not obj:
             # if we don't have the object, we can't undo it. happens a lot with boosts
             return
@@ -112,9 +133,9 @@ class Follow(Verb):
     object: str
     type: str = "Follow"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """relationship save"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
 
 
 @dataclass(init=False)
@@ -124,9 +145,9 @@ class Block(Verb):
     object: str
     type: str = "Block"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """relationship save"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
 
 
 @dataclass(init=False)
@@ -136,7 +157,7 @@ class Accept(Verb):
     object: Follow
     type: str = "Accept"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """accept a request"""
         obj = self.object.to_model(save=False, allow_create=True)
         obj.accept()
@@ -149,10 +170,20 @@ class Reject(Verb):
     object: Follow
     type: str = "Reject"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
-        """reject a follow request"""
+        """reject a follow or follow request"""
-        obj = self.object.to_model(save=False, allow_create=False)
+
-        obj.reject()
+        for model_name in ["UserFollowRequest", "UserFollows", None]:
+            model = apps.get_model(f"bookwyrm.{model_name}") if model_name else None
+            if obj := self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            ):
+                # Reject the first model that can be built.
+                obj.reject()
+                break
 
 
 @dataclass(init=False)
@@ -163,7 +194,7 @@ class Add(Verb):
     object: CollectionItem
     type: str = "Add"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """figure out the target to assign the item to a collection"""
         target = resolve_remote_id(self.target)
         item = self.object.to_model(save=False)
@@ -177,7 +208,7 @@ class Remove(Add):
 
     type: str = "Remove"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """find and remove the activity object"""
         obj = self.object.to_model(save=False, allow_create=False)
         if obj:
@@ -191,9 +222,9 @@ class Like(Verb):
     object: str
     type: str = "Like"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """like"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
 
 
 # pylint: disable=invalid-name
@@ -207,6 +238,33 @@ class Announce(Verb):
     object: str
     type: str = "Announce"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """boost"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
+
+
+@dataclass(init=False)
+class Move(Verb):
+    """a user moving an object"""
+
+    object: str
+    type: str = "Move"
+    origin: str = None
+    target: str = None
+
+    def action(self, allow_external_connections=True):
+        """move"""
+
+        object_is_user = resolve_remote_id(remote_id=self.object, model="User")
+
+        if object_is_user:
+            model = apps.get_model("bookwyrm.MoveUser")
+
+            self.to_model(
+                model=model,
+                save=True,
+                allow_external_connections=allow_external_connections,
+            )
+        else:
+            # we might do something with this to move other objects at some point
+            pass

bookwyrm/activitystreams.py

@@ -4,27 +4,32 @@ from django.dispatch import receiver
 from django.db import transaction
 from django.db.models import signals, Q
 from django.utils import timezone
+from opentelemetry import trace
 
 from bookwyrm import models
 from bookwyrm.redis_store import RedisStore, r
-from bookwyrm.tasks import app, LOW, MEDIUM, HIGH
+from bookwyrm.tasks import app, STREAMS, IMPORT_TRIGGERED
+from bookwyrm.telemetry import open_telemetry
+
+
+tracer = open_telemetry.tracer()
 
 
 class ActivityStream(RedisStore):
     """a category of activity stream (like home, local, books)"""
 
-    def stream_id(self, user):
+    def stream_id(self, user_id):
         """the redis key for this user's instance of this stream"""
-        return f"{user.id}-{self.key}"
+        return f"{user_id}-{self.key}"
 
-    def unread_id(self, user):
+    def unread_id(self, user_id):
         """the redis key for this user's unread count for this stream"""
-        stream_id = self.stream_id(user)
+        stream_id = self.stream_id(user_id)
         return f"{stream_id}-unread"
 
-    def unread_by_status_type_id(self, user):
+    def unread_by_status_type_id(self, user_id):
         """the redis key for this user's unread count for this stream"""
-        stream_id = self.stream_id(user)
+        stream_id = self.stream_id(user_id)
         return f"{stream_id}-unread-by-type"
 
     def get_rank(self, obj):  # pylint: disable=no-self-use
@@ -33,16 +38,19 @@ class ActivityStream(RedisStore):
 
     def add_status(self, status, increment_unread=False):
         """add a status to users' feeds"""
+        audience = self.get_audience(status)
         # the pipeline contains all the add-to-stream activities
-        pipeline = self.add_object_to_related_stores(status, execute=False)
+        pipeline = self.add_object_to_stores(
+            status, self.get_stores_for_users(audience), execute=False
+        )
 
         if increment_unread:
-            for user in self.get_audience(status):
+            for user_id in audience:
                 # add to the unread status count
-                pipeline.incr(self.unread_id(user))
+                pipeline.incr(self.unread_id(user_id))
                 # add to the unread status count for status type
                 pipeline.hincrby(
-                    self.unread_by_status_type_id(user), get_status_type(status), 1
+                    self.unread_by_status_type_id(user_id), get_status_type(status), 1
                 )
 
         # and go!
@@ -52,21 +60,21 @@ class ActivityStream(RedisStore):
         """add a user's statuses to another user's feed"""
         # only add the statuses that the viewer should be able to see (ie, not dms)
         statuses = models.Status.privacy_filter(viewer).filter(user=user)
-        self.bulk_add_objects_to_store(statuses, self.stream_id(viewer))
+        self.bulk_add_objects_to_store(statuses, self.stream_id(viewer.id))
 
     def remove_user_statuses(self, viewer, user):
         """remove a user's status from another user's feed"""
         # remove all so that followers only statuses are removed
         statuses = user.status_set.all()
-        self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer))
+        self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer.id))
 
     def get_activity_stream(self, user):
         """load the statuses to be displayed"""
         # clear unreads for this feed
-        r.set(self.unread_id(user), 0)
+        r.set(self.unread_id(user.id), 0)
-        r.delete(self.unread_by_status_type_id(user))
+        r.delete(self.unread_by_status_type_id(user.id))
 
-        statuses = self.get_store(self.stream_id(user))
+        statuses = self.get_store(self.stream_id(user.id))
         return (
             models.Status.objects.select_subclasses()
             .filter(id__in=statuses)
@@ -83,11 +91,11 @@ class ActivityStream(RedisStore):
 
     def get_unread_count(self, user):
         """get the unread status count for this user's feed"""
-        return int(r.get(self.unread_id(user)) or 0)
+        return int(r.get(self.unread_id(user.id)) or 0)
 
     def get_unread_count_by_status_type(self, user):
         """get the unread status count for this user's feed's status types"""
-        status_types = r.hgetall(self.unread_by_status_type_id(user))
+        status_types = r.hgetall(self.unread_by_status_type_id(user.id))
         return {
             str(key.decode("utf-8")): int(value) or 0
             for key, value in status_types.items()
@@ -95,13 +103,20 @@ class ActivityStream(RedisStore):
 
     def populate_streams(self, user):
         """go from zero to a timeline"""
-        self.populate_store(self.stream_id(user))
+        self.populate_store(self.stream_id(user.id))
 
-    def get_audience(self, status):  # pylint: disable=no-self-use
+    @tracer.start_as_current_span("ActivityStream._get_audience")
-        """given a status, what users should see it"""
+    def _get_audience(self, status):  # pylint: disable=no-self-use
-        # direct messages don't appeard in feeds, direct comments/reviews/etc do
+        """given a status, what users should see it, excluding the author"""
+        trace.get_current_span().set_attribute("status_type", status.status_type)
+        trace.get_current_span().set_attribute("status_privacy", status.privacy)
+        trace.get_current_span().set_attribute(
+            "status_reply_parent_privacy",
+            status.reply_parent.privacy if status.reply_parent else status.privacy,
+        )
+        # direct messages don't appear in feeds, direct comments/reviews/etc do
         if status.privacy == "direct" and status.status_type == "Note":
-            return []
+            return models.User.objects.none()
 
         # everybody who could plausibly see this status
         audience = models.User.objects.filter(
@@ -114,15 +129,13 @@ class ActivityStream(RedisStore):
         # only visible to the poster and mentioned users
         if status.privacy == "direct":
             audience = audience.filter(
-                Q(id=status.user.id)  # if the user is the post's author
+                Q(id__in=status.mention_users.all())  # if the user is mentioned
-                | Q(id__in=status.mention_users.all())  # if the user is mentioned
             )
 
         # don't show replies to statuses the user can't see
         elif status.reply_parent and status.reply_parent.privacy == "followers":
             audience = audience.filter(
-                Q(id=status.user.id)  # if the user is the post's author
+                Q(id=status.reply_parent.user.id)  # if the user is the OG author
-                | Q(id=status.reply_parent.user.id)  # if the user is the OG author
                 | (
                     Q(following=status.user) & Q(following=status.reply_parent.user)
                 )  # if the user is following both authors
@@ -131,13 +144,23 @@ class ActivityStream(RedisStore):
         # only visible to the poster's followers and tagged users
         elif status.privacy == "followers":
             audience = audience.filter(
-                Q(id=status.user.id)  # if the user is the post's author
+                Q(following=status.user)  # if the user is following the author
-                | Q(following=status.user)  # if the user is following the author
             )
         return audience.distinct()
 
-    def get_stores_for_object(self, obj):
+    @tracer.start_as_current_span("ActivityStream.get_audience")
-        return [self.stream_id(u) for u in self.get_audience(obj)]
+    def get_audience(self, status):
+        """given a status, what users should see it"""
+        trace.get_current_span().set_attribute("stream_id", self.key)
+        audience = self._get_audience(status).values_list("id", flat=True)
+        status_author = models.User.objects.filter(
+            is_active=True, local=True, id=status.user.id
+        ).values_list("id", flat=True)
+        return list(set(list(audience) + list(status_author)))
+
+    def get_stores_for_users(self, user_ids):
+        """convert a list of user ids into redis store ids"""
+        return [self.stream_id(user_id) for user_id in user_ids]
 
     def get_statuses_for_user(self, user):  # pylint: disable=no-self-use
         """given a user, what statuses should they see on this stream"""
@@ -156,14 +179,19 @@ class HomeStream(ActivityStream):
 
     key = "home"
 
+    @tracer.start_as_current_span("HomeStream.get_audience")
     def get_audience(self, status):
-        audience = super().get_audience(status)
+        trace.get_current_span().set_attribute("stream_id", self.key)
+        audience = super()._get_audience(status)
         if not audience:
             return []
-        return audience.filter(
+        # if the user is following the author
-            Q(id=status.user.id)  # if the user is the post's author
+        audience = audience.filter(following=status.user).values_list("id", flat=True)
-            | Q(following=status.user)  # if the user is following the author
+        # if the user is the post's author
-        ).distinct()
+        status_author = models.User.objects.filter(
+            is_active=True, local=True, id=status.user.id
+        ).values_list("id", flat=True)
+        return list(set(list(audience) + list(status_author)))
 
     def get_statuses_for_user(self, user):
         return models.Status.privacy_filter(
@@ -202,8 +230,20 @@ class BooksStream(ActivityStream):
 
     key = "books"
 
-    def get_audience(self, status):
+    def _get_audience(self, status):
         """anyone with the mentioned book on their shelves"""
+        work = (
+            status.book.parent_work
+            if hasattr(status, "book")
+            else status.mention_books.first().parent_work
+        )
+
+        audience = super()._get_audience(status)
+        if not audience:
+            return models.User.objects.none()
+        return audience.filter(shelfbook__book__parent_work=work).distinct()
+
+    def get_audience(self, status):
         # only show public statuses on the books feed,
         # and only statuses that mention books
         if status.privacy != "public" or not (
@@ -211,16 +251,7 @@ class BooksStream(ActivityStream):
         ):
             return []
 
-        work = (
+        return super().get_audience(status)
-            status.book.parent_work
-            if hasattr(status, "book")
-            else status.mention_books.first().parent_work
-        )
-
-        audience = super().get_audience(status)
-        if not audience:
-            return []
-        return audience.filter(shelfbook__book__parent_work=work).distinct()
 
     def get_statuses_for_user(self, user):
         """any public status that mentions the user's books"""
@@ -244,38 +275,38 @@ class BooksStream(ActivityStream):
     def add_book_statuses(self, user, book):
         """add statuses about a book to a user's feed"""
         work = book.parent_work
-        statuses = (
+        statuses = models.Status.privacy_filter(
-            models.Status.privacy_filter(
+            user,
-                user,
+            privacy_levels=["public"],
-                privacy_levels=["public"],
-            )
-            .filter(
-                Q(comment__book__parent_work=work)
-                | Q(quotation__book__parent_work=work)
-                | Q(review__book__parent_work=work)
-                | Q(mention_books__parent_work=work)
-            )
-            .distinct()
         )
-        self.bulk_add_objects_to_store(statuses, self.stream_id(user))
+
+        book_comments = statuses.filter(Q(comment__book__parent_work=work))
+        book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
+        book_reviews = statuses.filter(Q(review__book__parent_work=work))
+        book_mentions = statuses.filter(Q(mention_books__parent_work=work))
+
+        self.bulk_add_objects_to_store(book_comments, self.stream_id(user.id))
+        self.bulk_add_objects_to_store(book_quotations, self.stream_id(user.id))
+        self.bulk_add_objects_to_store(book_reviews, self.stream_id(user.id))
+        self.bulk_add_objects_to_store(book_mentions, self.stream_id(user.id))
 
     def remove_book_statuses(self, user, book):
         """add statuses about a book to a user's feed"""
         work = book.parent_work
-        statuses = (
+        statuses = models.Status.privacy_filter(
-            models.Status.privacy_filter(
+            user,
-                user,
+            privacy_levels=["public"],
-                privacy_levels=["public"],
-            )
-            .filter(
-                Q(comment__book__parent_work=work)
-                | Q(quotation__book__parent_work=work)
-                | Q(review__book__parent_work=work)
-                | Q(mention_books__parent_work=work)
-            )
-            .distinct()
         )
-        self.bulk_remove_objects_from_store(statuses, self.stream_id(user))
+
+        book_comments = statuses.filter(Q(comment__book__parent_work=work))
+        book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
+        book_reviews = statuses.filter(Q(review__book__parent_work=work))
+        book_mentions = statuses.filter(Q(mention_books__parent_work=work))
+
+        self.bulk_remove_objects_from_store(book_comments, self.stream_id(user.id))
+        self.bulk_remove_objects_from_store(book_quotations, self.stream_id(user.id))
+        self.bulk_remove_objects_from_store(book_reviews, self.stream_id(user.id))
+        self.bulk_remove_objects_from_store(book_mentions, self.stream_id(user.id))
 
 
 # determine which streams are enabled in settings.py
@@ -298,10 +329,9 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
         remove_status_task.delay(instance.id)
         return
 
-    # To avoid creating a zillion unnecessary tasks caused by re-saving the model,
+    # We don't want to create multiple add_status_tasks for each status, and because
-    # check if it's actually ready to send before we go. We're trusting this was
+    # the transactions are atomic, on_commit won't run until the status is ready to add.
-    # set correctly by the inbox or view
+    if not created:
-    if not instance.ready:
         return
 
     # when creating new things, gotta wait on the transaction
@@ -312,13 +342,21 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
 
 def add_status_on_create_command(sender, instance, created):
     """runs this code only after the database commit completes"""
-    priority = HIGH
+    # boosts trigger 'saves" twice, so don't bother duplicating the task
+    if sender == models.Boost and not created:
+        return
+
+    priority = STREAMS
     # check if this is an old status, de-prioritize if so
     # (this will happen if federation is very slow, or, more expectedly, on csv import)
     if instance.published_date < timezone.now() - timedelta(
         days=1
     ) or instance.created_date < instance.published_date - timedelta(days=1):
-        priority = LOW
+        # a backdated status from a local user is an import, don't add it
+        if instance.user.local:
+            return
+        # an out of date remote status is a low priority but should be added
+        priority = IMPORT_TRIGGERED
 
     add_status_task.apply_async(
         args=(instance.id,),
@@ -462,7 +500,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs):
 # ---- TASKS
 
 
-@app.task(queue=LOW)
+@app.task(queue=STREAMS)
 def add_book_statuses_task(user_id, book_id):
     """add statuses related to a book on shelve"""
     user = models.User.objects.get(id=user_id)
@@ -470,7 +508,7 @@ def add_book_statuses_task(user_id, book_id):
     BooksStream().add_book_statuses(user, book)
 
 
-@app.task(queue=LOW)
+@app.task(queue=STREAMS)
 def remove_book_statuses_task(user_id, book_id):
     """remove statuses about a book from a user's books feed"""
     user = models.User.objects.get(id=user_id)
@@ -478,7 +516,7 @@ def remove_book_statuses_task(user_id, book_id):
     BooksStream().remove_book_statuses(user, book)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def populate_stream_task(stream, user_id):
     """background task for populating an empty activitystream"""
     user = models.User.objects.get(id=user_id)
@@ -486,7 +524,7 @@ def populate_stream_task(stream, user_id):
     stream.populate_streams(user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def remove_status_task(status_ids):
     """remove a status from any stream it might be in"""
     # this can take an id or a list of ids
@@ -496,10 +534,12 @@ def remove_status_task(status_ids):
 
     for stream in streams.values():
         for status in statuses:
-            stream.remove_object_from_related_stores(status)
+            stream.remove_object_from_stores(
+                status, stream.get_stores_for_users(stream.get_audience(status))
+            )
 
 
-@app.task(queue=HIGH)
+@app.task(queue=STREAMS)
 def add_status_task(status_id, increment_unread=False):
     """add a status to any stream it should be in"""
     status = models.Status.objects.select_subclasses().get(id=status_id)
@@ -511,7 +551,7 @@ def add_status_task(status_id, increment_unread=False):
         stream.add_status(status, increment_unread=increment_unread)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
     """remove all statuses by a user from a viewer's stream"""
     stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -521,7 +561,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
         stream.remove_user_statuses(viewer, user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def add_user_statuses_task(viewer_id, user_id, stream_list=None):
     """add all statuses by a user to a viewer's stream"""
     stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -531,7 +571,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None):
         stream.add_user_statuses(viewer, user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def handle_boost_task(boost_id):
     """remove the original post and other, earlier boosts"""
     instance = models.Status.objects.get(id=boost_id)
@@ -545,10 +585,10 @@ def handle_boost_task(boost_id):
 
     for stream in streams.values():
         # people who should see the boost (not people who see the original status)
-        audience = stream.get_stores_for_object(instance)
+        audience = stream.get_stores_for_users(stream.get_audience(instance))
-        stream.remove_object_from_related_stores(boosted, stores=audience)
+        stream.remove_object_from_stores(boosted, audience)
         for status in old_versions:
-            stream.remove_object_from_related_stores(status, stores=audience)
+            stream.remove_object_from_stores(status, audience)
 
 
 def get_status_type(status):

bookwyrm/apps.py

@@ -1,4 +1,5 @@
 """Do further startup configuration and initialization"""
+
 import os
 import urllib
 import logging
@@ -14,16 +15,16 @@ def download_file(url, destination):
     """Downloads a file to the given path"""
     try:
         # Ensure our destination directory exists
-        os.makedirs(os.path.dirname(destination))
+        os.makedirs(os.path.dirname(destination), exist_ok=True)
         with urllib.request.urlopen(url) as stream:
             with open(destination, "b+w") as outfile:
                 outfile.write(stream.read())
-    except (urllib.error.HTTPError, urllib.error.URLError):
+    except (urllib.error.HTTPError, urllib.error.URLError) as err:
-        logger.info("Failed to download file %s", url)
+        logger.error("Failed to download file %s: %s", url, err)
-    except OSError:
+    except OSError as err:
-        logger.info("Couldn't open font file %s for writing", destination)
+        logger.error("Couldn't open font file %s for writing: %s", destination, err)
-    except:  # pylint: disable=bare-except
+    except Exception as err:  # pylint:disable=broad-except
-        logger.info("Unknown error in file download")
+        logger.error("Unknown error in file download: %s", err)
 
 
 class BookwyrmConfig(AppConfig):
@@ -35,11 +36,12 @@ class BookwyrmConfig(AppConfig):
     # pylint: disable=no-self-use
     def ready(self):
         """set up OTLP and preview image files, if desired"""
-        if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+        if settings.OTEL_EXPORTER_OTLP_ENDPOINT or settings.OTEL_EXPORTER_CONSOLE:
             # pylint: disable=import-outside-toplevel
             from bookwyrm.telemetry import open_telemetry
 
             open_telemetry.instrumentDjango()
+            open_telemetry.instrumentPostgres()
 
         if settings.ENABLE_PREVIEW_IMAGES and settings.FONTS:
             # Download any fonts that we don't have yet

bookwyrm/book_search.py

@@ -1,28 +1,69 @@
 """ using a bookwyrm instance as a source of book data """
+from __future__ import annotations
 from dataclasses import asdict, dataclass
 from functools import reduce
 import operator
+from typing import Optional, Union, Any, Literal, overload
 
 from django.contrib.postgres.search import SearchRank, SearchQuery
-from django.db.models import OuterRef, Subquery, F, Q
+from django.db.models import F, Q
+from django.db.models.query import QuerySet
 
 from bookwyrm import models
 from bookwyrm import connectors
 from bookwyrm.settings import MEDIA_FULL_URL
 
 
+@overload
+def search(
+    query: str,
+    *,
+    min_confidence: float = 0,
+    filters: Optional[list[Any]] = None,
+    return_first: Literal[False],
+) -> QuerySet[models.Edition]:
+    ...
+
+
+@overload
+def search(
+    query: str,
+    *,
+    min_confidence: float = 0,
+    filters: Optional[list[Any]] = None,
+    return_first: Literal[True],
+) -> Optional[models.Edition]:
+    ...
+
+
 # pylint: disable=arguments-differ
-def search(query, min_confidence=0, filters=None, return_first=False):
+def search(
+    query: str,
+    *,
+    min_confidence: float = 0,
+    filters: Optional[list[Any]] = None,
+    return_first: bool = False,
+    books: Optional[QuerySet[models.Edition]] = None,
+) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
     """search your local database"""
     filters = filters or []
     if not query:
-        return []
+        return None if return_first else []
-    # first, try searching unqiue identifiers
+    query = query.strip()
-    results = search_identifiers(query, *filters, return_first=return_first)
+
+    results = None
+    # first, try searching unique identifiers
+    # unique identifiers never have spaces, title/author usually do
+    if not " " in query:
+        results = search_identifiers(
+            query, *filters, return_first=return_first, books=books
+        )
+
+    # if there were no identifier results...
     if not results:
         # then try searching title/author
         results = search_title_author(
-            query, min_confidence, *filters, return_first=return_first
+            query, min_confidence, *filters, return_first=return_first, books=books
         )
     return results
 
@@ -35,24 +76,10 @@ def isbn_search(query):
     # If the ISBN has only 9 characters, prepend missing zero
     query = query.strip().upper().rjust(10, "0")
     filters = [{f: query} for f in ["isbn_10", "isbn_13"]]
-    results = models.Edition.objects.filter(
+    return models.Edition.objects.filter(
         reduce(operator.or_, (Q(**f) for f in filters))
     ).distinct()
 
-    # when there are multiple editions of the same work, pick the default.
-    # it would be odd for this to happen.
-
-    default_editions = models.Edition.objects.filter(
-        parent_work=OuterRef("parent_work")
-    ).order_by("-edition_rank")
-    results = (
-        results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter(
-            default_id=F("id")
-        )
-        or results
-    )
-    return results
-
 
 def format_search_result(search_result):
     """convert a book object into a search result object"""
@@ -73,8 +100,18 @@ def format_search_result(search_result):
     ).json()
 
 
-def search_identifiers(query, *filters, return_first=False):
+def search_identifiers(
-    """tries remote_id, isbn; defined as dedupe fields on the model"""
+    query,
+    *filters,
+    return_first=False,
+    books=None,
+) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
+    """search Editions by deduplication fields
+
+    Best for cases when we can assume someone is searching for an exact match on
+    commonly unique data identifiers like isbn or specific library ids.
+    """
+    books = books or models.Edition.objects
     if connectors.maybe_isbn(query):
         # Oh did you think the 'S' in ISBN stood for 'standard'?
         normalized_isbn = query.strip().upper().rjust(10, "0")
@@ -85,54 +122,43 @@ def search_identifiers(query, *filters, return_first=False):
         for f in models.Edition._meta.get_fields()
         if hasattr(f, "deduplication_field") and f.deduplication_field
     ]
-    results = models.Edition.objects.filter(
+    results = books.filter(
         *filters, reduce(operator.or_, (Q(**f) for f in or_filters))
     ).distinct()
-    if results.count() <= 1:
-        if return_first:
-            return results.first()
-        return results
 
-    # when there are multiple editions of the same work, pick the default.
-    # it would be odd for this to happen.
-    default_editions = models.Edition.objects.filter(
-        parent_work=OuterRef("parent_work")
-    ).order_by("-edition_rank")
-    results = (
-        results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter(
-            default_id=F("id")
-        )
-        or results
-    )
     if return_first:
         return results.first()
     return results
 
 
-def search_title_author(query, min_confidence, *filters, return_first=False):
+def search_title_author(
+    query,
+    min_confidence,
+    *filters,
+    return_first=False,
+    books=None,
+) -> QuerySet[models.Edition]:
     """searches for title and author"""
+    books = books or models.Edition.objects
     query = SearchQuery(query, config="simple") | SearchQuery(query, config="english")
     results = (
-        models.Edition.objects.filter(*filters, search_vector=query)
+        books.filter(*filters, search_vector=query)
         .annotate(rank=SearchRank(F("search_vector"), query))
         .filter(rank__gt=min_confidence)
         .order_by("-rank")
     )
 
     # when there are multiple editions of the same work, pick the closest
-    editions_of_work = results.values("parent_work__id").values_list("parent_work__id")
+    editions_of_work = results.values_list("parent_work__id", flat=True).distinct()
 
     # filter out multiple editions of the same work
     list_results = []
-    for work_id in set(editions_of_work):
+    for work_id in editions_of_work[:30]:
-        editions = results.filter(parent_work=work_id)
+        result = (
-        default = editions.order_by("-edition_rank").first()
+            results.filter(parent_work=work_id)
-        default_rank = default.rank if default else 0
+            .order_by("-rank", "-edition_rank")
-        # if mutliple books have the top rank, pick the default edition
+            .first()
-        if default_rank == editions.first().rank:
+        )
-            result = default
-        else:
-            result = editions.first()
 
         if return_first:
             return result
@@ -147,11 +173,11 @@ class SearchResult:
     title: str
     key: str
     connector: object
-    view_link: str = None
+    view_link: Optional[str] = None
-    author: str = None
+    author: Optional[str] = None
-    year: str = None
+    year: Optional[str] = None
-    cover: str = None
+    cover: Optional[str] = None
-    confidence: int = 1
+    confidence: float = 1.0
 
     def __repr__(self):
         # pylint: disable=consider-using-f-string

bookwyrm/connectors/abstract_connector.py

@@ -1,44 +1,57 @@
 """ functionality outline for a book data connector """
+from __future__ import annotations
 from abc import ABC, abstractmethod
-import imghdr
+from typing import Optional, TypedDict, Any, Callable, Union, Iterator
+from urllib.parse import quote_plus
+
+# pylint: disable-next=deprecated-module
+import imghdr  # Deprecated in 3.11 for removal in 3.13; no good alternative yet
 import logging
 import re
+import asyncio
+import requests
+from requests.exceptions import RequestException
+import aiohttp
 
 from django.core.files.base import ContentFile
 from django.db import transaction
-import requests
-from requests.exceptions import RequestException
 
 from bookwyrm import activitypub, models, settings
+from bookwyrm.settings import USER_AGENT
 from .connector_manager import load_more_data, ConnectorException, raise_not_valid_url
 from .format_mappings import format_mappings
-
+from ..book_search import SearchResult
 
 logger = logging.getLogger(__name__)
 
+JsonDict = dict[str, Any]
+
+
+class ConnectorResults(TypedDict):
+    """TypedDict for results returned by connector"""
+
+    connector: AbstractMinimalConnector
+    results: list[SearchResult]
+
 
 class AbstractMinimalConnector(ABC):
     """just the bare bones, for other bookwyrm instances"""
 
-    def __init__(self, identifier):
+    def __init__(self, identifier: str):
         # load connector settings
         info = models.Connector.objects.get(identifier=identifier)
         self.connector = info
 
         # the things in the connector model to copy over
-        self_fields = [
+        self.base_url = info.base_url
-            "base_url",
+        self.books_url = info.books_url
-            "books_url",
+        self.covers_url = info.covers_url
-            "covers_url",
+        self.search_url = info.search_url
-            "search_url",
+        self.isbn_search_url = info.isbn_search_url
-            "isbn_search_url",
+        self.name = info.name
-            "name",
+        self.identifier = info.identifier
-            "identifier",
-        ]
-        for field in self_fields:
-            setattr(self, field, getattr(info, field))
 
-    def get_search_url(self, query):
+    def get_search_url(self, query: str) -> str:
         """format the query url"""
         # Check if the query resembles an ISBN
         if maybe_isbn(query) and self.isbn_search_url and self.isbn_search_url != "":
@@ -48,44 +61,93 @@ class AbstractMinimalConnector(ABC):
             return f"{self.isbn_search_url}{normalized_query}"
         # NOTE: previously, we tried searching isbn and if that produces no results,
         # searched as free text. This, instead, only searches isbn if it's isbn-y
-        return f"{self.search_url}{query}"
+        return f"{self.search_url}{quote_plus(query)}"
 
-    def process_search_response(self, query, data, min_confidence):
+    def process_search_response(
-        """Format the search results based on the formt of the query"""
+        self, query: str, data: Any, min_confidence: float
+    ) -> list[SearchResult]:
+        """Format the search results based on the format of the query"""
         if maybe_isbn(query):
             return list(self.parse_isbn_search_data(data))[:10]
         return list(self.parse_search_data(data, min_confidence))[:10]
 
+    async def get_results(
+        self,
+        session: aiohttp.ClientSession,
+        url: str,
+        min_confidence: float,
+        query: str,
+    ) -> Optional[ConnectorResults]:
+        """try this specific connector"""
+        # pylint: disable=line-too-long
+        headers = {
+            "Accept": (
+                'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'
+            ),
+            "User-Agent": USER_AGENT,
+        }
+        params = {"min_confidence": min_confidence}
+        try:
+            async with session.get(url, headers=headers, params=params) as response:
+                if not response.ok:
+                    logger.info("Unable to connect to %s: %s", url, response.reason)
+                    return None
+
+                try:
+                    raw_data = await response.json()
+                except aiohttp.client_exceptions.ContentTypeError as err:
+                    logger.exception(err)
+                    return None
+
+                return ConnectorResults(
+                    connector=self,
+                    results=self.process_search_response(
+                        query, raw_data, min_confidence
+                    ),
+                )
+        except asyncio.TimeoutError:
+            logger.info("Connection timed out for url: %s", url)
+        except aiohttp.ClientError as err:
+            logger.info(err)
+        return None
+
     @abstractmethod
-    def get_or_create_book(self, remote_id):
+    def get_or_create_book(self, remote_id: str) -> Optional[models.Book]:
         """pull up a book record by whatever means possible"""
 
     @abstractmethod
-    def parse_search_data(self, data, min_confidence):
+    def parse_search_data(
+        self, data: Any, min_confidence: float
+    ) -> Iterator[SearchResult]:
         """turn the result json from a search into a list"""
 
     @abstractmethod
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(self, data: Any) -> Iterator[SearchResult]:
         """turn the result json from a search into a list"""
 
 
 class AbstractConnector(AbstractMinimalConnector):
     """generic book data connector"""
 
-    def __init__(self, identifier):
+    generated_remote_link_field = ""
+
+    def __init__(self, identifier: str):
         super().__init__(identifier)
         # fields we want to look for in book data to copy over
         # title we handle separately.
-        self.book_mappings = []
+        self.book_mappings: list[Mapping] = []
+        self.author_mappings: list[Mapping] = []
 
-    def get_or_create_book(self, remote_id):
+    def get_or_create_book(self, remote_id: str) -> Optional[models.Book]:
         """translate arbitrary json into an Activitypub dataclass"""
         # first, check if we have the origin_id saved
         existing = models.Edition.find_existing_by_remote_id(
             remote_id
         ) or models.Work.find_existing_by_remote_id(remote_id)
         if existing:
-            if hasattr(existing, "default_edition"):
+            if hasattr(existing, "default_edition") and isinstance(
+                existing.default_edition, models.Edition
+            ):
                 return existing.default_edition
             return existing
 
@@ -117,6 +179,9 @@ class AbstractConnector(AbstractMinimalConnector):
             )
             # this will dedupe automatically
             work = work_activity.to_model(model=models.Work, overwrite=False)
+            if not work:
+                return None
+
             for author in self.get_authors_from_data(work_data):
                 work.authors.add(author)
 
@@ -124,12 +189,21 @@ class AbstractConnector(AbstractMinimalConnector):
         load_more_data.delay(self.connector.id, work.id)
         return edition
 
-    def get_book_data(self, remote_id):  # pylint: disable=no-self-use
+    def get_book_data(self, remote_id: str) -> JsonDict:  # pylint: disable=no-self-use
         """this allows connectors to override the default behavior"""
         return get_data(remote_id)
 
-    def create_edition_from_data(self, work, edition_data, instance=None):
+    def create_edition_from_data(
+        self,
+        work: models.Work,
+        edition_data: Union[str, JsonDict],
+        instance: Optional[models.Edition] = None,
+    ) -> Optional[models.Edition]:
         """if we already have the work, we're ready"""
+        if isinstance(edition_data, str):
+            # We don't expect a string here
+            return None
+
         mapped_data = dict_from_mappings(edition_data, self.book_mappings)
         mapped_data["work"] = work.remote_id
         edition_activity = activitypub.Edition(**mapped_data)
@@ -137,6 +211,9 @@ class AbstractConnector(AbstractMinimalConnector):
             model=models.Edition, overwrite=False, instance=instance
         )
 
+        if not edition:
+            return None
+
         # if we're updating an existing instance, we don't need to load authors
         if instance:
             return edition
@@ -153,7 +230,9 @@ class AbstractConnector(AbstractMinimalConnector):
 
         return edition
 
-    def get_or_create_author(self, remote_id, instance=None):
+    def get_or_create_author(
+        self, remote_id: str, instance: Optional[models.Author] = None
+    ) -> Optional[models.Author]:
         """load that author"""
         if not instance:
             existing = models.Author.find_existing_by_remote_id(remote_id)
@@ -173,46 +252,51 @@ class AbstractConnector(AbstractMinimalConnector):
             model=models.Author, overwrite=False, instance=instance
         )
 
-    def get_remote_id_from_model(self, obj):
+    def get_remote_id_from_model(self, obj: models.BookDataModel) -> Optional[str]:
         """given the data stored, how can we look this up"""
-        return getattr(obj, getattr(self, "generated_remote_link_field"))
+        remote_id: Optional[str] = getattr(obj, self.generated_remote_link_field)
+        return remote_id
 
-    def update_author_from_remote(self, obj):
+    def update_author_from_remote(self, obj: models.Author) -> Optional[models.Author]:
         """load the remote data from this connector and add it to an existing author"""
         remote_id = self.get_remote_id_from_model(obj)
+        if not remote_id:
+            return None
         return self.get_or_create_author(remote_id, instance=obj)
 
-    def update_book_from_remote(self, obj):
+    def update_book_from_remote(self, obj: models.Edition) -> Optional[models.Edition]:
         """load the remote data from this connector and add it to an existing book"""
         remote_id = self.get_remote_id_from_model(obj)
+        if not remote_id:
+            return None
         data = self.get_book_data(remote_id)
         return self.create_edition_from_data(obj.parent_work, data, instance=obj)
 
     @abstractmethod
-    def is_work_data(self, data):
+    def is_work_data(self, data: JsonDict) -> bool:
         """differentiate works and editions"""
 
     @abstractmethod
-    def get_edition_from_work_data(self, data):
+    def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
         """every work needs at least one edition"""
 
     @abstractmethod
-    def get_work_from_edition_data(self, data):
+    def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
         """every edition needs a work"""
 
     @abstractmethod
-    def get_authors_from_data(self, data):
+    def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
         """load author data"""
 
     @abstractmethod
-    def expand_book_data(self, book):
+    def expand_book_data(self, book: models.Book) -> None:
         """get more info on a book"""
 
 
-def dict_from_mappings(data, mappings):
+def dict_from_mappings(data: JsonDict, mappings: list[Mapping]) -> JsonDict:
     """create a dict in Activitypub format, using mappings supplies by
     the subclass"""
-    result = {}
+    result: JsonDict = {}
     for mapping in mappings:
         # sometimes there are multiple mappings for one field, don't
         # overwrite earlier writes in that case
@@ -222,7 +306,11 @@ def dict_from_mappings(data, mappings):
     return result
 
 
-def get_data(url, params=None, timeout=10):
+def get_data(
+    url: str,
+    params: Optional[dict[str, str]] = None,
+    timeout: int = settings.QUERY_TIMEOUT,
+) -> JsonDict:
     """wrapper for request.get"""
     # check if the url is blocked
     raise_not_valid_url(url)
@@ -244,17 +332,26 @@ def get_data(url, params=None, timeout=10):
         raise ConnectorException(err)
 
     if not resp.ok:
-        raise ConnectorException()
+        if resp.status_code == 401:
+            # this is probably an AUTHORIZED_FETCH issue
+            resp.raise_for_status()
+        else:
+            raise ConnectorException()
     try:
         data = resp.json()
     except ValueError as err:
         logger.info(err)
         raise ConnectorException(err)
 
+    if not isinstance(data, dict):
+        raise ConnectorException("Unexpected data format")
+
     return data
 
 
-def get_image(url, timeout=10):
+def get_image(
+    url: str, timeout: int = 10
+) -> Union[tuple[ContentFile[bytes], str], tuple[None, None]]:
     """wrapper for requesting an image"""
     raise_not_valid_url(url)
     try:
@@ -284,14 +381,19 @@ def get_image(url, timeout=10):
 class Mapping:
     """associate a local database field with a field in an external dataset"""
 
-    def __init__(self, local_field, remote_field=None, formatter=None):
+    def __init__(
+        self,
+        local_field: str,
+        remote_field: Optional[str] = None,
+        formatter: Optional[Callable[[Any], Any]] = None,
+    ):
         noop = lambda x: x
 
         self.local_field = local_field
         self.remote_field = remote_field or local_field
         self.formatter = formatter or noop
 
-    def get_value(self, data):
+    def get_value(self, data: JsonDict) -> Optional[Any]:
         """pull a field from incoming json and return the formatted version"""
         value = data.get(self.remote_field)
         if not value:
@@ -302,7 +404,7 @@ class Mapping:
             return None
 
 
-def infer_physical_format(format_text):
+def infer_physical_format(format_text: str) -> Optional[str]:
     """try to figure out what the standardized format is from the free value"""
     format_text = format_text.lower()
     if format_text in format_mappings:
@@ -315,8 +417,8 @@ def infer_physical_format(format_text):
     return matches[0]
 
 
-def unique_physical_format(format_text):
+def unique_physical_format(format_text: str) -> Optional[str]:
-    """only store the format if it isn't diretly in the format mappings"""
+    """only store the format if it isn't directly in the format mappings"""
     format_text = format_text.lower()
     if format_text in format_mappings:
         # try a direct match, so saving this would be redundant
@@ -324,7 +426,7 @@ def unique_physical_format(format_text):
     return format_text
 
 
-def maybe_isbn(query):
+def maybe_isbn(query: str) -> bool:
     """check if a query looks like an isbn"""
     isbn = re.sub(r"[\W_]", "", query)  # removes filler characters
     # ISBNs must be numeric except an ISBN10 checkdigit can be 'X'

bookwyrm/connectors/bookwyrm_connector.py

@@ -1,4 +1,7 @@
 """ using another bookwyrm instance as a source of book data """
+from __future__ import annotations
+from typing import Any, Iterator
+
 from bookwyrm import activitypub, models
 from bookwyrm.book_search import SearchResult
 from .abstract_connector import AbstractMinimalConnector
@@ -7,15 +10,19 @@ from .abstract_connector import AbstractMinimalConnector
 class Connector(AbstractMinimalConnector):
     """this is basically just for search"""
 
-    def get_or_create_book(self, remote_id):
+    def get_or_create_book(self, remote_id: str) -> models.Edition:
         return activitypub.resolve_remote_id(remote_id, model=models.Edition)
 
-    def parse_search_data(self, data, min_confidence):
+    def parse_search_data(
+        self, data: list[dict[str, Any]], min_confidence: float
+    ) -> Iterator[SearchResult]:
         for search_result in data:
             search_result["connector"] = self
             yield SearchResult(**search_result)
 
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(
+        self, data: list[dict[str, Any]]
+    ) -> Iterator[SearchResult]:
         for search_result in data:
             search_result["connector"] = self
             yield SearchResult(**search_result)

bookwyrm/connectors/connector_manager.py

@@ -1,8 +1,11 @@
 """ interface with whatever connectors the app has """
+from __future__ import annotations
 import asyncio
 import importlib
 import ipaddress
 import logging
+from asyncio import Future
+from typing import Iterator, Any, Optional, Union, overload, Literal
 from urllib.parse import urlparse
 
 import aiohttp
@@ -12,8 +15,10 @@ from django.db.models import signals
 from requests import HTTPError
 
 from bookwyrm import book_search, models
-from bookwyrm.settings import SEARCH_TIMEOUT, USER_AGENT
+from bookwyrm.book_search import SearchResult
-from bookwyrm.tasks import app, LOW
+from bookwyrm.connectors import abstract_connector
+from bookwyrm.settings import SEARCH_TIMEOUT
+from bookwyrm.tasks import app, CONNECTORS
 
 logger = logging.getLogger(__name__)
 
@@ -22,61 +27,46 @@ class ConnectorException(HTTPError):
     """when the connector can't do what was asked"""
 
 
-async def get_results(session, url, min_confidence, query, connector):
+async def async_connector_search(
-    """try this specific connector"""
+    query: str,
-    # pylint: disable=line-too-long
+    items: list[tuple[str, abstract_connector.AbstractConnector]],
-    headers = {
+    min_confidence: float,
-        "Accept": (
+) -> list[Optional[abstract_connector.ConnectorResults]]:
-            'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'
-        ),
-        "User-Agent": USER_AGENT,
-    }
-    params = {"min_confidence": min_confidence}
-    try:
-        async with session.get(url, headers=headers, params=params) as response:
-            if not response.ok:
-                logger.info("Unable to connect to %s: %s", url, response.reason)
-                return
-
-            try:
-                raw_data = await response.json()
-            except aiohttp.client_exceptions.ContentTypeError as err:
-                logger.exception(err)
-                return
-
-            return {
-                "connector": connector,
-                "results": connector.process_search_response(
-                    query, raw_data, min_confidence
-                ),
-            }
-    except asyncio.TimeoutError:
-        logger.info("Connection timed out for url: %s", url)
-    except aiohttp.ClientError as err:
-        logger.info(err)
-
-
-async def async_connector_search(query, items, min_confidence):
     """Try a number of requests simultaneously"""
     timeout = aiohttp.ClientTimeout(total=SEARCH_TIMEOUT)
     async with aiohttp.ClientSession(timeout=timeout) as session:
-        tasks = []
+        tasks: list[Future[Optional[abstract_connector.ConnectorResults]]] = []
         for url, connector in items:
             tasks.append(
                 asyncio.ensure_future(
-                    get_results(session, url, min_confidence, query, connector)
+                    connector.get_results(session, url, min_confidence, query)
                 )
             )
 
         results = await asyncio.gather(*tasks)
-        return results
+        return list(results)
 
 
-def search(query, min_confidence=0.1, return_first=False):
+@overload
-    """find books based on arbitary keywords"""
+def search(
+    query: str, *, min_confidence: float = 0.1, return_first: Literal[False]
+) -> list[abstract_connector.ConnectorResults]:
+    ...
+
+
+@overload
+def search(
+    query: str, *, min_confidence: float = 0.1, return_first: Literal[True]
+) -> Optional[SearchResult]:
+    ...
+
+
+def search(
+    query: str, *, min_confidence: float = 0.1, return_first: bool = False
+) -> Union[list[abstract_connector.ConnectorResults], Optional[SearchResult]]:
+    """find books based on arbitrary keywords"""
     if not query:
-        return []
+        return None if return_first else []
-    results = []
 
     items = []
     for connector in get_connectors():
@@ -91,8 +81,12 @@ def search(query, min_confidence=0.1, return_first=False):
         items.append((url, connector))
 
     # load as many results as we can
-    results = asyncio.run(async_connector_search(query, items, min_confidence))
+    # failed requests will return None, so filter those out
-    results = [r for r in results if r]
+    results = [
+        r
+        for r in asyncio.run(async_connector_search(query, items, min_confidence))
+        if r
+    ]
 
     if return_first:
         # find the best result from all the responses and return that
@@ -100,11 +94,12 @@ def search(query, min_confidence=0.1, return_first=False):
         all_results = sorted(all_results, key=lambda r: r.confidence, reverse=True)
         return all_results[0] if all_results else None
 
-    # failed requests will return None, so filter those out
     return results
 
 
-def first_search_result(query, min_confidence=0.1):
+def first_search_result(
+    query: str, min_confidence: float = 0.1
+) -> Union[models.Edition, SearchResult, None]:
     """search until you find a result that fits"""
     # try local search first
     result = book_search.search(query, min_confidence=min_confidence, return_first=True)
@@ -114,13 +109,13 @@ def first_search_result(query, min_confidence=0.1):
     return search(query, min_confidence=min_confidence, return_first=True) or None
 
 
-def get_connectors():
+def get_connectors() -> Iterator[abstract_connector.AbstractConnector]:
     """load all connectors"""
     for info in models.Connector.objects.filter(active=True).order_by("priority").all():
         yield load_connector(info)
 
 
-def get_or_create_connector(remote_id):
+def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnector:
     """get the connector related to the object's server"""
     url = urlparse(remote_id)
     identifier = url.netloc
@@ -143,8 +138,8 @@ def get_or_create_connector(remote_id):
     return load_connector(connector_info)
 
 
-@app.task(queue=LOW)
+@app.task(queue=CONNECTORS)
-def load_more_data(connector_id, book_id):
+def load_more_data(connector_id: str, book_id: str) -> None:
     """background the work of getting all 10,000 editions of LoTR"""
     connector_info = models.Connector.objects.get(id=connector_id)
     connector = load_connector(connector_info)
@@ -152,8 +147,10 @@ def load_more_data(connector_id, book_id):
     connector.expand_book_data(book)
 
 
-@app.task(queue=LOW)
+@app.task(queue=CONNECTORS)
-def create_edition_task(connector_id, work_id, data):
+def create_edition_task(
+    connector_id: int, work_id: int, data: Union[str, abstract_connector.JsonDict]
+) -> None:
     """separate task for each of the 10,000 editions of LoTR"""
     connector_info = models.Connector.objects.get(id=connector_id)
     connector = load_connector(connector_info)
@@ -161,23 +158,31 @@ def create_edition_task(connector_id, work_id, data):
     connector.create_edition_from_data(work, data)
 
 
-def load_connector(connector_info):
+def load_connector(
+    connector_info: models.Connector,
+) -> abstract_connector.AbstractConnector:
     """instantiate the connector class"""
     connector = importlib.import_module(
         f"bookwyrm.connectors.{connector_info.connector_file}"
     )
-    return connector.Connector(connector_info.identifier)
+    return connector.Connector(connector_info.identifier)  # type: ignore[no-any-return]
 
 
 @receiver(signals.post_save, sender="bookwyrm.FederatedServer")
 # pylint: disable=unused-argument
-def create_connector(sender, instance, created, *args, **kwargs):
+def create_connector(
+    sender: Any,
+    instance: models.FederatedServer,
+    created: Any,
+    *args: Any,
+    **kwargs: Any,
+) -> None:
     """create a connector to an external bookwyrm server"""
     if instance.application_type == "bookwyrm":
         get_or_create_connector(f"https://{instance.server_name}")
 
 
-def raise_not_valid_url(url):
+def raise_not_valid_url(url: str) -> None:
     """do some basic reality checks on the url"""
     parsed = urlparse(url)
     if not parsed.scheme in ["http", "https"]:

bookwyrm/connectors/inventaire.py

@@ -1,9 +1,10 @@
 """ inventaire data connector """
 import re
+from typing import Any, Union, Optional, Iterator, Iterable
 
 from bookwyrm import models
 from bookwyrm.book_search import SearchResult
-from .abstract_connector import AbstractConnector, Mapping
+from .abstract_connector import AbstractConnector, Mapping, JsonDict
 from .abstract_connector import get_data
 from .connector_manager import ConnectorException, create_edition_task
 
@@ -13,7 +14,7 @@ class Connector(AbstractConnector):
 
     generated_remote_link_field = "inventaire_id"
 
-    def __init__(self, identifier):
+    def __init__(self, identifier: str):
         super().__init__(identifier)
 
         get_first = lambda a: a[0]
@@ -60,13 +61,13 @@ class Connector(AbstractConnector):
             Mapping("died", remote_field="wdt:P570", formatter=get_first),
         ] + shared_mappings
 
-    def get_remote_id(self, value):
+    def get_remote_id(self, value: str) -> str:
         """convert an id/uri into a url"""
         return f"{self.books_url}?action=by-uris&uris={value}"
 
-    def get_book_data(self, remote_id):
+    def get_book_data(self, remote_id: str) -> JsonDict:
         data = get_data(remote_id)
-        extracted = list(data.get("entities").values())
+        extracted = list(data.get("entities", {}).values())
         try:
             data = extracted[0]
         except (KeyError, IndexError):
@@ -74,10 +75,16 @@ class Connector(AbstractConnector):
         # flatten the data so that images, uri, and claims are on the same level
         return {
             **data.get("claims", {}),
-            **{k: data.get(k) for k in ["uri", "image", "labels", "sitelinks", "type"]},
+            **{
+                k: data.get(k)
+                for k in ["uri", "image", "labels", "sitelinks", "type"]
+                if k in data
+            },
         }
 
-    def parse_search_data(self, data, min_confidence):
+    def parse_search_data(
+        self, data: JsonDict, min_confidence: float
+    ) -> Iterator[SearchResult]:
         for search_result in data.get("results", []):
             images = search_result.get("image")
             cover = f"{self.covers_url}/img/entities/{images[0]}" if images else None
@@ -96,8 +103,8 @@ class Connector(AbstractConnector):
                 connector=self,
             )
 
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
-        """got some daaaata"""
+        """got some data"""
         results = data.get("entities")
         if not results:
             return
@@ -114,35 +121,44 @@ class Connector(AbstractConnector):
                 connector=self,
             )
 
-    def is_work_data(self, data):
+    def is_work_data(self, data: JsonDict) -> bool:
         return data.get("type") == "work"
 
-    def load_edition_data(self, work_uri):
+    def load_edition_data(self, work_uri: str) -> JsonDict:
         """get a list of editions for a work"""
         # pylint: disable=line-too-long
         url = f"{self.books_url}?action=reverse-claims&property=wdt:P629&value={work_uri}&sort=true"
         return get_data(url)
 
-    def get_edition_from_work_data(self, data):
+    def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
-        data = self.load_edition_data(data.get("uri"))
+        work_uri = data.get("uri")
+        if not work_uri:
+            raise ConnectorException("Invalid URI")
+        data = self.load_edition_data(work_uri)
         try:
             uri = data.get("uris", [])[0]
         except IndexError:
             raise ConnectorException("Invalid book data")
         return self.get_book_data(self.get_remote_id(uri))
 
-    def get_work_from_edition_data(self, data):
+    def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
-        uri = data.get("wdt:P629", [None])[0]
+        try:
+            uri = data.get("wdt:P629", [])[0]
+        except IndexError:
+            raise ConnectorException("Invalid book data")
+
         if not uri:
             raise ConnectorException("Invalid book data")
         return self.get_book_data(self.get_remote_id(uri))
 
-    def get_authors_from_data(self, data):
+    def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
         authors = data.get("wdt:P50", [])
         for author in authors:
-            yield self.get_or_create_author(self.get_remote_id(author))
+            model = self.get_or_create_author(self.get_remote_id(author))
+            if model:
+                yield model
 
-    def expand_book_data(self, book):
+    def expand_book_data(self, book: models.Book) -> None:
         work = book
         # go from the edition to the work, if necessary
         if isinstance(book, models.Edition):
@@ -154,36 +170,45 @@ class Connector(AbstractConnector):
             # who knows, man
             return
 
-        for edition_uri in edition_options.get("uris"):
+        for edition_uri in edition_options.get("uris", []):
             remote_id = self.get_remote_id(edition_uri)
             create_edition_task.delay(self.connector.id, work.id, remote_id)
 
-    def create_edition_from_data(self, work, edition_data, instance=None):
+    def create_edition_from_data(
+        self,
+        work: models.Work,
+        edition_data: Union[str, JsonDict],
+        instance: Optional[models.Edition] = None,
+    ) -> Optional[models.Edition]:
         """pass in the url as data and then call the version in abstract connector"""
         if isinstance(edition_data, str):
             try:
                 edition_data = self.get_book_data(edition_data)
             except ConnectorException:
                 # who, indeed, knows
-                return
+                return None
-        super().create_edition_from_data(work, edition_data, instance=instance)
+        return super().create_edition_from_data(work, edition_data, instance=instance)
 
-    def get_cover_url(self, cover_blob, *_):
+    def get_cover_url(
+        self, cover_blob: Union[list[JsonDict], JsonDict], *_: Any
+    ) -> Optional[str]:
         """format the relative cover url into an absolute one:
         {"url": "/img/entities/e794783f01b9d4f897a1ea9820b96e00d346994f"}
         """
         # covers may or may not be a list
-        if isinstance(cover_blob, list) and len(cover_blob) > 0:
+        if isinstance(cover_blob, list):
+            if len(cover_blob) == 0:
+                return None
             cover_blob = cover_blob[0]
         cover_id = cover_blob.get("url")
-        if not cover_id:
+        if not isinstance(cover_id, str):
             return None
         # cover may or may not be an absolute url already
         if re.match(r"^http", cover_id):
             return cover_id
         return f"{self.covers_url}{cover_id}"
 
-    def resolve_keys(self, keys):
+    def resolve_keys(self, keys: Iterable[str]) -> list[str]:
         """cool, it's "wd:Q3156592" now what the heck does that mean"""
         results = []
         for uri in keys:
@@ -191,10 +216,10 @@ class Connector(AbstractConnector):
                 data = self.get_book_data(self.get_remote_id(uri))
             except ConnectorException:
                 continue
-            results.append(get_language_code(data.get("labels")))
+            results.append(get_language_code(data.get("labels", {})))
         return results
 
-    def get_description(self, links):
+    def get_description(self, links: JsonDict) -> str:
         """grab an extracted excerpt from wikipedia"""
         link = links.get("enwiki")
         if not link:
@@ -204,15 +229,15 @@ class Connector(AbstractConnector):
             data = get_data(url)
         except ConnectorException:
             return ""
-        return data.get("extract")
+        return data.get("extract", "")
 
-    def get_remote_id_from_model(self, obj):
+    def get_remote_id_from_model(self, obj: models.BookDataModel) -> str:
         """use get_remote_id to figure out the link from a model obj"""
         remote_id_value = obj.inventaire_id
         return self.get_remote_id(remote_id_value)
 
 
-def get_language_code(options, code="en"):
+def get_language_code(options: JsonDict, code: str = "en") -> Any:
     """when there are a bunch of translation but we need a single field"""
     result = options.get(code)
     if result:

bookwyrm/connectors/openlibrary.py

@@ -1,9 +1,13 @@
 """ openlibrary data connector """
 import re
+from typing import Any, Optional, Union, Iterator, Iterable
+
+from markdown import markdown
 
 from bookwyrm import models
 from bookwyrm.book_search import SearchResult
-from .abstract_connector import AbstractConnector, Mapping
+from bookwyrm.utils.sanitizer import clean
+from .abstract_connector import AbstractConnector, Mapping, JsonDict
 from .abstract_connector import get_data, infer_physical_format, unique_physical_format
 from .connector_manager import ConnectorException, create_edition_task
 from .openlibrary_languages import languages
@@ -14,7 +18,7 @@ class Connector(AbstractConnector):
 
     generated_remote_link_field = "openlibrary_link"
 
-    def __init__(self, identifier):
+    def __init__(self, identifier: str):
         super().__init__(identifier)
 
         get_first = lambda a, *args: a[0]
@@ -94,14 +98,14 @@ class Connector(AbstractConnector):
             Mapping("inventaire_id", remote_field="links", formatter=get_inventaire_id),
         ]
 
-    def get_book_data(self, remote_id):
+    def get_book_data(self, remote_id: str) -> JsonDict:
         data = get_data(remote_id)
         if data.get("type", {}).get("key") == "/type/redirect":
-            remote_id = self.base_url + data.get("location")
+            remote_id = self.base_url + data.get("location", "")
             return get_data(remote_id)
         return data
 
-    def get_remote_id_from_data(self, data):
+    def get_remote_id_from_data(self, data: JsonDict) -> str:
         """format a url from an openlibrary id field"""
         try:
             key = data["key"]
@@ -109,10 +113,10 @@ class Connector(AbstractConnector):
             raise ConnectorException("Invalid book data")
         return f"{self.books_url}{key}"
 
-    def is_work_data(self, data):
+    def is_work_data(self, data: JsonDict) -> bool:
         return bool(re.match(r"^[\/\w]+OL\d+W$", data["key"]))
 
-    def get_edition_from_work_data(self, data):
+    def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
         try:
             key = data["key"]
         except KeyError:
@@ -124,7 +128,7 @@ class Connector(AbstractConnector):
             raise ConnectorException("No editions for work")
         return edition
 
-    def get_work_from_edition_data(self, data):
+    def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
         try:
             key = data["works"][0]["key"]
         except (IndexError, KeyError):
@@ -132,7 +136,7 @@ class Connector(AbstractConnector):
         url = f"{self.books_url}{key}"
         return self.get_book_data(url)
 
-    def get_authors_from_data(self, data):
+    def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
         """parse author json and load or create authors"""
         for author_blob in data.get("authors", []):
             author_blob = author_blob.get("author", author_blob)
@@ -144,7 +148,7 @@ class Connector(AbstractConnector):
                 continue
             yield author
 
-    def get_cover_url(self, cover_blob, size="L"):
+    def get_cover_url(self, cover_blob: list[str], size: str = "L") -> Optional[str]:
         """ask openlibrary for the cover"""
         if not cover_blob:
             return None
@@ -152,8 +156,10 @@ class Connector(AbstractConnector):
         image_name = f"{cover_id}-{size}.jpg"
         return f"{self.covers_url}/b/id/{image_name}"
 
-    def parse_search_data(self, data, min_confidence):
+    def parse_search_data(
-        for idx, search_result in enumerate(data.get("docs")):
+        self, data: JsonDict, min_confidence: float
+    ) -> Iterator[SearchResult]:
+        for idx, search_result in enumerate(data.get("docs", [])):
             # build the remote id from the openlibrary key
             key = self.books_url + search_result["key"]
             author = search_result.get("author_name") or ["Unknown"]
@@ -174,7 +180,7 @@ class Connector(AbstractConnector):
                 confidence=confidence,
             )
 
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
         for search_result in list(data.values()):
             # build the remote id from the openlibrary key
             key = self.books_url + search_result["key"]
@@ -188,12 +194,12 @@ class Connector(AbstractConnector):
                 year=search_result.get("publish_date"),
             )
 
-    def load_edition_data(self, olkey):
+    def load_edition_data(self, olkey: str) -> JsonDict:
         """query openlibrary for editions of a work"""
         url = f"{self.books_url}/works/{olkey}/editions"
         return self.get_book_data(url)
 
-    def expand_book_data(self, book):
+    def expand_book_data(self, book: models.Book) -> None:
         work = book
         # go from the edition to the work, if necessary
         if isinstance(book, models.Edition):
@@ -206,14 +212,14 @@ class Connector(AbstractConnector):
             # who knows, man
             return
 
-        for edition_data in edition_options.get("entries"):
+        for edition_data in edition_options.get("entries", []):
             # does this edition have ANY interesting data?
             if ignore_edition(edition_data):
                 continue
             create_edition_task.delay(self.connector.id, work.id, edition_data)
 
 
-def ignore_edition(edition_data):
+def ignore_edition(edition_data: JsonDict) -> bool:
     """don't load a million editions that have no metadata"""
     # an isbn, we love to see it
     if edition_data.get("isbn_13") or edition_data.get("isbn_10"):
@@ -232,19 +238,30 @@ def ignore_edition(edition_data):
     return True
 
 
-def get_description(description_blob):
+def get_description(description_blob: Union[JsonDict, str]) -> str:
     """descriptions can be a string or a dict"""
     if isinstance(description_blob, dict):
-        return description_blob.get("value")
+        description = markdown(description_blob.get("value", ""))
-    return description_blob
+    else:
+        description = markdown(description_blob)
+
+    if (
+        description.startswith("<p>")
+        and description.endswith("</p>")
+        and description.count("<p>") == 1
+    ):
+        # If there is just one <p> tag and it is around the text remove it
+        return description[len("<p>") : -len("</p>")].strip()
+
+    return clean(description)
 
 
-def get_openlibrary_key(key):
+def get_openlibrary_key(key: str) -> str:
     """convert /books/OL27320736M into OL27320736M"""
     return key.split("/")[-1]
 
 
-def get_languages(language_blob):
+def get_languages(language_blob: Iterable[JsonDict]) -> list[Optional[str]]:
     """/language/eng -> English"""
     langs = []
     for lang in language_blob:
@@ -252,14 +269,14 @@ def get_languages(language_blob):
     return langs
 
 
-def get_dict_field(blob, field_name):
+def get_dict_field(blob: Optional[JsonDict], field_name: str) -> Optional[Any]:
     """extract the isni from the remote id data for the author"""
     if not blob or not isinstance(blob, dict):
         return None
     return blob.get(field_name)
 
 
-def get_wikipedia_link(links):
+def get_wikipedia_link(links: list[Any]) -> Optional[str]:
     """extract wikipedia links"""
     if not isinstance(links, list):
         return None
@@ -272,7 +289,7 @@ def get_wikipedia_link(links):
     return None
 
 
-def get_inventaire_id(links):
+def get_inventaire_id(links: list[Any]) -> Optional[str]:
     """extract and format inventaire ids"""
     if not isinstance(links, list):
         return None
@@ -282,11 +299,13 @@ def get_inventaire_id(links):
             continue
         if link.get("title") == "inventaire.io":
             iv_link = link.get("url")
+            if not isinstance(iv_link, str):
+                return None
             return iv_link.split("/")[-1]
     return None
 
 
-def pick_default_edition(options):
+def pick_default_edition(options: list[JsonDict]) -> Optional[JsonDict]:
     """favor physical copies with covers in english"""
     if not options:
         return None

bookwyrm/emailing.py

@@ -3,7 +3,7 @@ from django.core.mail import EmailMultiAlternatives
 from django.template.loader import get_template
 
 from bookwyrm import models, settings
-from bookwyrm.tasks import app, HIGH
+from bookwyrm.tasks import app, EMAIL
 from bookwyrm.settings import DOMAIN
 
 
@@ -18,6 +18,12 @@ def email_data():
     }
 
 
+def test_email(user):
+    """Just an admin checking if emails are sending"""
+    data = email_data()
+    send_email(user.email, *format_email("test", data))
+
+
 def email_confirmation_email(user):
     """newly registered users confirm email address"""
     data = email_data()
@@ -38,7 +44,7 @@ def password_reset_email(reset_code):
     data = email_data()
     data["reset_link"] = reset_code.link
     data["user"] = reset_code.user.display_name
-    send_email.delay(reset_code.user.email, *format_email("password_reset", data))
+    send_email(reset_code.user.email, *format_email("password_reset", data))
 
 
 def moderation_report_email(report):
@@ -69,7 +75,7 @@ def format_email(email_name, data):
     return (subject, html_content, text_content)
 
 
-@app.task(queue=HIGH)
+@app.task(queue=EMAIL)
 def send_email(recipient, subject, html_content, text_content):
     """use a task to send the email"""
     email = EmailMultiAlternatives(

bookwyrm/forms/admin.py

@@ -15,7 +15,7 @@ from .custom_form import CustomForm, StyledForm
 # pylint: disable=missing-class-docstring
 class ExpiryWidget(widgets.Select):
     def value_from_datadict(self, data, files, name):
-        """human-readable exiration time buckets"""
+        """human-readable expiration time buckets"""
         selected_string = super().value_from_datadict(data, files, name)
 
         if selected_string == "day":
@@ -55,11 +55,46 @@ class CreateInviteForm(CustomForm):
 class SiteForm(CustomForm):
     class Meta:
         model = models.SiteSettings
-        exclude = ["admin_code", "install_mode"]
+        fields = [
+            "name",
+            "instance_tagline",
+            "instance_description",
+            "instance_short_description",
+            "default_theme",
+            "code_of_conduct",
+            "privacy_policy",
+            "impressum",
+            "show_impressum",
+            "logo",
+            "logo_small",
+            "favicon",
+            "support_link",
+            "support_title",
+            "admin_email",
+            "footer_item",
+        ]
         widgets = {
             "instance_short_description": forms.TextInput(
                 attrs={"aria-describedby": "desc_instance_short_description"}
             ),
+        }
+
+
+class RegistrationForm(CustomForm):
+    class Meta:
+        model = models.SiteSettings
+        fields = [
+            "allow_registration",
+            "allow_invite_requests",
+            "registration_closed_text",
+            "invite_request_text",
+            "invite_request_question",
+            "invite_question_text",
+            "require_confirm_email",
+            "default_user_auth_group",
+        ]
+
+        widgets = {
             "require_confirm_email": forms.CheckboxInput(
                 attrs={"aria-describedby": "desc_require_confirm_email"}
             ),
@@ -69,6 +104,23 @@ class SiteForm(CustomForm):
         }
 
 
+class RegistrationLimitedForm(CustomForm):
+    class Meta:
+        model = models.SiteSettings
+        fields = [
+            "registration_closed_text",
+            "invite_request_text",
+            "invite_request_question",
+            "invite_question_text",
+        ]
+
+        widgets = {
+            "invite_request_text": forms.Textarea(
+                attrs={"aria-describedby": "desc_invite_request_text"}
+            ),
+        }
+
+
 class ThemeForm(CustomForm):
     class Meta:
         model = models.Theme

bookwyrm/forms/author.py

@@ -15,12 +15,15 @@ class AuthorForm(CustomForm):
             "aliases",
             "bio",
             "wikipedia_link",
+            "wikidata",
+            "website",
             "born",
             "died",
             "openlibrary_key",
             "inventaire_id",
             "librarything_key",
             "goodreads_key",
+            "isfdb",
             "isni",
         ]
         widgets = {
@@ -30,10 +33,12 @@ class AuthorForm(CustomForm):
             "wikipedia_link": forms.TextInput(
                 attrs={"aria-describedby": "desc_wikipedia_link"}
             ),
+            "wikidata": forms.TextInput(attrs={"aria-describedby": "desc_wikidata"}),
+            "website": forms.TextInput(attrs={"aria-describedby": "desc_website"}),
             "born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}),
             "died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}),
-            "oepnlibrary_key": forms.TextInput(
+            "openlibrary_key": forms.TextInput(
-                attrs={"aria-describedby": "desc_oepnlibrary_key"}
+                attrs={"aria-describedby": "desc_openlibrary_key"}
             ),
             "inventaire_id": forms.TextInput(
                 attrs={"aria-describedby": "desc_inventaire_id"}

bookwyrm/forms/books.py

@@ -1,8 +1,9 @@
 """ using django model forms """
 from django import forms
 
+from file_resubmit.widgets import ResubmitImageWidget
+
 from bookwyrm import models
-from bookwyrm.models.fields import ClearableFileInputWithWarning
 from .custom_form import CustomForm
 from .widgets import ArrayWidget, SelectDateWidget, Select
 
@@ -18,22 +19,37 @@ class CoverForm(CustomForm):
 class EditionForm(CustomForm):
     class Meta:
         model = models.Edition
-        exclude = [
+        fields = [
-            "remote_id",
+            "title",
-            "origin_id",
+            "sort_title",
-            "created_date",
+            "subtitle",
-            "updated_date",
+            "description",
-            "edition_rank",
+            "series",
-            "authors",
+            "series_number",
-            "parent_work",
+            "languages",
-            "shelves",
+            "subjects",
-            "connector",
+            "publishers",
-            "search_vector",
+            "first_published_date",
-            "links",
+            "published_date",
-            "file_links",
+            "cover",
+            "physical_format",
+            "physical_format_detail",
+            "pages",
+            "isbn_13",
+            "isbn_10",
+            "openlibrary_key",
+            "inventaire_id",
+            "goodreads_key",
+            "oclc_number",
+            "asin",
+            "aasin",
+            "isfdb",
         ]
         widgets = {
             "title": forms.TextInput(attrs={"aria-describedby": "desc_title"}),
+            "sort_title": forms.TextInput(
+                attrs={"aria-describedby": "desc_sort_title"}
+            ),
             "subtitle": forms.TextInput(attrs={"aria-describedby": "desc_subtitle"}),
             "description": forms.Textarea(
                 attrs={"aria-describedby": "desc_description"}
@@ -55,9 +71,7 @@ class EditionForm(CustomForm):
             "published_date": SelectDateWidget(
                 attrs={"aria-describedby": "desc_published_date"}
             ),
-            "cover": ClearableFileInputWithWarning(
+            "cover": ResubmitImageWidget(attrs={"aria-describedby": "desc_cover"}),
-                attrs={"aria-describedby": "desc_cover"}
-            ),
             "physical_format": Select(
                 attrs={"aria-describedby": "desc_physical_format"}
             ),
@@ -73,10 +87,15 @@ class EditionForm(CustomForm):
             "inventaire_id": forms.TextInput(
                 attrs={"aria-describedby": "desc_inventaire_id"}
             ),
+            "goodreads_key": forms.TextInput(
+                attrs={"aria-describedby": "desc_goodreads_key"}
+            ),
             "oclc_number": forms.TextInput(
                 attrs={"aria-describedby": "desc_oclc_number"}
             ),
             "ASIN": forms.TextInput(attrs={"aria-describedby": "desc_ASIN"}),
+            "AASIN": forms.TextInput(attrs={"aria-describedby": "desc_AASIN"}),
+            "isfdb": forms.TextInput(attrs={"aria-describedby": "desc_isfdb"}),
         }
 
 
@@ -91,6 +110,7 @@ class EditionFromWorkForm(CustomForm):
         model = models.Work
         fields = [
             "title",
+            "sort_title",
             "subtitle",
             "authors",
             "description",

bookwyrm/forms/edit_user.py

@@ -70,6 +70,22 @@ class DeleteUserForm(CustomForm):
         fields = ["password"]
 
 
+class MoveUserForm(CustomForm):
+    target = forms.CharField(widget=forms.TextInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+
+
+class AliasUserForm(CustomForm):
+    username = forms.CharField(widget=forms.TextInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+
+
 class ChangePasswordForm(CustomForm):
     current_password = forms.CharField(widget=forms.PasswordInput)
     confirm_password = forms.CharField(widget=forms.PasswordInput)

bookwyrm/forms/forms.py

@@ -25,6 +25,10 @@ class ImportForm(forms.Form):
     csv_file = forms.FileField()
 
 
+class ImportUserForm(forms.Form):
+    archive_file = forms.FileField()
+
+
 class ShelfForm(CustomForm):
     class Meta:
         model = models.Shelf

bookwyrm/forms/landing.py

@@ -8,6 +8,7 @@ import pyotp
 
 from bookwyrm import models
 from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import TWO_FACTOR_LOGIN_VALIDITY_WINDOW
 from .custom_form import CustomForm
 
 
@@ -108,7 +109,7 @@ class Confirm2FAForm(CustomForm):
         otp = self.data.get("otp")
         totp = pyotp.TOTP(self.instance.otp_secret)
 
-        if not totp.verify(otp):
+        if not totp.verify(otp, valid_window=TWO_FACTOR_LOGIN_VALIDITY_WINDOW):
 
             if self.instance.hotp_secret:
                 # maybe it's a backup code?

bookwyrm/forms/links.py

@@ -1,4 +1,5 @@
 """ using django model forms """
+
 from urllib.parse import urlparse
 
 from django.utils.translation import gettext_lazy as _
@@ -36,13 +37,15 @@ class FileLinkForm(CustomForm):
                         "This domain is blocked. Please contact your administrator if you think this is an error."
                     ),
                 )
-            elif models.FileLink.objects.filter(
+        if (
-                url=url, book=book, filetype=filetype
+            models.FileLink.objects.filter(url=url, book=book, filetype=filetype)
-            ).exists():
+            .exclude(pk=self.instance)
-                # pylint: disable=line-too-long
+            .exists()
-                self.add_error(
+        ):
-                    "url",
+            # pylint: disable=line-too-long
-                    _(
+            self.add_error(
-                        "This link with file type has already been added for this book. If it is not visible, the domain is still pending."
+                "url",
-                    ),
+                _(
-                )
+                    "This link with file type has already been added for this book. If it is not visible, the domain is still pending."
+                ),
+            )

bookwyrm/forms/lists.py

@@ -24,7 +24,7 @@ class SortListForm(forms.Form):
     sort_by = ChoiceField(
         choices=(
             ("order", _("List Order")),
-            ("title", _("Book Title")),
+            ("sort_title", _("Book Title")),
             ("rating", _("Rating")),
         ),
         label=_("Sort By"),

bookwyrm/forms/status.py

@@ -53,6 +53,7 @@ class QuotationForm(CustomForm):
             "sensitive",
             "privacy",
             "position",
+            "endposition",
             "position_mode",
         ]
 

bookwyrm/importers/__init__.py

@@ -1,6 +1,7 @@
 """ import classes """
 
 from .importer import Importer
+from .bookwyrm_import import BookwyrmImporter
 from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter

bookwyrm/importers/bookwyrm_import.py

@@ -0,0 +1,24 @@
+"""Import data from Bookwyrm export files"""
+from django.http import QueryDict
+
+from bookwyrm.models import User
+from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
+
+
+class BookwyrmImporter:
+    """Import a Bookwyrm User export file.
+    This is kind of a combination of an importer and a connector.
+    """
+
+    # pylint: disable=no-self-use
+    def process_import(
+        self, user: User, archive_file: bytes, settings: QueryDict
+    ) -> BookwyrmImportJob:
+        """import user data from a Bookwyrm export file"""
+
+        required = [k for k in settings if settings.get(k) == "on"]
+
+        job = BookwyrmImportJob.objects.create(
+            user=user, archive_file=archive_file, required=required
+        )
+        return job

bookwyrm/importers/calibre_import.py

@@ -1,4 +1,6 @@
 """ handle reading a csv from calibre """
+from typing import Any, Optional
+
 from bookwyrm.models import Shelf
 
 from . import Importer
@@ -9,7 +11,7 @@ class CalibreImporter(Importer):
 
     service = "Calibre"
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         # Add timestamp to row_mappings_guesses for date_added to avoid
         # integrity error
         row_mappings_guesses = []
@@ -23,6 +25,6 @@ class CalibreImporter(Importer):
         self.row_mappings_guesses = row_mappings_guesses
         super().__init__(*args, **kwargs)
 
-    def get_shelf(self, normalized_row):
+    def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         # Calibre export does not indicate which shelf to use. Use a default one for now
         return Shelf.TO_READ

bookwyrm/importers/importer.py

@@ -1,7 +1,10 @@
 """ handle reading a csv from an external service, defaults are from Goodreads """
 import csv
+from datetime import timedelta
+from typing import Iterable, Optional
+
 from django.utils import timezone
-from bookwyrm.models import ImportJob, ImportItem
+from bookwyrm.models import ImportJob, ImportItem, SiteSettings, User
 
 
 class Importer:
@@ -16,8 +19,8 @@ class Importer:
         ("id", ["id", "book id"]),
         ("title", ["title"]),
         ("authors", ["author", "authors", "primary author"]),
-        ("isbn_10", ["isbn10", "isbn"]),
+        ("isbn_10", ["isbn10", "isbn", "isbn/uid"]),
-        ("isbn_13", ["isbn13", "isbn", "isbns"]),
+        ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]),
         ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]),
         ("review_name", ["review name"]),
         ("review_body", ["my review", "review"]),
@@ -33,26 +36,48 @@ class Importer:
         "reading": ["currently-reading", "reading", "currently reading"],
     }
 
-    def create_job(self, user, csv_file, include_reviews, privacy):
+    # pylint: disable=too-many-locals
+    def create_job(
+        self, user: User, csv_file: Iterable[str], include_reviews: bool, privacy: str
+    ) -> ImportJob:
         """check over a csv and creates a database entry for the job"""
         csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
-        rows = enumerate(list(csv_reader))
+        rows = list(csv_reader)
+        if len(rows) < 1:
+            raise ValueError("CSV file is empty")
+
+        mappings = (
+            self.create_row_mappings(list(fieldnames))
+            if (fieldnames := csv_reader.fieldnames)
+            else {}
+        )
+
         job = ImportJob.objects.create(
             user=user,
             include_reviews=include_reviews,
             privacy=privacy,
-            mappings=self.create_row_mappings(csv_reader.fieldnames),
+            mappings=mappings,
             source=self.service,
         )
 
-        for index, entry in rows:
+        enforce_limit, allowed_imports = self.get_import_limit(user)
+        if enforce_limit and allowed_imports <= 0:
+            job.complete_job()
+            return job
+        for index, entry in enumerate(rows):
+            if enforce_limit and index >= allowed_imports:
+                break
             self.create_item(job, index, entry)
         return job
 
-    def update_legacy_job(self, job):
+    def update_legacy_job(self, job: ImportJob) -> None:
         """patch up a job that was in the old format"""
         items = job.items
-        headers = list(items.first().data.keys())
+        first_item = items.first()
+        if first_item is None:
+            return
+
+        headers = list(first_item.data.keys())
         job.mappings = self.create_row_mappings(headers)
         job.updated_date = timezone.now()
         job.save()
@@ -63,24 +88,24 @@ class Importer:
             item.normalized_data = normalized
             item.save()
 
-    def create_row_mappings(self, headers):
+    def create_row_mappings(self, headers: list[str]) -> dict[str, Optional[str]]:
         """guess what the headers mean"""
         mappings = {}
         for (key, guesses) in self.row_mappings_guesses:
-            value = [h for h in headers if h.lower() in guesses]
+            values = [h for h in headers if h.lower() in guesses]
-            value = value[0] if len(value) else None
+            value = values[0] if len(values) else None
             if value:
                 headers.remove(value)
             mappings[key] = value
         return mappings
 
-    def create_item(self, job, index, data):
+    def create_item(self, job: ImportJob, index: int, data: dict[str, str]) -> None:
         """creates and saves an import item"""
         normalized = self.normalize_row(data, job.mappings)
         normalized["shelf"] = self.get_shelf(normalized)
         ImportItem(job=job, index=index, data=data, normalized_data=normalized).save()
 
-    def get_shelf(self, normalized_row):
+    def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         """determine which shelf to use"""
         shelf_name = normalized_row.get("shelf")
         if not shelf_name:
@@ -91,11 +116,35 @@ class Importer:
         ]
         return shelf[0] if shelf else None
 
-    def normalize_row(self, entry, mappings):  # pylint: disable=no-self-use
+    # pylint: disable=no-self-use
+    def normalize_row(
+        self, entry: dict[str, str], mappings: dict[str, Optional[str]]
+    ) -> dict[str, Optional[str]]:
         """use the dataclass to create the formatted row of data"""
-        return {k: entry.get(v) for k, v in mappings.items()}
+        return {k: entry.get(v) if v else None for k, v in mappings.items()}
 
-    def create_retry_job(self, user, original_job, items):
+    # pylint: disable=no-self-use
+    def get_import_limit(self, user: User) -> tuple[int, int]:
+        """check if import limit is set and return how many imports are left"""
+        site_settings = SiteSettings.objects.get()
+        import_size_limit = site_settings.import_size_limit
+        import_limit_reset = site_settings.import_limit_reset
+        enforce_limit = import_size_limit and import_limit_reset
+        allowed_imports = 0
+
+        if enforce_limit:
+            time_range = timezone.now() - timedelta(days=import_limit_reset)
+            import_jobs = ImportJob.objects.filter(
+                user=user, created_date__gte=time_range
+            )
+            # pylint: disable=consider-using-generator
+            imported_books = sum([job.successful_item_count for job in import_jobs])
+            allowed_imports = import_size_limit - imported_books
+        return enforce_limit, allowed_imports
+
+    def create_retry_job(
+        self, user: User, original_job: ImportJob, items: list[ImportItem]
+    ) -> ImportJob:
         """retry items that didn't import"""
         job = ImportJob.objects.create(
             user=user,
@@ -106,7 +155,13 @@ class Importer:
             mappings=original_job.mappings,
             retry=True,
         )
-        for item in items:
+        enforce_limit, allowed_imports = self.get_import_limit(user)
+        if enforce_limit and allowed_imports <= 0:
+            job.complete_job()
+            return job
+        for index, item in enumerate(items):
+            if enforce_limit and index >= allowed_imports:
+                break
             # this will re-normalize the raw data
             self.create_item(job, item.index, item.data)
         return job

bookwyrm/importers/librarything_import.py

@@ -1,11 +1,16 @@
 """ handle reading a tsv from librarything """
 import re
+from typing import Optional
 
 from bookwyrm.models import Shelf
 
 from . import Importer
 
 
+def _remove_brackets(value: Optional[str]) -> Optional[str]:
+    return re.sub(r"\[|\]", "", value) if value else None
+
+
 class LibrarythingImporter(Importer):
     """csv downloads from librarything"""
 
@@ -13,16 +18,19 @@ class LibrarythingImporter(Importer):
     delimiter = "\t"
     encoding = "ISO-8859-1"
 
-    def normalize_row(self, entry, mappings):  # pylint: disable=no-self-use
+    def normalize_row(
+        self, entry: dict[str, str], mappings: dict[str, Optional[str]]
+    ) -> dict[str, Optional[str]]:  # pylint: disable=no-self-use
         """use the dataclass to create the formatted row of data"""
-        remove_brackets = lambda v: re.sub(r"\[|\]", "", v) if v else None
+        normalized = {
-        normalized = {k: remove_brackets(entry.get(v)) for k, v in mappings.items()}
+            k: _remove_brackets(entry.get(v) if v else None)
-        isbn_13 = normalized.get("isbn_13")
+            for k, v in mappings.items()
-        isbn_13 = isbn_13.split(", ") if isbn_13 else []
+        }
-        normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 0 else None
+        isbn_13 = value.split(", ") if (value := normalized.get("isbn_13")) else []
+        normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 1 else None
         return normalized
 
-    def get_shelf(self, normalized_row):
+    def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         if normalized_row["date_finished"]:
             return Shelf.READ_FINISHED
         if normalized_row["date_started"]:

bookwyrm/importers/openlibrary_import.py

@@ -1,4 +1,6 @@
 """ handle reading a csv from openlibrary"""
+from typing import Any
+
 from . import Importer
 
 
@@ -7,7 +9,7 @@ class OpenLibraryImporter(Importer):
 
     service = "OpenLibrary"
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         self.row_mappings_guesses.append(("openlibrary_key", ["edition id"]))
         self.row_mappings_guesses.append(("openlibrary_work_key", ["work id"]))
         super().__init__(*args, **kwargs)

bookwyrm/isbn/isbn.py

@@ -0,0 +1,128 @@
+""" Use the range message from isbn-international to hyphenate ISBNs """
+import os
+from typing import Optional
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+
+import requests
+
+from bookwyrm import settings
+
+
+def _get_rules(element: Element) -> list[Element]:
+    if (rules_el := element.find("Rules")) is not None:
+        return rules_el.findall("Rule")
+    return []
+
+
+class IsbnHyphenator:
+    """Class to manage the range message xml file and use it to hyphenate ISBNs"""
+
+    __range_message_url = "https://www.isbn-international.org/export_rangemessage.xml"
+    __range_file_path = os.path.join(
+        settings.BASE_DIR, "bookwyrm", "isbn", "RangeMessage.xml"
+    )
+    __element_tree = None
+
+    def update_range_message(self) -> None:
+        """Download the range message xml file and save it locally"""
+        response = requests.get(self.__range_message_url, timeout=15)
+        with open(self.__range_file_path, "w", encoding="utf-8") as file:
+            file.write(response.text)
+        self.__element_tree = None
+
+    def hyphenate(self, isbn_13: Optional[str]) -> Optional[str]:
+        """hyphenate the given ISBN-13 number using the range message"""
+        if isbn_13 is None:
+            return None
+
+        if self.__element_tree is None:
+            self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+        gs1_prefix = isbn_13[:3]
+        try:
+            reg_group = self.__find_reg_group(isbn_13, gs1_prefix)
+        except ValueError:
+            # if the reg groups are invalid, just return the original isbn
+            return isbn_13
+
+        if reg_group is None:
+            return isbn_13  # failed to hyphenate
+
+        registrant = self.__find_registrant(isbn_13, gs1_prefix, reg_group)
+        if registrant is None:
+            return isbn_13  # failed to hyphenate
+
+        publication = isbn_13[len(gs1_prefix) + len(reg_group) + len(registrant) : -1]
+        check_digit = isbn_13[-1:]
+        return "-".join((gs1_prefix, reg_group, registrant, publication, check_digit))
+
+    def __find_reg_group(self, isbn_13: str, gs1_prefix: str) -> Optional[str]:
+        if self.__element_tree is None:
+            self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+        ucc_prefixes_el = self.__element_tree.find("EAN.UCCPrefixes")
+        if ucc_prefixes_el is None:
+            return None
+
+        for ean_ucc_el in ucc_prefixes_el.findall("EAN.UCC"):
+            if (
+                prefix_el := ean_ucc_el.find("Prefix")
+            ) is not None and prefix_el.text == gs1_prefix:
+                for rule_el in _get_rules(ean_ucc_el):
+                    length_el = rule_el.find("Length")
+                    if length_el is None:
+                        continue
+                    length = int(text) if (text := length_el.text) else 0
+                    if length == 0:
+                        continue
+
+                    range_el = rule_el.find("Range")
+                    if range_el is None or range_el.text is None:
+                        continue
+
+                    reg_grp_range = [int(x[:length]) for x in range_el.text.split("-")]
+                    reg_group = isbn_13[len(gs1_prefix) : len(gs1_prefix) + length]
+                    if reg_grp_range[0] <= int(reg_group) <= reg_grp_range[1]:
+                        return reg_group
+                return None
+        return None
+
+    def __find_registrant(
+        self, isbn_13: str, gs1_prefix: str, reg_group: str
+    ) -> Optional[str]:
+        from_ind = len(gs1_prefix) + len(reg_group)
+
+        if self.__element_tree is None:
+            self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+        reg_groups_el = self.__element_tree.find("RegistrationGroups")
+        if reg_groups_el is None:
+            return None
+
+        for group_el in reg_groups_el.findall("Group"):
+            if (
+                prefix_el := group_el.find("Prefix")
+            ) is not None and prefix_el.text == "-".join((gs1_prefix, reg_group)):
+                for rule_el in _get_rules(group_el):
+                    length_el = rule_el.find("Length")
+                    if length_el is None:
+                        continue
+                    length = int(text) if (text := length_el.text) else 0
+                    if length == 0:
+                        continue
+
+                    range_el = rule_el.find("Range")
+                    if range_el is None or range_el.text is None:
+                        continue
+                    registrant_range = [
+                        int(x[:length]) for x in range_el.text.split("-")
+                    ]
+                    registrant = isbn_13[from_ind : from_ind + length]
+                    if registrant_range[0] <= int(registrant) <= registrant_range[1]:
+                        return registrant
+                return None
+        return None
+
+
+hyphenator_singleton = IsbnHyphenator()

bookwyrm/lists_stream.py

@@ -5,7 +5,7 @@ from django.db.models import signals, Count, Q
 
 from bookwyrm import models
 from bookwyrm.redis_store import RedisStore
-from bookwyrm.tasks import app, MEDIUM, HIGH
+from bookwyrm.tasks import app, LISTS
 
 
 class ListsStream(RedisStore):
@@ -24,8 +24,7 @@ class ListsStream(RedisStore):
 
     def add_list(self, book_list):
         """add a list to users' feeds"""
-        # the pipeline contains all the add-to-stream activities
+        self.add_object_to_stores(book_list, self.get_stores_for_object(book_list))
-        self.add_object_to_related_stores(book_list)
 
     def add_user_lists(self, viewer, user):
         """add a user's lists to another user's feed"""
@@ -86,18 +85,19 @@ class ListsStream(RedisStore):
             if group:
                 audience = audience.filter(
                     Q(id=book_list.user.id)  # if the user is the list's owner
-                    | Q(following=book_list.user)  # if the user is following the pwmer
+                    | Q(following=book_list.user)  # if the user is following the owner
                     # if a user is in the group
                     | Q(memberships__group__id=book_list.group.id)
                 )
             else:
                 audience = audience.filter(
                     Q(id=book_list.user.id)  # if the user is the list's owner
-                    | Q(following=book_list.user)  # if the user is following the pwmer
+                    | Q(following=book_list.user)  # if the user is following the owner
                 )
         return audience.distinct()
 
     def get_stores_for_object(self, obj):
+        """the stores that an object belongs in"""
         return [self.stream_id(u) for u in self.get_audience(obj)]
 
     def get_lists_for_user(self, user):  # pylint: disable=no-self-use
@@ -217,14 +217,14 @@ def add_list_on_account_create_command(user_id):
 
 
 # ---- TASKS
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def populate_lists_task(user_id):
     """background task for populating an empty list stream"""
     user = models.User.objects.get(id=user_id)
     ListsStream().populate_lists(user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def remove_list_task(list_id, re_add=False):
     """remove a list from any stream it might be in"""
     stores = models.User.objects.filter(local=True, is_active=True).values_list(
@@ -233,20 +233,20 @@ def remove_list_task(list_id, re_add=False):
 
     # delete for every store
     stores = [ListsStream().stream_id(idx) for idx in stores]
-    ListsStream().remove_object_from_related_stores(list_id, stores=stores)
+    ListsStream().remove_object_from_stores(list_id, stores)
 
     if re_add:
         add_list_task.delay(list_id)
 
 
-@app.task(queue=HIGH)
+@app.task(queue=LISTS)
 def add_list_task(list_id):
     """add a list to any stream it should be in"""
     book_list = models.List.objects.get(id=list_id)
     ListsStream().add_list(book_list)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
     """remove all lists by a user from a viewer's stream"""
     viewer = models.User.objects.get(id=viewer_id)
@@ -254,7 +254,7 @@ def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
     ListsStream().remove_user_lists(viewer, user, exclude_privacy=exclude_privacy)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def add_user_lists_task(viewer_id, user_id):
     """add all lists by a user to a viewer's stream"""
     viewer = models.User.objects.get(id=viewer_id)

bookwyrm/management/commands/compile_themes.py

@@ -0,0 +1,48 @@
+""" Our own command to all scss themes """
+import glob
+import os
+
+import sass
+
+from django.core.management.base import BaseCommand
+
+from sass_processor.apps import APPS_INCLUDE_DIRS
+from sass_processor.processor import SassProcessor
+from sass_processor.utils import get_custom_functions
+
+from bookwyrm import settings
+
+
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "SCSS compile all BookWyrm themes"
+
+    # pylint: disable=unused-argument
+    def handle(self, *args, **options):
+        """compile"""
+        themes_dir = os.path.join(
+            settings.BASE_DIR, "bookwyrm", "static", "css", "themes", "*.scss"
+        )
+        for theme_scss in glob.glob(themes_dir):
+            basename, _ = os.path.splitext(theme_scss)
+            theme_css = f"{basename}.css"
+            self.compile_sass(theme_scss, theme_css)
+
+    def compile_sass(self, sass_path, css_path):
+        compile_kwargs = {
+            "filename": sass_path,
+            "include_paths": SassProcessor.include_paths + APPS_INCLUDE_DIRS,
+            "custom_functions": get_custom_functions(),
+            "precision": getattr(settings, "SASS_PRECISION", 8),
+            "output_style": getattr(
+                settings,
+                "SASS_OUTPUT_STYLE",
+                "nested" if settings.DEBUG else "compressed",
+            ),
+        }
+
+        content = sass.compile(**compile_kwargs)
+        with open(css_path, "w") as f:
+            f.write(content)
+        self.stdout.write("Compiled SASS/SCSS file: '{0}'\n".format(sass_path))

bookwyrm/management/commands/confirm_email.py

@@ -0,0 +1,19 @@
+""" manually confirm e-mail of user """
+from django.core.management.base import BaseCommand
+
+from bookwyrm import models
+
+
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Manually confirm email for user"
+
+    def add_arguments(self, parser):
+        parser.add_argument("username")
+
+    def handle(self, *args, **options):
+        name = options["username"]
+        user = models.User.objects.get(localname=name)
+        user.reactivate()
+        self.stdout.write(self.style.SUCCESS("User's email is now confirmed."))

bookwyrm/management/commands/deduplicate_book_data.py

@@ -3,38 +3,7 @@ merge book data objects """
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from bookwyrm import models
-
+from bookwyrm.management.merge import merge_objects
-
-def update_related(canonical, obj):
-    """update all the models with fk to the object being removed"""
-    # move related models to canonical
-    related_models = [
-        (r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
-    ]
-    for (related_field, related_model) in related_models:
-        related_objs = related_model.objects.filter(**{related_field: obj})
-        for related_obj in related_objs:
-            print("replacing in", related_model.__name__, related_field, related_obj.id)
-            try:
-                setattr(related_obj, related_field, canonical)
-                related_obj.save()
-            except TypeError:
-                getattr(related_obj, related_field).add(canonical)
-                getattr(related_obj, related_field).remove(obj)
-
-
-def copy_data(canonical, obj):
-    """try to get the most data possible"""
-    for data_field in obj._meta.get_fields():
-        if not hasattr(data_field, "activitypub_field"):
-            continue
-        data_value = getattr(obj, data_field.name)
-        if not data_value:
-            continue
-        if not getattr(canonical, data_field.name):
-            print("setting data field", data_field.name, data_value)
-            setattr(canonical, data_field.name, data_value)
-    canonical.save()
 
 
 def dedupe_model(model):
@@ -61,19 +30,16 @@ def dedupe_model(model):
             print("keeping", canonical.remote_id)
             for obj in objs[1:]:
                 print(obj.remote_id)
-                copy_data(canonical, obj)
+                merge_objects(canonical, obj)
-                update_related(canonical, obj)
-                # remove the outdated entry
-                obj.delete()
 
 
 class Command(BaseCommand):
-    """dedplucate allllll the book data models"""
+    """deduplicate allllll the book data models"""
 
     help = "merges duplicate book data"
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
-        """run deudplications"""
+        """run deduplications"""
         dedupe_model(models.Edition)
         dedupe_model(models.Work)
         dedupe_model(models.Author)

bookwyrm/management/commands/erase_deleted_user_data.py

@@ -0,0 +1,43 @@
+""" Erase any data stored about deleted users """
+import sys
+from django.core.management.base import BaseCommand, CommandError
+from bookwyrm import models
+from bookwyrm.models.user import erase_user_data
+
+# pylint: disable=missing-function-docstring
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Remove Two Factor Authorisation from user"
+
+    def add_arguments(self, parser):  # pylint: disable=no-self-use
+        parser.add_argument(
+            "--dryrun",
+            action="store_true",
+            help="Preview users to be cleared without altering the database",
+        )
+
+    def handle(self, *args, **options):  # pylint: disable=unused-argument
+
+        # Check for anything fishy
+        bad_state = models.User.objects.filter(is_deleted=True, is_active=True)
+        if bad_state.exists():
+            raise CommandError(
+                f"{bad_state.count()} user(s) marked as both active and deleted"
+            )
+
+        deleted_users = models.User.objects.filter(is_deleted=True)
+        self.stdout.write(f"Found {deleted_users.count()} deleted users")
+        if options["dryrun"]:
+            self.stdout.write("\n".join(u.username for u in deleted_users[:5]))
+            if deleted_users.count() > 5:
+                self.stdout.write("... and more")
+            sys.exit()
+
+        self.stdout.write("Erasing user data:")
+        for user_id in deleted_users.values_list("id", flat=True):
+            erase_user_data.delay(user_id)
+            self.stdout.write(".", ending="")
+
+        self.stdout.write("")
+        self.stdout.write("Tasks created successfully")

bookwyrm/management/commands/erase_streams.py

@@ -4,12 +4,7 @@ import redis
 
 from bookwyrm import settings
 
-r = redis.Redis(
+r = redis.from_url(settings.REDIS_ACTIVITY_URL)
-    host=settings.REDIS_ACTIVITY_HOST,
-    port=settings.REDIS_ACTIVITY_PORT,
-    password=settings.REDIS_ACTIVITY_PASSWORD,
-    db=settings.REDIS_ACTIVITY_DB_INDEX,
-)
 
 
 def erase_streams():

bookwyrm/management/commands/initdb.py

@@ -8,54 +8,64 @@ from bookwyrm import models
 
 def init_groups():
     """permission levels"""
-    groups = ["admin", "moderator", "editor"]
+    groups = ["admin", "owner", "moderator", "editor"]
     for group in groups:
-        Group.objects.create(name=group)
+        Group.objects.get_or_create(name=group)
 
 
 def init_permissions():
     """permission types"""
     permissions = [
+        {
+            "codename": "manage_registration",
+            "name": "allow or prevent user registration",
+            "groups": ["admin"],
+        },
+        {
+            "codename": "system_administration",
+            "name": "technical controls",
+            "groups": ["admin"],
+        },
         {
             "codename": "edit_instance_settings",
             "name": "change the instance info",
-            "groups": ["admin"],
+            "groups": ["admin", "owner"],
         },
         {
             "codename": "set_user_group",
             "name": "change what group a user is in",
-            "groups": ["admin", "moderator"],
+            "groups": ["admin", "owner", "moderator"],
         },
         {
             "codename": "control_federation",
             "name": "control who to federate with",
-            "groups": ["admin", "moderator"],
+            "groups": ["admin", "owner", "moderator"],
         },
         {
             "codename": "create_invites",
             "name": "issue invitations to join",
-            "groups": ["admin", "moderator"],
+            "groups": ["admin", "owner", "moderator"],
         },
         {
             "codename": "moderate_user",
             "name": "deactivate or silence a user",
-            "groups": ["admin", "moderator"],
+            "groups": ["admin", "owner", "moderator"],
         },
         {
             "codename": "moderate_post",
             "name": "delete other users' posts",
-            "groups": ["admin", "moderator"],
+            "groups": ["admin", "owner", "moderator"],
         },
         {
             "codename": "edit_book",
             "name": "edit book info",
-            "groups": ["admin", "moderator", "editor"],
+            "groups": ["admin", "owner", "moderator", "editor"],
         },
     ]
 
     content_type = ContentType.objects.get_for_model(models.User)
     for permission in permissions:
-        permission_obj = Permission.objects.create(
+        permission_obj, _ = Permission.objects.get_or_create(
             codename=permission["codename"],
             name=permission["name"],
             content_type=content_type,
@@ -107,10 +117,12 @@ def init_connectors():
 
 def init_settings():
     """info about the instance"""
+    group_editor = Group.objects.filter(name="editor").first()
     models.SiteSettings.objects.create(
         support_link="https://www.patreon.com/bookwyrm",
         support_title="Patreon",
         install_mode=True,
+        default_user_auth_group=group_editor,
     )
 
 

bookwyrm/management/commands/instance_version.py

@@ -1,54 +0,0 @@
-""" Get your admin code to allow install """
-from django.core.management.base import BaseCommand
-
-from bookwyrm import models
-from bookwyrm.settings import VERSION
-
-
-# pylint: disable=no-self-use
-class Command(BaseCommand):
-    """command-line options"""
-
-    help = "What version is this?"
-
-    def add_arguments(self, parser):
-        """specify which function to run"""
-        parser.add_argument(
-            "--current",
-            action="store_true",
-            help="Version stored in database",
-        )
-        parser.add_argument(
-            "--target",
-            action="store_true",
-            help="Version stored in settings",
-        )
-        parser.add_argument(
-            "--update",
-            action="store_true",
-            help="Update database version",
-        )
-
-    # pylint: disable=unused-argument
-    def handle(self, *args, **options):
-        """execute init"""
-        site = models.SiteSettings.objects.get()
-        current = site.version or "0.0.1"
-        target = VERSION
-        if options.get("current"):
-            print(current)
-            return
-
-        if options.get("target"):
-            print(target)
-            return
-
-        if options.get("update"):
-            site.version = target
-            site.save()
-            return
-
-        if current != target:
-            print(f"{current}/{target}")
-        else:
-            print(current)

bookwyrm/management/commands/merge_authors.py

@@ -0,0 +1,12 @@
+""" PROCEED WITH CAUTION: uses deduplication fields to permanently
+merge author data objects """
+from bookwyrm import models
+from bookwyrm.management.merge_command import MergeCommand
+
+
+class Command(MergeCommand):
+    """merges two authors by ID"""
+
+    help = "merges specified authors into one"
+
+    MODEL = models.Author

bookwyrm/management/commands/merge_editions.py

@@ -0,0 +1,12 @@
+""" PROCEED WITH CAUTION: uses deduplication fields to permanently
+merge edition data objects """
+from bookwyrm import models
+from bookwyrm.management.merge_command import MergeCommand
+
+
+class Command(MergeCommand):
+    """merges two editions by ID"""
+
+    help = "merges specified editions into one"
+
+    MODEL = models.Edition

bookwyrm/management/commands/merge_works.py

@@ -0,0 +1,12 @@
+""" PROCEED WITH CAUTION: uses deduplication fields to permanently
+merge work data objects """
+from bookwyrm import models
+from bookwyrm.management.merge_command import MergeCommand
+
+
+class Command(MergeCommand):
+    """merges two works by ID"""
+
+    help = "merges specified works into one"
+
+    MODEL = models.Work

bookwyrm/management/commands/remove_editions.py

@@ -33,10 +33,10 @@ def remove_editions():
 
 
 class Command(BaseCommand):
-    """dedplucate allllll the book data models"""
+    """deduplicate allllll the book data models"""
 
     help = "merges duplicate book data"
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
-        """run deudplications"""
+        """run deduplications"""
         remove_editions()

bookwyrm/management/commands/remove_remote_user_preview_images.py

@@ -0,0 +1,40 @@
+""" Remove preview images for remote users """
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+
+from bookwyrm import models, preview_images
+
+
+# pylint: disable=line-too-long
+class Command(BaseCommand):
+    """Remove preview images for remote users"""
+
+    help = "Remove preview images for remote users"
+
+    # pylint: disable=no-self-use,unused-argument
+    def handle(self, *args, **options):
+        """generate preview images"""
+        self.stdout.write(
+            "   | Hello! I will be removing preview images from remote users."
+        )
+        self.stdout.write(
+            "🧑‍🚒 ⎨ This might take quite long if your instance has a lot of remote users."
+        )
+        self.stdout.write("   | ✧ Thank you for your patience ✧")
+
+        users = models.User.objects.filter(local=False).exclude(
+            Q(preview_image="") | Q(preview_image=None)
+        )
+
+        if len(users) > 0:
+            self.stdout.write(
+                f"   → Remote user preview images ({len(users)}): ", ending=""
+            )
+            for user in users:
+                preview_images.remove_user_preview_image_task.delay(user.id)
+                self.stdout.write(".", ending="")
+            self.stdout.write(" OK 🖼")
+        else:
+            self.stdout.write(f"   | There was no remote users with preview images.")
+
+        self.stdout.write("🧑‍🚒 ⎨ I’m all done! ✧ Enjoy ✧")

bookwyrm/management/commands/repair_editions.py

@@ -0,0 +1,21 @@
+""" Repair editions with missing works """
+from django.core.management.base import BaseCommand
+from bookwyrm import models
+
+
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Repairs an edition that is in a broken state"
+
+    # pylint: disable=unused-argument
+    def handle(self, *args, **options):
+        """Find and repair broken editions"""
+        # Find broken editions
+        editions = models.Edition.objects.filter(parent_work__isnull=True)
+        self.stdout.write(f"Repairing {editions.count()} edition(s):")
+
+        # Do repair
+        for edition in editions:
+            edition.repair()
+            self.stdout.write(".", ending="")

bookwyrm/management/commands/revoke_preview_image_tasks.py

@@ -9,7 +9,7 @@ class Command(BaseCommand):
 
     # pylint: disable=unused-argument
     def handle(self, *args, **options):
-        """reveoke nonessential low priority tasks"""
+        """revoke nonessential low priority tasks"""
         types = [
             "bookwyrm.preview_images.generate_edition_preview_image_task",
             "bookwyrm.preview_images.generate_user_preview_image_task",

bookwyrm/management/merge.py

@@ -0,0 +1,50 @@
+from django.db.models import ManyToManyField
+
+
+def update_related(canonical, obj):
+    """update all the models with fk to the object being removed"""
+    # move related models to canonical
+    related_models = [
+        (r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
+    ]
+    for (related_field, related_model) in related_models:
+        # Skip the ManyToMany fields that aren’t auto-created. These
+        # should have a corresponding OneToMany field in the model for
+        # the linking table anyway. If we update it through that model
+        # instead then we won’t lose the extra fields in the linking
+        # table.
+        related_field_obj = related_model._meta.get_field(related_field)
+        if isinstance(related_field_obj, ManyToManyField):
+            through = related_field_obj.remote_field.through
+            if not through._meta.auto_created:
+                continue
+        related_objs = related_model.objects.filter(**{related_field: obj})
+        for related_obj in related_objs:
+            print("replacing in", related_model.__name__, related_field, related_obj.id)
+            try:
+                setattr(related_obj, related_field, canonical)
+                related_obj.save()
+            except TypeError:
+                getattr(related_obj, related_field).add(canonical)
+                getattr(related_obj, related_field).remove(obj)
+
+
+def copy_data(canonical, obj):
+    """try to get the most data possible"""
+    for data_field in obj._meta.get_fields():
+        if not hasattr(data_field, "activitypub_field"):
+            continue
+        data_value = getattr(obj, data_field.name)
+        if not data_value:
+            continue
+        if not getattr(canonical, data_field.name):
+            print("setting data field", data_field.name, data_value)
+            setattr(canonical, data_field.name, data_value)
+    canonical.save()
+
+
+def merge_objects(canonical, obj):
+    copy_data(canonical, obj)
+    update_related(canonical, obj)
+    # remove the outdated entry
+    obj.delete()

bookwyrm/management/merge_command.py

@@ -0,0 +1,29 @@
+from bookwyrm.management.merge import merge_objects
+from django.core.management.base import BaseCommand
+
+
+class MergeCommand(BaseCommand):
+    """base class for merge commands"""
+
+    def add_arguments(self, parser):
+        """add the arguments for this command"""
+        parser.add_argument("--canonical", type=int, required=True)
+        parser.add_argument("--other", type=int, required=True)
+
+    # pylint: disable=no-self-use,unused-argument
+    def handle(self, *args, **options):
+        """merge the two objects"""
+        model = self.MODEL
+
+        try:
+            canonical = model.objects.get(id=options["canonical"])
+        except model.DoesNotExist:
+            print("canonical book doesn’t exist!")
+            return
+        try:
+            other = model.objects.get(id=options["other"])
+        except model.DoesNotExist:
+            print("other book doesn’t exist!")
+            return
+
+        merge_objects(canonical, other)

bookwyrm/middleware/__init__.py

@@ -1,3 +1,4 @@
 """ look at all this nice middleware! """
 from .timezone_middleware import TimezoneMiddleware
 from .ip_middleware import IPBlocklistMiddleware
+from .file_too_big import FileTooBig

bookwyrm/middleware/file_too_big.py

@@ -0,0 +1,30 @@
+"""Middleware to display a custom 413 error page"""
+
+from django.http import HttpResponse
+from django.shortcuts import render
+from django.core.exceptions import RequestDataTooBig
+
+
+class FileTooBig:
+    """Middleware to display a custom page when a
+    RequestDataTooBig exception is thrown"""
+
+    def __init__(self, get_response):
+        """boilerplate __init__ from Django docs"""
+
+        self.get_response = get_response
+
+    def __call__(self, request):
+        """If RequestDataTooBig is thrown, render the 413 error page"""
+
+        try:
+            body = request.body  # pylint: disable=unused-variable
+
+        except RequestDataTooBig:
+
+            rendered = render(request, "413.html")
+            response = HttpResponse(rendered)
+            return response
+
+        response = self.get_response(request)
+        return response

bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py

@@ -1467,7 +1467,7 @@ class Migration(migrations.Migration):
                 (
                     "expiry",
                     models.DateTimeField(
-                        default=bookwyrm.models.site.get_passowrd_reset_expiry
+                        default=bookwyrm.models.site.get_password_reset_expiry
                     ),
                 ),
                 (

bookwyrm/migrations/0101_auto_20210929_1847.py

@@ -6,7 +6,7 @@ from bookwyrm.connectors.abstract_connector import infer_physical_format
 
 
 def infer_format(app_registry, schema_editor):
-    """set the new phsyical format field based on existing format data"""
+    """set the new physical format field based on existing format data"""
     db_alias = schema_editor.connection.alias
 
     editions = (

bookwyrm/migrations/0102_remove_connector_local.py

@@ -5,7 +5,7 @@ from bookwyrm.settings import DOMAIN
 
 
 def remove_self_connector(app_registry, schema_editor):
-    """set the new phsyical format field based on existing format data"""
+    """set the new physical format field based on existing format data"""
     db_alias = schema_editor.connection.alias
     app_registry.get_model("bookwyrm", "Connector").objects.using(db_alias).filter(
         connector_file="self_connector"

bookwyrm/migrations/0166_sitesettings_imports_enabled.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-17 21:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0165_alter_inviterequest_answer"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="imports_enabled",
+            field=models.BooleanField(default=True),
+        ),
+    ]

bookwyrm/migrations/0167_auto_20221125_1900.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2022-11-25 19:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0166_sitesettings_imports_enabled"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="impressum",
+            field=models.TextField(default="Add a impressum here."),
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="show_impressum",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0167_sitesettings_import_size_limit.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2022-12-05 13:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0166_sitesettings_imports_enabled"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="import_size_limit",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="import_limit_reset",
+            field=models.IntegerField(default=0),
+        ),
+    ]

bookwyrm/migrations/0168_auto_20221205_1701.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.16 on 2022-12-05 17:01
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0167_auto_20221125_1900"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="aasin",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="aasin",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+    ]

bookwyrm/migrations/0168_auto_20221205_2331.py

@@ -0,0 +1,63 @@
+""" I added two new permission types and a new group to the management command that
+creates the database on install, this creates them for existing instances """
+# Generated by Django 3.2.16 on 2022-12-05 23:31
+
+from django.db import migrations
+
+
+def create_groups_and_perms(apps, schema_editor):
+    """create the new "owner" group and "system admin" permission"""
+    db_alias = schema_editor.connection.alias
+    group_model = apps.get_model("auth", "Group")
+    # Add the "owner" group, if needed
+    owner_group, group_created = group_model.objects.using(db_alias).get_or_create(
+        name="owner"
+    )
+
+    # Create perms, if needed
+    user_model = apps.get_model("bookwyrm", "User")
+    content_type_model = apps.get_model("contenttypes", "ContentType")
+    content_type = content_type_model.objects.get_for_model(user_model)
+    perms_model = apps.get_model("auth", "Permission")
+    reg_perm, perm_created = perms_model.objects.using(db_alias).get_or_create(
+        codename="manage_registration",
+        name="allow or prevent user registration",
+        content_type=content_type,
+    )
+    admin_perm, admin_perm_created = perms_model.objects.using(db_alias).get_or_create(
+        codename="system_administration",
+        name="technical controls",
+        content_type=content_type,
+    )
+
+    # Add perms to the group if anything was created
+    if group_created or perm_created or admin_perm_created:
+        perms = [
+            "edit_instance_settings",
+            "set_user_group",
+            "control_federation",
+            "create_invites",
+            "moderate_user",
+            "moderate_post",
+            "edit_book",
+        ]
+        owner_group.permissions.set(
+            perms_model.objects.using(db_alias).filter(codename__in=perms).all()
+        )
+
+        # also extend these perms to admins
+        # This is get or create so the tests don't fail -- it should already exist
+        admin_group, _ = group_model.objects.using(db_alias).get_or_create(name="admin")
+        admin_group.permissions.add(reg_perm)
+        admin_group.permissions.add(admin_perm)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0167_auto_20221125_1900"),
+    ]
+
+    operations = [
+        migrations.RunPython(create_groups_and_perms, migrations.RunPython.noop)
+    ]

bookwyrm/migrations/0169_auto_20221206_0902.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.16 on 2022-12-06 09:02
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0168_auto_20221205_1701"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="isfdb",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="isfdb",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+    ]

bookwyrm/migrations/0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2022-12-11 20:00
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0168_auto_20221205_2331"),
+        ("bookwyrm", "0169_auto_20221206_0902"),
+    ]
+
+    operations = []

bookwyrm/migrations/0171_alter_user_preferred_timezone.py

@@ -0,0 +1,631 @@
+# Generated by Django 3.2.16 on 2022-12-19 15:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_timezone",
+            field=models.CharField(
+                choices=[
+                    ("Africa/Abidjan", "Africa/Abidjan"),
+                    ("Africa/Accra", "Africa/Accra"),
+                    ("Africa/Addis_Ababa", "Africa/Addis_Ababa"),
+                    ("Africa/Algiers", "Africa/Algiers"),
+                    ("Africa/Asmara", "Africa/Asmara"),
+                    ("Africa/Asmera", "Africa/Asmera"),
+                    ("Africa/Bamako", "Africa/Bamako"),
+                    ("Africa/Bangui", "Africa/Bangui"),
+                    ("Africa/Banjul", "Africa/Banjul"),
+                    ("Africa/Bissau", "Africa/Bissau"),
+                    ("Africa/Blantyre", "Africa/Blantyre"),
+                    ("Africa/Brazzaville", "Africa/Brazzaville"),
+                    ("Africa/Bujumbura", "Africa/Bujumbura"),
+                    ("Africa/Cairo", "Africa/Cairo"),
+                    ("Africa/Casablanca", "Africa/Casablanca"),
+                    ("Africa/Ceuta", "Africa/Ceuta"),
+                    ("Africa/Conakry", "Africa/Conakry"),
+                    ("Africa/Dakar", "Africa/Dakar"),
+                    ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"),
+                    ("Africa/Djibouti", "Africa/Djibouti"),
+                    ("Africa/Douala", "Africa/Douala"),
+                    ("Africa/El_Aaiun", "Africa/El_Aaiun"),
+                    ("Africa/Freetown", "Africa/Freetown"),
+                    ("Africa/Gaborone", "Africa/Gaborone"),
+                    ("Africa/Harare", "Africa/Harare"),
+                    ("Africa/Johannesburg", "Africa/Johannesburg"),
+                    ("Africa/Juba", "Africa/Juba"),
+                    ("Africa/Kampala", "Africa/Kampala"),
+                    ("Africa/Khartoum", "Africa/Khartoum"),
+                    ("Africa/Kigali", "Africa/Kigali"),
+                    ("Africa/Kinshasa", "Africa/Kinshasa"),
+                    ("Africa/Lagos", "Africa/Lagos"),
+                    ("Africa/Libreville", "Africa/Libreville"),
+                    ("Africa/Lome", "Africa/Lome"),
+                    ("Africa/Luanda", "Africa/Luanda"),
+                    ("Africa/Lubumbashi", "Africa/Lubumbashi"),
+                    ("Africa/Lusaka", "Africa/Lusaka"),
+                    ("Africa/Malabo", "Africa/Malabo"),
+                    ("Africa/Maputo", "Africa/Maputo"),
+                    ("Africa/Maseru", "Africa/Maseru"),
+                    ("Africa/Mbabane", "Africa/Mbabane"),
+                    ("Africa/Mogadishu", "Africa/Mogadishu"),
+                    ("Africa/Monrovia", "Africa/Monrovia"),
+                    ("Africa/Nairobi", "Africa/Nairobi"),
+                    ("Africa/Ndjamena", "Africa/Ndjamena"),
+                    ("Africa/Niamey", "Africa/Niamey"),
+                    ("Africa/Nouakchott", "Africa/Nouakchott"),
+                    ("Africa/Ouagadougou", "Africa/Ouagadougou"),
+                    ("Africa/Porto-Novo", "Africa/Porto-Novo"),
+                    ("Africa/Sao_Tome", "Africa/Sao_Tome"),
+                    ("Africa/Timbuktu", "Africa/Timbuktu"),
+                    ("Africa/Tripoli", "Africa/Tripoli"),
+                    ("Africa/Tunis", "Africa/Tunis"),
+                    ("Africa/Windhoek", "Africa/Windhoek"),
+                    ("America/Adak", "America/Adak"),
+                    ("America/Anchorage", "America/Anchorage"),
+                    ("America/Anguilla", "America/Anguilla"),
+                    ("America/Antigua", "America/Antigua"),
+                    ("America/Araguaina", "America/Araguaina"),
+                    (
+                        "America/Argentina/Buenos_Aires",
+                        "America/Argentina/Buenos_Aires",
+                    ),
+                    ("America/Argentina/Catamarca", "America/Argentina/Catamarca"),
+                    (
+                        "America/Argentina/ComodRivadavia",
+                        "America/Argentina/ComodRivadavia",
+                    ),
+                    ("America/Argentina/Cordoba", "America/Argentina/Cordoba"),
+                    ("America/Argentina/Jujuy", "America/Argentina/Jujuy"),
+                    ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"),
+                    ("America/Argentina/Mendoza", "America/Argentina/Mendoza"),
+                    (
+                        "America/Argentina/Rio_Gallegos",
+                        "America/Argentina/Rio_Gallegos",
+                    ),
+                    ("America/Argentina/Salta", "America/Argentina/Salta"),
+                    ("America/Argentina/San_Juan", "America/Argentina/San_Juan"),
+                    ("America/Argentina/San_Luis", "America/Argentina/San_Luis"),
+                    ("America/Argentina/Tucuman", "America/Argentina/Tucuman"),
+                    ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"),
+                    ("America/Aruba", "America/Aruba"),
+                    ("America/Asuncion", "America/Asuncion"),
+                    ("America/Atikokan", "America/Atikokan"),
+                    ("America/Atka", "America/Atka"),
+                    ("America/Bahia", "America/Bahia"),
+                    ("America/Bahia_Banderas", "America/Bahia_Banderas"),
+                    ("America/Barbados", "America/Barbados"),
+                    ("America/Belem", "America/Belem"),
+                    ("America/Belize", "America/Belize"),
+                    ("America/Blanc-Sablon", "America/Blanc-Sablon"),
+                    ("America/Boa_Vista", "America/Boa_Vista"),
+                    ("America/Bogota", "America/Bogota"),
+                    ("America/Boise", "America/Boise"),
+                    ("America/Buenos_Aires", "America/Buenos_Aires"),
+                    ("America/Cambridge_Bay", "America/Cambridge_Bay"),
+                    ("America/Campo_Grande", "America/Campo_Grande"),
+                    ("America/Cancun", "America/Cancun"),
+                    ("America/Caracas", "America/Caracas"),
+                    ("America/Catamarca", "America/Catamarca"),
+                    ("America/Cayenne", "America/Cayenne"),
+                    ("America/Cayman", "America/Cayman"),
+                    ("America/Chicago", "America/Chicago"),
+                    ("America/Chihuahua", "America/Chihuahua"),
+                    ("America/Ciudad_Juarez", "America/Ciudad_Juarez"),
+                    ("America/Coral_Harbour", "America/Coral_Harbour"),
+                    ("America/Cordoba", "America/Cordoba"),
+                    ("America/Costa_Rica", "America/Costa_Rica"),
+                    ("America/Creston", "America/Creston"),
+                    ("America/Cuiaba", "America/Cuiaba"),
+                    ("America/Curacao", "America/Curacao"),
+                    ("America/Danmarkshavn", "America/Danmarkshavn"),
+                    ("America/Dawson", "America/Dawson"),
+                    ("America/Dawson_Creek", "America/Dawson_Creek"),
+                    ("America/Denver", "America/Denver"),
+                    ("America/Detroit", "America/Detroit"),
+                    ("America/Dominica", "America/Dominica"),
+                    ("America/Edmonton", "America/Edmonton"),
+                    ("America/Eirunepe", "America/Eirunepe"),
+                    ("America/El_Salvador", "America/El_Salvador"),
+                    ("America/Ensenada", "America/Ensenada"),
+                    ("America/Fort_Nelson", "America/Fort_Nelson"),
+                    ("America/Fort_Wayne", "America/Fort_Wayne"),
+                    ("America/Fortaleza", "America/Fortaleza"),
+                    ("America/Glace_Bay", "America/Glace_Bay"),
+                    ("America/Godthab", "America/Godthab"),
+                    ("America/Goose_Bay", "America/Goose_Bay"),
+                    ("America/Grand_Turk", "America/Grand_Turk"),
+                    ("America/Grenada", "America/Grenada"),
+                    ("America/Guadeloupe", "America/Guadeloupe"),
+                    ("America/Guatemala", "America/Guatemala"),
+                    ("America/Guayaquil", "America/Guayaquil"),
+                    ("America/Guyana", "America/Guyana"),
+                    ("America/Halifax", "America/Halifax"),
+                    ("America/Havana", "America/Havana"),
+                    ("America/Hermosillo", "America/Hermosillo"),
+                    ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"),
+                    ("America/Indiana/Knox", "America/Indiana/Knox"),
+                    ("America/Indiana/Marengo", "America/Indiana/Marengo"),
+                    ("America/Indiana/Petersburg", "America/Indiana/Petersburg"),
+                    ("America/Indiana/Tell_City", "America/Indiana/Tell_City"),
+                    ("America/Indiana/Vevay", "America/Indiana/Vevay"),
+                    ("America/Indiana/Vincennes", "America/Indiana/Vincennes"),
+                    ("America/Indiana/Winamac", "America/Indiana/Winamac"),
+                    ("America/Indianapolis", "America/Indianapolis"),
+                    ("America/Inuvik", "America/Inuvik"),
+                    ("America/Iqaluit", "America/Iqaluit"),
+                    ("America/Jamaica", "America/Jamaica"),
+                    ("America/Jujuy", "America/Jujuy"),
+                    ("America/Juneau", "America/Juneau"),
+                    ("America/Kentucky/Louisville", "America/Kentucky/Louisville"),
+                    ("America/Kentucky/Monticello", "America/Kentucky/Monticello"),
+                    ("America/Knox_IN", "America/Knox_IN"),
+                    ("America/Kralendijk", "America/Kralendijk"),
+                    ("America/La_Paz", "America/La_Paz"),
+                    ("America/Lima", "America/Lima"),
+                    ("America/Los_Angeles", "America/Los_Angeles"),
+                    ("America/Louisville", "America/Louisville"),
+                    ("America/Lower_Princes", "America/Lower_Princes"),
+                    ("America/Maceio", "America/Maceio"),
+                    ("America/Managua", "America/Managua"),
+                    ("America/Manaus", "America/Manaus"),
+                    ("America/Marigot", "America/Marigot"),
+                    ("America/Martinique", "America/Martinique"),
+                    ("America/Matamoros", "America/Matamoros"),
+                    ("America/Mazatlan", "America/Mazatlan"),
+                    ("America/Mendoza", "America/Mendoza"),
+                    ("America/Menominee", "America/Menominee"),
+                    ("America/Merida", "America/Merida"),
+                    ("America/Metlakatla", "America/Metlakatla"),
+                    ("America/Mexico_City", "America/Mexico_City"),
+                    ("America/Miquelon", "America/Miquelon"),
+                    ("America/Moncton", "America/Moncton"),
+                    ("America/Monterrey", "America/Monterrey"),
+                    ("America/Montevideo", "America/Montevideo"),
+                    ("America/Montreal", "America/Montreal"),
+                    ("America/Montserrat", "America/Montserrat"),
+                    ("America/Nassau", "America/Nassau"),
+                    ("America/New_York", "America/New_York"),
+                    ("America/Nipigon", "America/Nipigon"),
+                    ("America/Nome", "America/Nome"),
+                    ("America/Noronha", "America/Noronha"),
+                    ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"),
+                    ("America/North_Dakota/Center", "America/North_Dakota/Center"),
+                    (
+                        "America/North_Dakota/New_Salem",
+                        "America/North_Dakota/New_Salem",
+                    ),
+                    ("America/Nuuk", "America/Nuuk"),
+                    ("America/Ojinaga", "America/Ojinaga"),
+                    ("America/Panama", "America/Panama"),
+                    ("America/Pangnirtung", "America/Pangnirtung"),
+                    ("America/Paramaribo", "America/Paramaribo"),
+                    ("America/Phoenix", "America/Phoenix"),
+                    ("America/Port-au-Prince", "America/Port-au-Prince"),
+                    ("America/Port_of_Spain", "America/Port_of_Spain"),
+                    ("America/Porto_Acre", "America/Porto_Acre"),
+                    ("America/Porto_Velho", "America/Porto_Velho"),
+                    ("America/Puerto_Rico", "America/Puerto_Rico"),
+                    ("America/Punta_Arenas", "America/Punta_Arenas"),
+                    ("America/Rainy_River", "America/Rainy_River"),
+                    ("America/Rankin_Inlet", "America/Rankin_Inlet"),
+                    ("America/Recife", "America/Recife"),
+                    ("America/Regina", "America/Regina"),
+                    ("America/Resolute", "America/Resolute"),
+                    ("America/Rio_Branco", "America/Rio_Branco"),
+                    ("America/Rosario", "America/Rosario"),
+                    ("America/Santa_Isabel", "America/Santa_Isabel"),
+                    ("America/Santarem", "America/Santarem"),
+                    ("America/Santiago", "America/Santiago"),
+                    ("America/Santo_Domingo", "America/Santo_Domingo"),
+                    ("America/Sao_Paulo", "America/Sao_Paulo"),
+                    ("America/Scoresbysund", "America/Scoresbysund"),
+                    ("America/Shiprock", "America/Shiprock"),
+                    ("America/Sitka", "America/Sitka"),
+                    ("America/St_Barthelemy", "America/St_Barthelemy"),
+                    ("America/St_Johns", "America/St_Johns"),
+                    ("America/St_Kitts", "America/St_Kitts"),
+                    ("America/St_Lucia", "America/St_Lucia"),
+                    ("America/St_Thomas", "America/St_Thomas"),
+                    ("America/St_Vincent", "America/St_Vincent"),
+                    ("America/Swift_Current", "America/Swift_Current"),
+                    ("America/Tegucigalpa", "America/Tegucigalpa"),
+                    ("America/Thule", "America/Thule"),
+                    ("America/Thunder_Bay", "America/Thunder_Bay"),
+                    ("America/Tijuana", "America/Tijuana"),
+                    ("America/Toronto", "America/Toronto"),
+                    ("America/Tortola", "America/Tortola"),
+                    ("America/Vancouver", "America/Vancouver"),
+                    ("America/Virgin", "America/Virgin"),
+                    ("America/Whitehorse", "America/Whitehorse"),
+                    ("America/Winnipeg", "America/Winnipeg"),
+                    ("America/Yakutat", "America/Yakutat"),
+                    ("America/Yellowknife", "America/Yellowknife"),
+                    ("Antarctica/Casey", "Antarctica/Casey"),
+                    ("Antarctica/Davis", "Antarctica/Davis"),
+                    ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"),
+                    ("Antarctica/Macquarie", "Antarctica/Macquarie"),
+                    ("Antarctica/Mawson", "Antarctica/Mawson"),
+                    ("Antarctica/McMurdo", "Antarctica/McMurdo"),
+                    ("Antarctica/Palmer", "Antarctica/Palmer"),
+                    ("Antarctica/Rothera", "Antarctica/Rothera"),
+                    ("Antarctica/South_Pole", "Antarctica/South_Pole"),
+                    ("Antarctica/Syowa", "Antarctica/Syowa"),
+                    ("Antarctica/Troll", "Antarctica/Troll"),
+                    ("Antarctica/Vostok", "Antarctica/Vostok"),
+                    ("Arctic/Longyearbyen", "Arctic/Longyearbyen"),
+                    ("Asia/Aden", "Asia/Aden"),
+                    ("Asia/Almaty", "Asia/Almaty"),
+                    ("Asia/Amman", "Asia/Amman"),
+                    ("Asia/Anadyr", "Asia/Anadyr"),
+                    ("Asia/Aqtau", "Asia/Aqtau"),
+                    ("Asia/Aqtobe", "Asia/Aqtobe"),
+                    ("Asia/Ashgabat", "Asia/Ashgabat"),
+                    ("Asia/Ashkhabad", "Asia/Ashkhabad"),
+                    ("Asia/Atyrau", "Asia/Atyrau"),
+                    ("Asia/Baghdad", "Asia/Baghdad"),
+                    ("Asia/Bahrain", "Asia/Bahrain"),
+                    ("Asia/Baku", "Asia/Baku"),
+                    ("Asia/Bangkok", "Asia/Bangkok"),
+                    ("Asia/Barnaul", "Asia/Barnaul"),
+                    ("Asia/Beirut", "Asia/Beirut"),
+                    ("Asia/Bishkek", "Asia/Bishkek"),
+                    ("Asia/Brunei", "Asia/Brunei"),
+                    ("Asia/Calcutta", "Asia/Calcutta"),
+                    ("Asia/Chita", "Asia/Chita"),
+                    ("Asia/Choibalsan", "Asia/Choibalsan"),
+                    ("Asia/Chongqing", "Asia/Chongqing"),
+                    ("Asia/Chungking", "Asia/Chungking"),
+                    ("Asia/Colombo", "Asia/Colombo"),
+                    ("Asia/Dacca", "Asia/Dacca"),
+                    ("Asia/Damascus", "Asia/Damascus"),
+                    ("Asia/Dhaka", "Asia/Dhaka"),
+                    ("Asia/Dili", "Asia/Dili"),
+                    ("Asia/Dubai", "Asia/Dubai"),
+                    ("Asia/Dushanbe", "Asia/Dushanbe"),
+                    ("Asia/Famagusta", "Asia/Famagusta"),
+                    ("Asia/Gaza", "Asia/Gaza"),
+                    ("Asia/Harbin", "Asia/Harbin"),
+                    ("Asia/Hebron", "Asia/Hebron"),
+                    ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"),
+                    ("Asia/Hong_Kong", "Asia/Hong_Kong"),
+                    ("Asia/Hovd", "Asia/Hovd"),
+                    ("Asia/Irkutsk", "Asia/Irkutsk"),
+                    ("Asia/Istanbul", "Asia/Istanbul"),
+                    ("Asia/Jakarta", "Asia/Jakarta"),
+                    ("Asia/Jayapura", "Asia/Jayapura"),
+                    ("Asia/Jerusalem", "Asia/Jerusalem"),
+                    ("Asia/Kabul", "Asia/Kabul"),
+                    ("Asia/Kamchatka", "Asia/Kamchatka"),
+                    ("Asia/Karachi", "Asia/Karachi"),
+                    ("Asia/Kashgar", "Asia/Kashgar"),
+                    ("Asia/Kathmandu", "Asia/Kathmandu"),
+                    ("Asia/Katmandu", "Asia/Katmandu"),
+                    ("Asia/Khandyga", "Asia/Khandyga"),
+                    ("Asia/Kolkata", "Asia/Kolkata"),
+                    ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"),
+                    ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"),
+                    ("Asia/Kuching", "Asia/Kuching"),
+                    ("Asia/Kuwait", "Asia/Kuwait"),
+                    ("Asia/Macao", "Asia/Macao"),
+                    ("Asia/Macau", "Asia/Macau"),
+                    ("Asia/Magadan", "Asia/Magadan"),
+                    ("Asia/Makassar", "Asia/Makassar"),
+                    ("Asia/Manila", "Asia/Manila"),
+                    ("Asia/Muscat", "Asia/Muscat"),
+                    ("Asia/Nicosia", "Asia/Nicosia"),
+                    ("Asia/Novokuznetsk", "Asia/Novokuznetsk"),
+                    ("Asia/Novosibirsk", "Asia/Novosibirsk"),
+                    ("Asia/Omsk", "Asia/Omsk"),
+                    ("Asia/Oral", "Asia/Oral"),
+                    ("Asia/Phnom_Penh", "Asia/Phnom_Penh"),
+                    ("Asia/Pontianak", "Asia/Pontianak"),
+                    ("Asia/Pyongyang", "Asia/Pyongyang"),
+                    ("Asia/Qatar", "Asia/Qatar"),
+                    ("Asia/Qostanay", "Asia/Qostanay"),
+                    ("Asia/Qyzylorda", "Asia/Qyzylorda"),
+                    ("Asia/Rangoon", "Asia/Rangoon"),
+                    ("Asia/Riyadh", "Asia/Riyadh"),
+                    ("Asia/Saigon", "Asia/Saigon"),
+                    ("Asia/Sakhalin", "Asia/Sakhalin"),
+                    ("Asia/Samarkand", "Asia/Samarkand"),
+                    ("Asia/Seoul", "Asia/Seoul"),
+                    ("Asia/Shanghai", "Asia/Shanghai"),
+                    ("Asia/Singapore", "Asia/Singapore"),
+                    ("Asia/Srednekolymsk", "Asia/Srednekolymsk"),
+                    ("Asia/Taipei", "Asia/Taipei"),
+                    ("Asia/Tashkent", "Asia/Tashkent"),
+                    ("Asia/Tbilisi", "Asia/Tbilisi"),
+                    ("Asia/Tehran", "Asia/Tehran"),
+                    ("Asia/Tel_Aviv", "Asia/Tel_Aviv"),
+                    ("Asia/Thimbu", "Asia/Thimbu"),
+                    ("Asia/Thimphu", "Asia/Thimphu"),
+                    ("Asia/Tokyo", "Asia/Tokyo"),
+                    ("Asia/Tomsk", "Asia/Tomsk"),
+                    ("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"),
+                    ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"),
+                    ("Asia/Ulan_Bator", "Asia/Ulan_Bator"),
+                    ("Asia/Urumqi", "Asia/Urumqi"),
+                    ("Asia/Ust-Nera", "Asia/Ust-Nera"),
+                    ("Asia/Vientiane", "Asia/Vientiane"),
+                    ("Asia/Vladivostok", "Asia/Vladivostok"),
+                    ("Asia/Yakutsk", "Asia/Yakutsk"),
+                    ("Asia/Yangon", "Asia/Yangon"),
+                    ("Asia/Yekaterinburg", "Asia/Yekaterinburg"),
+                    ("Asia/Yerevan", "Asia/Yerevan"),
+                    ("Atlantic/Azores", "Atlantic/Azores"),
+                    ("Atlantic/Bermuda", "Atlantic/Bermuda"),
+                    ("Atlantic/Canary", "Atlantic/Canary"),
+                    ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"),
+                    ("Atlantic/Faeroe", "Atlantic/Faeroe"),
+                    ("Atlantic/Faroe", "Atlantic/Faroe"),
+                    ("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"),
+                    ("Atlantic/Madeira", "Atlantic/Madeira"),
+                    ("Atlantic/Reykjavik", "Atlantic/Reykjavik"),
+                    ("Atlantic/South_Georgia", "Atlantic/South_Georgia"),
+                    ("Atlantic/St_Helena", "Atlantic/St_Helena"),
+                    ("Atlantic/Stanley", "Atlantic/Stanley"),
+                    ("Australia/ACT", "Australia/ACT"),
+                    ("Australia/Adelaide", "Australia/Adelaide"),
+                    ("Australia/Brisbane", "Australia/Brisbane"),
+                    ("Australia/Broken_Hill", "Australia/Broken_Hill"),
+                    ("Australia/Canberra", "Australia/Canberra"),
+                    ("Australia/Currie", "Australia/Currie"),
+                    ("Australia/Darwin", "Australia/Darwin"),
+                    ("Australia/Eucla", "Australia/Eucla"),
+                    ("Australia/Hobart", "Australia/Hobart"),
+                    ("Australia/LHI", "Australia/LHI"),
+                    ("Australia/Lindeman", "Australia/Lindeman"),
+                    ("Australia/Lord_Howe", "Australia/Lord_Howe"),
+                    ("Australia/Melbourne", "Australia/Melbourne"),
+                    ("Australia/NSW", "Australia/NSW"),
+                    ("Australia/North", "Australia/North"),
+                    ("Australia/Perth", "Australia/Perth"),
+                    ("Australia/Queensland", "Australia/Queensland"),
+                    ("Australia/South", "Australia/South"),
+                    ("Australia/Sydney", "Australia/Sydney"),
+                    ("Australia/Tasmania", "Australia/Tasmania"),
+                    ("Australia/Victoria", "Australia/Victoria"),
+                    ("Australia/West", "Australia/West"),
+                    ("Australia/Yancowinna", "Australia/Yancowinna"),
+                    ("Brazil/Acre", "Brazil/Acre"),
+                    ("Brazil/DeNoronha", "Brazil/DeNoronha"),
+                    ("Brazil/East", "Brazil/East"),
+                    ("Brazil/West", "Brazil/West"),
+                    ("CET", "CET"),
+                    ("CST6CDT", "CST6CDT"),
+                    ("Canada/Atlantic", "Canada/Atlantic"),
+                    ("Canada/Central", "Canada/Central"),
+                    ("Canada/Eastern", "Canada/Eastern"),
+                    ("Canada/Mountain", "Canada/Mountain"),
+                    ("Canada/Newfoundland", "Canada/Newfoundland"),
+                    ("Canada/Pacific", "Canada/Pacific"),
+                    ("Canada/Saskatchewan", "Canada/Saskatchewan"),
+                    ("Canada/Yukon", "Canada/Yukon"),
+                    ("Chile/Continental", "Chile/Continental"),
+                    ("Chile/EasterIsland", "Chile/EasterIsland"),
+                    ("Cuba", "Cuba"),
+                    ("EET", "EET"),
+                    ("EST", "EST"),
+                    ("EST5EDT", "EST5EDT"),
+                    ("Egypt", "Egypt"),
+                    ("Eire", "Eire"),
+                    ("Etc/GMT", "Etc/GMT"),
+                    ("Etc/GMT+0", "Etc/GMT+0"),
+                    ("Etc/GMT+1", "Etc/GMT+1"),
+                    ("Etc/GMT+10", "Etc/GMT+10"),
+                    ("Etc/GMT+11", "Etc/GMT+11"),
+                    ("Etc/GMT+12", "Etc/GMT+12"),
+                    ("Etc/GMT+2", "Etc/GMT+2"),
+                    ("Etc/GMT+3", "Etc/GMT+3"),
+                    ("Etc/GMT+4", "Etc/GMT+4"),
+                    ("Etc/GMT+5", "Etc/GMT+5"),
+                    ("Etc/GMT+6", "Etc/GMT+6"),
+                    ("Etc/GMT+7", "Etc/GMT+7"),
+                    ("Etc/GMT+8", "Etc/GMT+8"),
+                    ("Etc/GMT+9", "Etc/GMT+9"),
+                    ("Etc/GMT-0", "Etc/GMT-0"),
+                    ("Etc/GMT-1", "Etc/GMT-1"),
+                    ("Etc/GMT-10", "Etc/GMT-10"),
+                    ("Etc/GMT-11", "Etc/GMT-11"),
+                    ("Etc/GMT-12", "Etc/GMT-12"),
+                    ("Etc/GMT-13", "Etc/GMT-13"),
+                    ("Etc/GMT-14", "Etc/GMT-14"),
+                    ("Etc/GMT-2", "Etc/GMT-2"),
+                    ("Etc/GMT-3", "Etc/GMT-3"),
+                    ("Etc/GMT-4", "Etc/GMT-4"),
+                    ("Etc/GMT-5", "Etc/GMT-5"),
+                    ("Etc/GMT-6", "Etc/GMT-6"),
+                    ("Etc/GMT-7", "Etc/GMT-7"),
+                    ("Etc/GMT-8", "Etc/GMT-8"),
+                    ("Etc/GMT-9", "Etc/GMT-9"),
+                    ("Etc/GMT0", "Etc/GMT0"),
+                    ("Etc/Greenwich", "Etc/Greenwich"),
+                    ("Etc/UCT", "Etc/UCT"),
+                    ("Etc/UTC", "Etc/UTC"),
+                    ("Etc/Universal", "Etc/Universal"),
+                    ("Etc/Zulu", "Etc/Zulu"),
+                    ("Europe/Amsterdam", "Europe/Amsterdam"),
+                    ("Europe/Andorra", "Europe/Andorra"),
+                    ("Europe/Astrakhan", "Europe/Astrakhan"),
+                    ("Europe/Athens", "Europe/Athens"),
+                    ("Europe/Belfast", "Europe/Belfast"),
+                    ("Europe/Belgrade", "Europe/Belgrade"),
+                    ("Europe/Berlin", "Europe/Berlin"),
+                    ("Europe/Bratislava", "Europe/Bratislava"),
+                    ("Europe/Brussels", "Europe/Brussels"),
+                    ("Europe/Bucharest", "Europe/Bucharest"),
+                    ("Europe/Budapest", "Europe/Budapest"),
+                    ("Europe/Busingen", "Europe/Busingen"),
+                    ("Europe/Chisinau", "Europe/Chisinau"),
+                    ("Europe/Copenhagen", "Europe/Copenhagen"),
+                    ("Europe/Dublin", "Europe/Dublin"),
+                    ("Europe/Gibraltar", "Europe/Gibraltar"),
+                    ("Europe/Guernsey", "Europe/Guernsey"),
+                    ("Europe/Helsinki", "Europe/Helsinki"),
+                    ("Europe/Isle_of_Man", "Europe/Isle_of_Man"),
+                    ("Europe/Istanbul", "Europe/Istanbul"),
+                    ("Europe/Jersey", "Europe/Jersey"),
+                    ("Europe/Kaliningrad", "Europe/Kaliningrad"),
+                    ("Europe/Kiev", "Europe/Kiev"),
+                    ("Europe/Kirov", "Europe/Kirov"),
+                    ("Europe/Kyiv", "Europe/Kyiv"),
+                    ("Europe/Lisbon", "Europe/Lisbon"),
+                    ("Europe/Ljubljana", "Europe/Ljubljana"),
+                    ("Europe/London", "Europe/London"),
+                    ("Europe/Luxembourg", "Europe/Luxembourg"),
+                    ("Europe/Madrid", "Europe/Madrid"),
+                    ("Europe/Malta", "Europe/Malta"),
+                    ("Europe/Mariehamn", "Europe/Mariehamn"),
+                    ("Europe/Minsk", "Europe/Minsk"),
+                    ("Europe/Monaco", "Europe/Monaco"),
+                    ("Europe/Moscow", "Europe/Moscow"),
+                    ("Europe/Nicosia", "Europe/Nicosia"),
+                    ("Europe/Oslo", "Europe/Oslo"),
+                    ("Europe/Paris", "Europe/Paris"),
+                    ("Europe/Podgorica", "Europe/Podgorica"),
+                    ("Europe/Prague", "Europe/Prague"),
+                    ("Europe/Riga", "Europe/Riga"),
+                    ("Europe/Rome", "Europe/Rome"),
+                    ("Europe/Samara", "Europe/Samara"),
+                    ("Europe/San_Marino", "Europe/San_Marino"),
+                    ("Europe/Sarajevo", "Europe/Sarajevo"),
+                    ("Europe/Saratov", "Europe/Saratov"),
+                    ("Europe/Simferopol", "Europe/Simferopol"),
+                    ("Europe/Skopje", "Europe/Skopje"),
+                    ("Europe/Sofia", "Europe/Sofia"),
+                    ("Europe/Stockholm", "Europe/Stockholm"),
+                    ("Europe/Tallinn", "Europe/Tallinn"),
+                    ("Europe/Tirane", "Europe/Tirane"),
+                    ("Europe/Tiraspol", "Europe/Tiraspol"),
+                    ("Europe/Ulyanovsk", "Europe/Ulyanovsk"),
+                    ("Europe/Uzhgorod", "Europe/Uzhgorod"),
+                    ("Europe/Vaduz", "Europe/Vaduz"),
+                    ("Europe/Vatican", "Europe/Vatican"),
+                    ("Europe/Vienna", "Europe/Vienna"),
+                    ("Europe/Vilnius", "Europe/Vilnius"),
+                    ("Europe/Volgograd", "Europe/Volgograd"),
+                    ("Europe/Warsaw", "Europe/Warsaw"),
+                    ("Europe/Zagreb", "Europe/Zagreb"),
+                    ("Europe/Zaporozhye", "Europe/Zaporozhye"),
+                    ("Europe/Zurich", "Europe/Zurich"),
+                    ("GB", "GB"),
+                    ("GB-Eire", "GB-Eire"),
+                    ("GMT", "GMT"),
+                    ("GMT+0", "GMT+0"),
+                    ("GMT-0", "GMT-0"),
+                    ("GMT0", "GMT0"),
+                    ("Greenwich", "Greenwich"),
+                    ("HST", "HST"),
+                    ("Hongkong", "Hongkong"),
+                    ("Iceland", "Iceland"),
+                    ("Indian/Antananarivo", "Indian/Antananarivo"),
+                    ("Indian/Chagos", "Indian/Chagos"),
+                    ("Indian/Christmas", "Indian/Christmas"),
+                    ("Indian/Cocos", "Indian/Cocos"),
+                    ("Indian/Comoro", "Indian/Comoro"),
+                    ("Indian/Kerguelen", "Indian/Kerguelen"),
+                    ("Indian/Mahe", "Indian/Mahe"),
+                    ("Indian/Maldives", "Indian/Maldives"),
+                    ("Indian/Mauritius", "Indian/Mauritius"),
+                    ("Indian/Mayotte", "Indian/Mayotte"),
+                    ("Indian/Reunion", "Indian/Reunion"),
+                    ("Iran", "Iran"),
+                    ("Israel", "Israel"),
+                    ("Jamaica", "Jamaica"),
+                    ("Japan", "Japan"),
+                    ("Kwajalein", "Kwajalein"),
+                    ("Libya", "Libya"),
+                    ("MET", "MET"),
+                    ("MST", "MST"),
+                    ("MST7MDT", "MST7MDT"),
+                    ("Mexico/BajaNorte", "Mexico/BajaNorte"),
+                    ("Mexico/BajaSur", "Mexico/BajaSur"),
+                    ("Mexico/General", "Mexico/General"),
+                    ("NZ", "NZ"),
+                    ("NZ-CHAT", "NZ-CHAT"),
+                    ("Navajo", "Navajo"),
+                    ("PRC", "PRC"),
+                    ("PST8PDT", "PST8PDT"),
+                    ("Pacific/Apia", "Pacific/Apia"),
+                    ("Pacific/Auckland", "Pacific/Auckland"),
+                    ("Pacific/Bougainville", "Pacific/Bougainville"),
+                    ("Pacific/Chatham", "Pacific/Chatham"),
+                    ("Pacific/Chuuk", "Pacific/Chuuk"),
+                    ("Pacific/Easter", "Pacific/Easter"),
+                    ("Pacific/Efate", "Pacific/Efate"),
+                    ("Pacific/Enderbury", "Pacific/Enderbury"),
+                    ("Pacific/Fakaofo", "Pacific/Fakaofo"),
+                    ("Pacific/Fiji", "Pacific/Fiji"),
+                    ("Pacific/Funafuti", "Pacific/Funafuti"),
+                    ("Pacific/Galapagos", "Pacific/Galapagos"),
+                    ("Pacific/Gambier", "Pacific/Gambier"),
+                    ("Pacific/Guadalcanal", "Pacific/Guadalcanal"),
+                    ("Pacific/Guam", "Pacific/Guam"),
+                    ("Pacific/Honolulu", "Pacific/Honolulu"),
+                    ("Pacific/Johnston", "Pacific/Johnston"),
+                    ("Pacific/Kanton", "Pacific/Kanton"),
+                    ("Pacific/Kiritimati", "Pacific/Kiritimati"),
+                    ("Pacific/Kosrae", "Pacific/Kosrae"),
+                    ("Pacific/Kwajalein", "Pacific/Kwajalein"),
+                    ("Pacific/Majuro", "Pacific/Majuro"),
+                    ("Pacific/Marquesas", "Pacific/Marquesas"),
+                    ("Pacific/Midway", "Pacific/Midway"),
+                    ("Pacific/Nauru", "Pacific/Nauru"),
+                    ("Pacific/Niue", "Pacific/Niue"),
+                    ("Pacific/Norfolk", "Pacific/Norfolk"),
+                    ("Pacific/Noumea", "Pacific/Noumea"),
+                    ("Pacific/Pago_Pago", "Pacific/Pago_Pago"),
+                    ("Pacific/Palau", "Pacific/Palau"),
+                    ("Pacific/Pitcairn", "Pacific/Pitcairn"),
+                    ("Pacific/Pohnpei", "Pacific/Pohnpei"),
+                    ("Pacific/Ponape", "Pacific/Ponape"),
+                    ("Pacific/Port_Moresby", "Pacific/Port_Moresby"),
+                    ("Pacific/Rarotonga", "Pacific/Rarotonga"),
+                    ("Pacific/Saipan", "Pacific/Saipan"),
+                    ("Pacific/Samoa", "Pacific/Samoa"),
+                    ("Pacific/Tahiti", "Pacific/Tahiti"),
+                    ("Pacific/Tarawa", "Pacific/Tarawa"),
+                    ("Pacific/Tongatapu", "Pacific/Tongatapu"),
+                    ("Pacific/Truk", "Pacific/Truk"),
+                    ("Pacific/Wake", "Pacific/Wake"),
+                    ("Pacific/Wallis", "Pacific/Wallis"),
+                    ("Pacific/Yap", "Pacific/Yap"),
+                    ("Poland", "Poland"),
+                    ("Portugal", "Portugal"),
+                    ("ROC", "ROC"),
+                    ("ROK", "ROK"),
+                    ("Singapore", "Singapore"),
+                    ("Turkey", "Turkey"),
+                    ("UCT", "UCT"),
+                    ("US/Alaska", "US/Alaska"),
+                    ("US/Aleutian", "US/Aleutian"),
+                    ("US/Arizona", "US/Arizona"),
+                    ("US/Central", "US/Central"),
+                    ("US/East-Indiana", "US/East-Indiana"),
+                    ("US/Eastern", "US/Eastern"),
+                    ("US/Hawaii", "US/Hawaii"),
+                    ("US/Indiana-Starke", "US/Indiana-Starke"),
+                    ("US/Michigan", "US/Michigan"),
+                    ("US/Mountain", "US/Mountain"),
+                    ("US/Pacific", "US/Pacific"),
+                    ("US/Samoa", "US/Samoa"),
+                    ("UTC", "UTC"),
+                    ("Universal", "Universal"),
+                    ("W-SU", "W-SU"),
+                    ("WET", "WET"),
+                    ("Zulu", "Zulu"),
+                ],
+                default="UTC",
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0171_merge_20221219_2020.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2022-12-19 20:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0167_sitesettings_import_size_limit"),
+        ("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
+    ]
+
+    operations = []

bookwyrm/migrations/0172_alter_user_preferred_language.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.16 on 2022-12-21 18:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0171_alter_user_preferred_timezone"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0173_author_website.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.16 on 2023-01-15 08:38
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0172_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="website",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+    ]

bookwyrm/migrations/0173_default_user_auth_group_setting.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.16 on 2022-12-27 21:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def backfill_sitesettings(apps, schema_editor):
+    db_alias = schema_editor.connection.alias
+    group_model = apps.get_model("auth", "Group")
+    editor_group = group_model.objects.using(db_alias).filter(name="editor").first()
+
+    sitesettings_model = apps.get_model("bookwyrm", "SiteSettings")
+    sitesettings_model.objects.update(default_user_auth_group=editor_group)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0175_merge_0173_author_website_0174_merge_20230111_1523"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="default_user_auth_group",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.RESTRICT,
+                to="auth.group",
+            ),
+        ),
+        migrations.RunPython(backfill_sitesettings, migrations.RunPython.noop),
+    ]

bookwyrm/migrations/0173_merge_20230102_1444.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2023-01-02 14:44
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0171_merge_20221219_2020"),
+        ("bookwyrm", "0172_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0174_auto_20230130_1240.py

@@ -0,0 +1,35 @@
+# Generated by Django 3.2.16 on 2023-01-30 12:40
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
+        ("bookwyrm", "0173_default_user_auth_group_setting"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="quotation",
+            name="endposition",
+            field=models.IntegerField(
+                blank=True,
+                null=True,
+                validators=[django.core.validators.MinValueValidator(0)],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="sitesettings",
+            name="default_user_auth_group",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="auth.group",
+            ),
+        ),
+    ]

bookwyrm/migrations/0174_auto_20230222_1742.py

@@ -0,0 +1,46 @@
+# Generated by Django 3.2.18 on 2023-02-22 17:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0174_auto_20230130_1240"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="notification",
+            name="related_link_domains",
+            field=models.ManyToManyField(to="bookwyrm.LinkDomain"),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("BOOST", "Boost"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0174_merge_20230111_1523.py

@@ -0,0 +1,12 @@
+# Generated by Django 3.2.16 on 2023-01-11 15:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0173_merge_20230102_1444"),
+    ]
+
+    operations = []

bookwyrm/migrations/0175_merge_0173_author_website_0174_merge_20230111_1523.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2023-01-19 20:17
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0173_author_website"),
+        ("bookwyrm", "0174_merge_20230111_1523"),
+    ]
+
+    operations = []

bookwyrm/migrations/0176_hashtag_support.py

@@ -0,0 +1,53 @@
+# Generated by Django 3.2.16 on 2022-12-17 19:28
+
+import bookwyrm.models.fields
+import django.contrib.postgres.fields.citext
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0174_auto_20230130_1240"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Hashtag",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                ("updated_date", models.DateTimeField(auto_now=True)),
+                (
+                    "remote_id",
+                    bookwyrm.models.fields.RemoteIdField(
+                        max_length=255,
+                        null=True,
+                        validators=[bookwyrm.models.fields.validate_remote_id],
+                    ),
+                ),
+                (
+                    "name",
+                    django.contrib.postgres.fields.citext.CICharField(max_length=256),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="status",
+            name="mention_hashtags",
+            field=bookwyrm.models.fields.TagField(
+                related_name="mention_hashtag", to="bookwyrm.Hashtag"
+            ),
+        ),
+    ]

bookwyrm/migrations/0177_merge_0174_auto_20230222_1742_0176_hashtag_support.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.18 on 2023-03-12 23:41
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0174_auto_20230222_1742"),
+        ("bookwyrm", "0176_hashtag_support"),
+    ]
+
+    operations = []

bookwyrm/migrations/0178_auto_20230328_2132.py

@@ -0,0 +1,61 @@
+# Generated by Django 3.2.18 on 2023-03-28 21:32
+
+import bookwyrm.models.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
+        ("bookwyrm", "0177_merge_0174_auto_20230222_1742_0176_hashtag_support"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="hashtag",
+            name="name",
+            field=bookwyrm.models.fields.CICharField(max_length=256),
+        ),
+        migrations.AlterField(
+            model_name="sitesettings",
+            name="default_user_auth_group",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.RESTRICT,
+                to="auth.group",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("eo-uy", "Esperanto (Esperanto)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0179_populate_sort_title.py

@@ -0,0 +1,51 @@
+import re
+from itertools import chain
+
+from django.db import migrations, transaction
+from django.db.models import Q
+
+from bookwyrm.settings import LANGUAGE_ARTICLES
+
+
+def set_sort_title(edition):
+    articles = chain(
+        *(LANGUAGE_ARTICLES.get(language, ()) for language in tuple(edition.languages))
+    )
+    edition.sort_title = re.sub(
+        f'^{" |^".join(articles)} ', "", str(edition.title).lower()
+    )
+    return edition
+
+
+@transaction.atomic
+def populate_sort_title(apps, schema_editor):
+    Edition = apps.get_model("bookwyrm", "Edition")
+    db_alias = schema_editor.connection.alias
+    editions_wo_sort_title = Edition.objects.using(db_alias).filter(
+        Q(sort_title__isnull=True) | Q(sort_title__exact="")
+    )
+    batch_size = 1000
+    start = 0
+    end = batch_size
+    while True:
+        batch = editions_wo_sort_title[start:end]
+        if not batch.exists():
+            break
+        Edition.objects.bulk_update(
+            (set_sort_title(edition) for edition in batch), ["sort_title"]
+        )
+        start = end
+        end += batch_size
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0178_auto_20230328_2132"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            populate_sort_title, reverse_code=migrations.RunPython.noop
+        ),
+    ]

bookwyrm/migrations/0179_reportcomment_comment_type.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.2.18 on 2023-05-16 16:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0178_auto_20230328_2132"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="reportcomment",
+            name="action_type",
+            field=models.CharField(
+                choices=[
+                    ("comment", "Comment"),
+                    ("resolve", "Resolved report"),
+                    ("reopen", "Re-opened report"),
+                    ("message_reporter", "Messaged reporter"),
+                    ("message_offender", "Messaged reported user"),
+                    ("user_suspension", "Suspended user"),
+                    ("user_unsuspension", "Un-suspended user"),
+                    ("user_perms", "Changed user permission level"),
+                    ("user_deletion", "Deleted user account"),
+                    ("block_domain", "Blocked domain"),
+                    ("approve_domain", "Approved domain"),
+                    ("delete_item", "Deleted item"),
+                ],
+                default="comment",
+                max_length=20,
+            ),
+        ),
+        migrations.RenameModel("ReportComment", "ReportAction"),
+    ]

bookwyrm/migrations/0180_alter_reportaction_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.18 on 2023-06-21 22:01
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0179_reportcomment_comment_type"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="reportaction",
+            options={"ordering": ("created_date",)},
+        ),
+    ]

bookwyrm/migrations/0180_alter_user_preferred_language.py

@@ -0,0 +1,44 @@
+# Generated by Django 3.2.19 on 2023-07-23 19:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0179_populate_sort_title"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("eo-uy", "Esperanto (Esperanto)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("nl-nl", "Nederlands (Dutch)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0181_merge_20230806_2302.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.20 on 2023-08-06 23:02
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0180_alter_reportaction_options"),
+        ("bookwyrm", "0180_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0182_auto_20231027_1122.py

@@ -0,0 +1,130 @@
+# Generated by Django 3.2.20 on 2023-10-27 11:22
+
+import bookwyrm.models.activitypub_mixin
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0181_merge_20230806_2302"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="also_known_as",
+            field=bookwyrm.models.fields.ManyToManyField(to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name="user",
+            name="moved_to",
+            field=bookwyrm.models.fields.RemoteIdField(
+                max_length=255,
+                null=True,
+                validators=[bookwyrm.models.fields.validate_remote_id],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("BOOST", "Boost"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+        migrations.CreateModel(
+            name="Move",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                ("updated_date", models.DateTimeField(auto_now=True)),
+                (
+                    "remote_id",
+                    bookwyrm.models.fields.RemoteIdField(
+                        max_length=255,
+                        null=True,
+                        validators=[bookwyrm.models.fields.validate_remote_id],
+                    ),
+                ),
+                ("object", bookwyrm.models.fields.CharField(max_length=255)),
+                (
+                    "origin",
+                    bookwyrm.models.fields.CharField(
+                        blank=True, default="", max_length=255, null=True
+                    ),
+                ),
+                (
+                    "user",
+                    bookwyrm.models.fields.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=(bookwyrm.models.activitypub_mixin.ActivityMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name="MoveUser",
+            fields=[
+                (
+                    "move_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.move",
+                    ),
+                ),
+                (
+                    "target",
+                    bookwyrm.models.fields.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="move_target",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.move",),
+        ),
+    ]

bookwyrm/migrations/0183_auto_20231105_1607.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-11-05 16:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0182_auto_20231027_1122"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="is_deleted",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0184_auto_20231106_0421.py

@@ -0,0 +1,35 @@
+# Generated by Django 3.2.20 on 2023-11-06 04:21
+
+from django.db import migrations
+from bookwyrm.models import User
+
+
+def update_deleted_users(apps, schema_editor):
+    """Find all the users who are deleted, not just inactive, and set deleted"""
+    users = apps.get_model("bookwyrm", "User")
+    db_alias = schema_editor.connection.alias
+    users.objects.using(db_alias).filter(
+        is_active=False,
+        deactivation_reason__in=[
+            "self_deletion",
+            "moderator_deletion",
+        ],
+    ).update(is_deleted=True)
+
+    # differente rules for remote users
+    users.objects.using(db_alias).filter(is_active=False, local=False,).exclude(
+        deactivation_reason="moderator_deactivation",
+    ).update(is_deleted=True)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0183_auto_20231105_1607"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            update_deleted_users, reverse_code=migrations.RunPython.noop
+        ),
+    ]

bookwyrm/migrations/0185_alter_notification_notification_type.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.20 on 2023-11-13 22:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0184_auto_20231106_0421"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]