Merge pull request #3460 from hughrun/3426

fix failing test from #3432

.env.example

@@ -16,6 +16,11 @@ DEFAULT_LANGUAGE="English"
 ## Leave unset to allow all hosts
 # ALLOWED_HOSTS="localhost,127.0.0.1,[::1]"
 
+# Specify when the site is served from a port that is not the default
+# for the protocol (80 for HTTP or 443 for HTTPS).
+# Probably only necessary in development.
+# PORT=1333
+
 MEDIA_ROOT=images/
 
 # Database configuration
@@ -71,14 +76,20 @@ ENABLE_THUMBNAIL_GENERATION=true
 USE_S3=false
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
+# seconds for signed S3 urls to expire
+# this is currently only used for user export files
+S3_SIGNED_URL_EXPIRY=900
 
 # Commented are example values if you use a non-AWS, S3-compatible service
 # AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
 # non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
-# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL.
+# AWS_S3_URL_PROTOCOL must end in ":" and defaults to the same protocol as
+# the BookWyrm instance ("http:" or "https:", based on USE_SSL).
 
 # AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
 # AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_URL_PROTOCOL=None        # "http:"
 # AWS_S3_REGION_NAME=None         # "fr-par"
 # AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
 
@@ -133,7 +144,14 @@ HTTP_X_FORWARDED_PROTO=false
 TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2
 TWO_FACTOR_LOGIN_MAX_SECONDS=60
 
-# Additional hosts to allow in the Content-Security-Policy, "self" (should be DOMAIN)
-# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
-# Value should be a comma-separated list of host names.
+# Additional hosts to allow in the Content-Security-Policy, "self" (should be
+# DOMAIN with optionally ":" + PORT) and AWS_S3_CUSTOM_DOMAIN (if used) are
+# added by default.  Value should be a comma-separated list of host names.
 CSP_ADDITIONAL_HOSTS=
+
+# Time before being logged out (in seconds)
+# SESSION_COOKIE_AGE=2592000   # current default: 30 days
+
+# Maximum allowed memory for file uploads (increase if users are having trouble
+# uploading BookWyrm export files).
+# DATA_UPLOAD_MAX_MEMORY_MiB=100

.github/pull_request_template.md

@@ -0,0 +1,68 @@
+<!--
+Thanks for contributing! This template has some checkboxes that help keep track of what changes go into a release.
+
+To check (tick) a list item, replace the space between square brackets with an x, like this:
+
+- [x] I have checked the box
+
+You can find more information and tips for BookWyrm contributors at https://docs.joinbookwyrm.com/contributing.html
+-->
+## Description
+<!--
+Describe what your pull request does here
+-->
+
+
+<!--
+For pull requests that relate or close an issue, please include them
+below.  We like to follow [Github's guidance on linking issues to pull requests](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue).
+
+For example having the text: "closes #1234" would connect the current pull
+request to issue 1234.  And when we merge the pull request, Github will
+automatically close the issue.
+-->
+
+- Related Issue #
+- Closes #
+
+## What type of Pull Request is this?
+<!-- Check all that apply -->
+
+- [ ] Bug Fix
+- [ ] Enhancement
+- [ ] Plumbing / Internals / Dependencies
+- [ ] Refactor
+
+## Does this PR change settings or dependencies, or break something?
+<!-- Check all that apply -->
+
+- [ ] This PR changes or adds default settings, configuration, or .env values
+- [ ] This PR changes or adds dependencies
+- [ ] This PR introduces other breaking changes
+
+### Details of breaking or configuration changes (if any of above checked)
+
+
+## Documentation
+<!--
+Documentation for users, admins, and developers is an important way to keep the BookWyrm community welcoming and make Bookwyrm easy to use.
+Our documentation is maintained in a separate repository at https://github.com/bookwyrm-social/documentation
+-->
+
+<!-- Check all that apply -->
+
+- [ ] New or amended documentation will be required if this PR is merged
+- [ ] I have created a matching pull request in the Documentation repository
+- [ ] I intend to create a matching pull request in the Documentation repository after this PR is merged
+
+<!-- Amazing! Thanks for filling that out. Your PR will need to have passing tests and happy linters before we can merge
+You will need to check your code with `black`, `pylint`, and `mypy`, or `./bw-dev formatters`
+-->
+
+### Tests
+<!-- Check one -->
+
+- [ ] My changes do not need new tests
+- [ ] All tests I have added are passing
+- [ ] I have written tests but need help to make them pass
+- [ ] I have not written tests and need help to write them

.github/release.yml

@@ -0,0 +1,26 @@
+changelog:
+  exclude:
+    labels:
+      - ignore-for-release
+  categories:
+    - title: ‼️ Breaking Changes & New Settings ⚙️
+      labels:
+        - breaking-change
+        - config-change
+    - title: Updated Dependencies 🧸
+      labels:
+        - dependencies
+    - title: New Features 🎉
+      labels:
+        - enhancement
+    - title: Bug Fixes 🐛
+      labels:
+        - fix
+        - bug
+    - title: Internals/Plumbing 👩‍🔧
+        - plumbing
+        - tests
+        - deployment
+    - title: Other Changes
+      labels:
+        - "*"

.github/workflows/black.yml

@@ -1,17 +0,0 @@
-name: Python Formatting (run ./bw-dev black to fix)
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-      - uses: psf/black@22.12.0
-        with:
-          version: 22.12.0

.github/workflows/codeql-analysis.yml

@@ -36,11 +36,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,7 +51,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -65,4 +65,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/curlylint.yaml

@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install curlylint
         run: pip install curlylint

.github/workflows/django-tests.yml

@@ -1,61 +0,0 @@
-name: Run Python Tests
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-20.04
-    services:
-      postgres:
-        image: postgres:13
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: hunter2
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Run Tests
-      env:
-        SECRET_KEY: beepbeep
-        DEBUG: false
-        USE_HTTPS: true
-        DOMAIN: your.domain.here
-        BOOKWYRM_DATABASE_BACKEND: postgres
-        MEDIA_ROOT: images/
-        POSTGRES_PASSWORD: hunter2
-        POSTGRES_USER: postgres
-        POSTGRES_DB: github_actions
-        POSTGRES_HOST: 127.0.0.1
-        CELERY_BROKER: ""
-        REDIS_BROKER_PORT: 6379
-        REDIS_BROKER_PASSWORD: beep
-        USE_DUMMY_CACHE: true
-        FLOWER_PORT: 8888
-        EMAIL_HOST: "smtp.mailgun.org"
-        EMAIL_PORT: 587
-        EMAIL_HOST_USER: ""
-        EMAIL_HOST_PASSWORD: ""
-        EMAIL_USE_TLS: true
-        ENABLE_PREVIEW_IMAGES: false
-        ENABLE_THUMBNAIL_GENERATION: true
-        HTTP_X_FORWARDED_PROTO: false
-      run: |
-        pytest -n 3

.github/workflows/lint-frontend.yaml

@@ -19,10 +19,11 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install modules
-        run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
+      #  run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
+        run: npm install eslint@^8.9.0
 
       # See .stylelintignore for files that are not linted.
       # - name: Run stylelint

.github/workflows/prettier.yaml

@@ -14,10 +14,10 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install modules
-        run: npm install prettier
+        run: npm install prettier@2.5.1
 
       - name: Run Prettier
         run: npx prettier --check bookwyrm/static/js/*.js

.github/workflows/pylint.yml

@@ -1,27 +0,0 @@
-name: Pylint
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Analysing the code with pylint
-      run: |
-        pylint bookwyrm/
-

.github/workflows/python.yml

@@ -0,0 +1,99 @@
+name: Python
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+# overrides for .env.example
+env:
+  POSTGRES_HOST: 127.0.0.1
+  PGPORT: 5432
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: hunter2
+  POSTGRES_DB: github_actions
+  SECRET_KEY: beepbeep
+  EMAIL_HOST_USER: ""
+  EMAIL_HOST_PASSWORD: ""
+
+jobs:
+  pytest:
+    name: Tests (pytest)
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:13
+        env: # does not inherit from jobs.build.env
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: hunter2
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pytest-github-actions-annotate-failures
+    - name: Set up .env
+      run: cp .env.example .env
+    - name: Check migrations up-to-date
+      run: python ./manage.py makemigrations --check -v 3
+    - name: Run Tests
+      run: pytest -n 3
+
+  pylint:
+    name: Linting (pylint)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Analyse code with pylint
+      run: pylint bookwyrm/
+
+  mypy:
+    name: Typing (mypy)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Set up .env
+      run: cp .env.example .env
+    - name: Analyse code with mypy
+      run: mypy bookwyrm celerywyrm
+
+  black:
+    name: Formatting (black; run ./bw-dev black to fix)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+    - uses: psf/black@stable
+      with:
+        version: "22.*"

.gitignore

@@ -16,6 +16,8 @@
 # BookWyrm
 .env
 /images/
+/exports/
+/static/
 bookwyrm/static/css/bookwyrm.css
 bookwyrm/static/css/themes/
 !bookwyrm/static/css/themes/bookwyrm-*.scss
@@ -36,3 +38,6 @@ nginx/default.conf
 
 #macOS
 **/.DS_Store
+
+# Docker
+docker-compose.override.yml

.prettierrc

@@ -0,0 +1 @@
+'trailingComma': 'es5'

.pylintrc

@@ -3,7 +3,19 @@ ignore=migrations
 load-plugins=pylint.extensions.no_self_use
 
 [MESSAGES CONTROL]
-disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801,C3001,import-error
+disable =
+    cyclic-import,
+    duplicate-code,
+    fixme,
+    no-member,
+    raise-missing-from,
+    too-few-public-methods,
+    too-many-ancestors,
+    too-many-instance-attributes,
+    unnecessary-lambda-assignment,
+    unsubscriptable-object,
+enable =
+    useless-suppression
 
 [FORMAT]
 max-line-length=88

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.11
 
 ENV PYTHONUNBUFFERED 1
 

FEDERATION.md

@@ -0,0 +1,334 @@
+# Federation
+
+BookWyrm uses the [ActivityPub](http://activitypub.rocks/) protocol to send and receive user activity between other BookWyrm instances and other services that implement ActivityPub. To handle book data, BookWyrm has a handful of extended Activity types which are not part of the standard, but are legible to other BookWyrm instances.
+
+## Activities and Objects
+
+### Users and relationships
+User relationship interactions follow the standard ActivityPub spec.
+
+- `Follow`: request to receive statuses from a user, and view their statuses that have followers-only privacy
+- `Accept`: approves a `Follow` and finalizes the relationship
+- `Reject`: denies a `Follow`
+- `Block`: prevent users from seeing one another's statuses, and prevents the blocked user from viewing the actor's profile
+- `Update`: updates a user's profile and settings
+- `Delete`: deactivates a user
+- `Undo`: reverses a `Block` or `Follow`
+
+### Activities
+- `Create/Status`: saves a new status in the database.
+- `Delete/Status`: Removes a status
+- `Like/Status`: Creates a favorite on the status
+- `Announce/Status`: Boosts the status into the actor's timeline
+- `Undo/*`,: Reverses an `Announce`, `Like`, or `Move`
+- `Move/User`: Moves a user from one ActivityPub id to another.
+
+### Collections
+User's books and lists are represented by [`OrderedCollection`](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection)
+
+### Statuses
+
+BookWyrm is focused on book reading activities - it is not a general-purpose messaging application. For this reason, BookWyrm only accepts status `Create` activities if they are:
+
+- Direct messages (i.e., `Note`s with the privacy level `direct`, which mention a local user),
+- Related to a book (of a custom status type that includes the field `inReplyToBook`),
+- Replies to existing statuses saved in the database
+
+All other statuses will be received by the instance inbox, but by design **will not be delivered to user inboxes or displayed to users**.
+
+### Custom Object types
+
+With the exception of `Note`, the following object types are used in Bookwyrm but are not currently provided with a custom JSON-LD `@context` extension IRI. This is likely to change in future to make them true deserialisable JSON-LD objects.
+
+##### Note
+
+Within BookWyrm a `Note` is constructed according to [the ActivityStreams vocabulary](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-note), however `Note`s can only be created as direct messages or as replies to other statuses. As mentioned above, this also applies to incoming `Note`s.
+
+##### Review
+
+A `Review` is a status in response to a book (indicated by the `inReplyToBook` field), which has a title, body, and numerical rating between 0 (not rated) and 5.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/user/library_lurker/review/2",
+    "type": "Review",
+    "published": "2023-06-30T21:43:46.013132+00:00",
+    "attributedTo": "https://example.net/user/library_lurker",
+    "content": "<p>This is an enjoyable book with great characters.</p>",
+    "to": ["https://example.net/user/library_lurker/followers"],
+    "cc": [],
+    "replies": {
+        "id": "https://example.net/user/library_lurker/review/2/replies",
+        "type": "OrderedCollection",
+        "totalItems": 0,
+        "first": "https://example.net/user/library_lurker/review/2/replies?page=1",
+        "last": "https://example.net/user/library_lurker/review/2/replies?page=1",
+        "@context": "https://www.w3.org/ns/activitystreams"
+        },
+        "summary": "Spoilers ahead!",
+        "tag": [],
+        "attachment": [],
+        "sensitive": true,
+    "inReplyToBook": "https://example.net/book/1",
+    "name": "What a cracking read",
+    "rating": 4.5,
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Comment
+
+A `Comment` on a book mentions a book and has a message body, reading status, and progress indicator.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/user/library_lurker/comment/9",
+    "type": "Comment",
+    "published": "2023-06-30T21:43:46.013132+00:00",
+    "attributedTo": "https://example.net/user/library_lurker",
+    "content": "<p>This is a very enjoyable book so far.</p>",
+    "to": ["https://example.net/user/library_lurker/followers"],
+    "cc": [],
+    "replies": {
+        "id": "https://example.net/user/library_lurker/comment/9/replies",
+        "type": "OrderedCollection",
+        "totalItems": 0,
+        "first": "https://example.net/user/library_lurker/comment/9/replies?page=1",
+        "last": "https://example.net/user/library_lurker/comment/9/replies?page=1",
+        "@context": "https://www.w3.org/ns/activitystreams"
+        },
+        "summary": "Spoilers ahead!",
+        "tag": [],
+        "attachment": [],
+        "sensitive": true,
+    "inReplyToBook": "https://example.net/book/1",
+    "readingStatus": "reading",
+    "progress": 25,
+    "progressMode": "PG",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Quotation
+
+A quotation (aka "quote") has a message body, an excerpt from a book including position as a page number or percentage indicator, and mentions a book.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/user/mouse/quotation/13",
+    "url": "https://example.net/user/mouse/quotation/13",
+    "inReplyTo": null,
+    "published": "2020-05-10T02:38:31.150343+00:00",
+    "attributedTo": "https://example.net/user/mouse",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+        ],
+    "cc": [
+        "https://example.net/user/mouse/followers"
+        ],
+    "sensitive": false,
+    "content": "I really like this quote",
+    "type": "Quotation",
+    "replies": {
+        "id": "https://example.net/user/mouse/quotation/13/replies",
+        "type": "Collection",
+        "first": {
+            "type": "CollectionPage",
+            "next": "https://example.net/user/mouse/quotation/13/replies?only_other_accounts=true&page=true",
+            "partOf": "https://example.net/user/mouse/quotation/13/replies",
+            "items": []
+            }
+        },
+    "inReplyToBook": "https://example.net/book/1",
+    "quote": "To be or not to be, that is the question.",
+    "position": 50,
+    "positionMode": "PCT",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+### Custom Objects
+
+##### Work
+A particular book, a "work" in the [FRBR](https://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records) sense.
+
+Example:
+
+```json
+{
+    "id": "https://bookwyrm.social/book/5988",
+    "type": "Work",
+    "authors": [
+        "https://bookwyrm.social/author/417"
+    ],
+    "first_published_date": null,
+    "published_date": null,
+    "title": "Piranesi",
+    "sort_title": null,
+    "subtitle": null,
+    "description": "**From the *New York Times* bestselling author of *Jonathan Strange & Mr. Norrell*, an intoxicating, hypnotic new novel set in a dreamlike alternative reality.",
+    "languages": [],
+    "series": null,
+    "series_number": null,
+    "subjects": [
+        "English literature"
+    ],
+    "subject_places": [],
+    "openlibrary_key": "OL20893680W",
+    "librarything_key": null,
+    "goodreads_key": null,
+    "attachment": [
+        {
+            "url": "https://bookwyrm.social/images/covers/10226290-M.jpg",
+            "type": "Image"
+        }
+    ],
+    "lccn": null,
+    "editions": [
+        "https://bookwyrm.social/book/5989"
+    ],
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Edition
+A particular _manifestation_ of a Work, in the [FRBR](https://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records) sense.
+
+Example:
+
+```json
+{
+    "id": "https://bookwyrm.social/book/5989",
+    "lastEditedBy": "https://example.net/users/rat",
+    "type": "Edition",
+    "authors": [
+        "https://bookwyrm.social/author/417"
+    ],
+    "first_published_date": null,
+    "published_date": "2020-09-15T00:00:00+00:00",
+    "title": "Piranesi",
+    "sort_title": null,
+    "subtitle": null,
+    "description": "Piranesi's house is no ordinary building; its rooms are infinite, its corridors endless, its walls are lined with thousands upon thousands of statues, each one different from all the others.",
+    "languages": [
+        "English"
+    ],
+    "series": null,
+    "series_number": null,
+    "subjects": [],
+    "subject_places": [],
+    "openlibrary_key": "OL29486417M",
+    "librarything_key": null,
+    "goodreads_key": null,
+    "isfdb": null,
+    "attachment": [
+        {
+            "url": "https://bookwyrm.social/images/covers/50202953._SX318_.jpg",
+            "type": "Image"
+        }
+    ],
+    "isbn_10": "1526622424",
+    "isbn_13": "9781526622426",
+    "oclc_number": null,
+    "asin": null,
+    "pages": 272,
+    "physical_format": null,
+    "publishers": [
+        "Bloomsbury Publishing Plc"
+    ],
+    "work": "https://bookwyrm.social/book/5988",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### Shelf
+
+A user's book collection. By default, every user has a `to-read`, `reading`, `read`, and `stopped-reading` shelf which are used to track reading progress. Users may create an unlimited number of additional shelves with their own ids.
+
+Example
+
+```json
+{
+    "id": "https://example.net/user/avid_reader/books/extraspecialbooks-5",
+    "type": "Shelf",
+    "totalItems": 0,
+    "first": "https://example.net/user/avid_reader/books/extraspecialbooks-5?page=1",
+    "last": "https://example.net/user/avid_reader/books/extraspecialbooks-5?page=1",
+    "name": "Extra special books",
+    "owner": "https://example.net/user/avid_reader",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "cc": [
+        "https://example.net/user/avid_reader/followers"
+    ],
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### List
+
+A collection of books that may have items contributed by users other than the one who created the list.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/list/1",
+    "type": "BookList",
+    "totalItems": 0,
+    "first": "https://example.net/list/1?page=1",
+    "last": "https://example.net/list/1?page=1",
+    "name": "My cool list",
+    "owner": "https://example.net/user/avid_reader",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "cc": [
+        "https://example.net/user/avid_reader/followers"
+    ],
+    "summary": "A list of books I like.",
+    "curation": "curated",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### Activities
+
+- `Create`: Adds a shelf or list to the database.
+- `Delete`: Removes a shelf or list.
+- `Add`: Adds a book to a shelf or list.
+- `Remove`: Removes a book from a shelf or list.
+
+## Alternative Serialization
+Because BookWyrm uses custom object types that aren't listed in [the standard ActivityStreams Vocabulary](https://www.w3.org/TR/activitystreams-vocabulary), some statuses are transformed into standard types when sent to or viewed by non-BookWyrm services. `Review`s are converted into `Article`s, and `Comment`s and `Quotation`s are converted into `Note`s, with a link to the book and the cover image attached.
+
+In future this may be done with [JSON-LD type arrays](https://www.w3.org/TR/json-ld/#specifying-the-type) instead.
+
+## Other extensions
+
+### Webfinger
+
+Bookwyrm uses the [Webfinger](https://datatracker.ietf.org/doc/html/rfc7033) standard to identify and disambiguate fediverse actors. The [Webfinger documentation on the Mastodon project](https://docs.joinmastodon.org/spec/webfinger/) provides a good overview of how Webfinger is used.
+
+### HTTP Signatures
+
+Bookwyrm uses and requires HTTP signatures for all `POST` requests. `GET` requests are not signed by default, but if Bookwyrm receives a `403` response to a `GET` it will re-send the request, signed by the default server user. This usually will have a user id of `https://example.net/user/bookwyrm.instance.actor`
+
+#### publicKey id
+
+In older versions of Bookwyrm the `publicKey.id` was incorrectly listed in request headers as `https://example.net/user/username#main-key`. As of v0.6.3 the id is now listed correctly, as `https://example.net/user/username/#main-key`. In most ActivityPub implementations this will make no difference as the URL will usually resolve to the same place.
+
+### NodeInfo
+
+Bookwyrm uses the [NodeInfo](http://nodeinfo.diaspora.software/) standard to provide statistics and version information for each instance.
+
+## Further Documentation
+
+See [docs.joinbookwyrm.com/](https://docs.joinbookwyrm.com/) for more documentation.

README.md

@@ -10,7 +10,6 @@ BookWyrm is a social network for tracking your reading, talking about books, wri
 ## Links
 
 [![Mastodon Follow](https://img.shields.io/mastodon/follow/000146121?domain=https%3A%2F%2Ftech.lgbt&style=social)](https://tech.lgbt/@bookwyrm)
-[![Twitter Follow](https://img.shields.io/twitter/follow/BookWyrmSocial?style=social)](https://twitter.com/BookWyrmSocial)
 
  - [Project homepage](https://joinbookwyrm.com/)
  - [Support](https://patreon.com/bookwyrm)

VERSION

@@ -0,0 +1 @@
+0.7.3

bookwyrm/activitypub/__init__.py

@@ -4,7 +4,11 @@ import sys
 
 from .base_activity import ActivityEncoder, Signature, naive_parse
 from .base_activity import Link, Mention, Hashtag
-from .base_activity import ActivitySerializerError, resolve_remote_id
+from .base_activity import (
+    ActivitySerializerError,
+    resolve_remote_id,
+    get_representative,
+)
 from .image import Document, Image
 from .note import Note, GeneratedNote, Article, Comment, Quotation
 from .note import Review, Rating
@@ -19,6 +23,7 @@ from .verbs import Create, Delete, Undo, Update
 from .verbs import Follow, Accept, Reject, Block
 from .verbs import Add, Remove
 from .verbs import Announce, Like
+from .verbs import Move
 
 # this creates a list of all the Activity types that we can serialize,
 # so when an Activity comes in from outside, we can check if it's known

bookwyrm/activitypub/base_activity.py

@@ -1,7 +1,10 @@
 """ basics for an activitypub serializer """
+from __future__ import annotations
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
 import logging
+from typing import Optional, Union, TypeVar, overload, Any
+
 import requests
 
 from django.apps import apps
@@ -10,12 +13,16 @@ from django.utils.http import http_date
 
 from bookwyrm import models
 from bookwyrm.connectors import ConnectorException, get_data
+from bookwyrm.models import base_model
 from bookwyrm.signatures import make_signature
 from bookwyrm.settings import DOMAIN, INSTANCE_ACTOR_USERNAME
-from bookwyrm.tasks import app, MEDIUM
+from bookwyrm.tasks import app, MISC
 
 logger = logging.getLogger(__name__)
 
+# pylint: disable=invalid-name
+TBookWyrmModel = TypeVar("TBookWyrmModel", bound=base_model.BookWyrmModel)
+
 
 class ActivitySerializerError(ValueError):
     """routine problems serializing activitypub json"""
@@ -65,7 +72,13 @@ class ActivityObject:
     id: str
     type: str
 
-    def __init__(self, activity_objects=None, **kwargs):
+    def __init__(
+        self,
+        activity_objects: Optional[
+            dict[str, Union[str, list[str], ActivityObject, base_model.BookWyrmModel]]
+        ] = None,
+        **kwargs: Any,
+    ):
         """this lets you pass in an object with fields that aren't in the
         dataclass, which it ignores. Any field in the dataclass is required or
         has a default value"""
@@ -101,13 +114,13 @@ class ActivityObject:
     # pylint: disable=too-many-locals,too-many-branches,too-many-arguments
     def to_model(
         self,
-        model=None,
-        instance=None,
-        allow_create=True,
-        save=True,
-        overwrite=True,
-        allow_external_connections=True,
-    ):
+        model: Optional[type[TBookWyrmModel]] = None,
+        instance: Optional[TBookWyrmModel] = None,
+        allow_create: bool = True,
+        save: bool = True,
+        overwrite: bool = True,
+        allow_external_connections: bool = True,
+    ) -> Optional[TBookWyrmModel]:
         """convert from an activity to a model instance. Args:
         model: the django model that this object is being converted to
             (will guess if not known)
@@ -224,7 +237,7 @@ class ActivityObject:
         omit = kwargs.get("omit", ())
         data = self.__dict__.copy()
         # recursively serialize
-        for (k, v) in data.items():
+        for k, v in data.items():
             try:
                 if issubclass(type(v), ActivityObject):
                     data[k] = v.serialize()
@@ -237,11 +250,14 @@ class ActivityObject:
                 pass
         data = {k: v for (k, v) in data.items() if v is not None and k not in omit}
         if "@context" not in omit:
-            data["@context"] = "https://www.w3.org/ns/activitystreams"
+            data["@context"] = [
+                "https://www.w3.org/ns/activitystreams",
+                {"Hashtag": "as:Hashtag"},
+            ]
         return data
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MISC)
 @transaction.atomic
 def set_related_field(
     model_name, origin_model_name, related_field_name, related_remote_id, data
@@ -296,14 +312,40 @@ def get_model_from_type(activity_type):
 
 
 # pylint: disable=too-many-arguments
+@overload
 def resolve_remote_id(
-    remote_id,
-    model=None,
-    refresh=False,
-    save=True,
-    get_activity=False,
-    allow_external_connections=True,
-):
+    remote_id: str,
+    model: type[TBookWyrmModel],
+    refresh: bool = False,
+    save: bool = True,
+    get_activity: bool = False,
+    allow_external_connections: bool = True,
+) -> TBookWyrmModel:
+    ...
+
+
+# pylint: disable=too-many-arguments
+@overload
+def resolve_remote_id(
+    remote_id: str,
+    model: Optional[str] = None,
+    refresh: bool = False,
+    save: bool = True,
+    get_activity: bool = False,
+    allow_external_connections: bool = True,
+) -> base_model.BookWyrmModel:
+    ...
+
+
+# pylint: disable=too-many-arguments
+def resolve_remote_id(
+    remote_id: str,
+    model: Optional[Union[str, type[base_model.BookWyrmModel]]] = None,
+    refresh: bool = False,
+    save: bool = True,
+    get_activity: bool = False,
+    allow_external_connections: bool = True,
+) -> base_model.BookWyrmModel:
     """take a remote_id and return an instance, creating if necessary. Args:
     remote_id: the unique url for looking up the object in the db or by http
     model: a string or object representing the model that corresponds to the object
@@ -327,16 +369,12 @@ def resolve_remote_id(
 
     # load the data and create the object
     try:
-        data = get_data(remote_id)
+        data = get_activitypub_data(remote_id)
     except ConnectionError:
         logger.info("Could not connect to host for remote_id: %s", remote_id)
         return None
     except requests.HTTPError as e:
-        if (e.response is not None) and e.response.status_code == 401:
-            # This most likely means it's a mastodon with secure fetch enabled.
-            data = get_activitypub_data(remote_id)
-        else:
-            logger.info("Could not connect to host for remote_id: %s", remote_id)
+        logger.exception("HTTP error - remote_id: %s - error: %s", remote_id, e)
         return None
     # determine the model implicitly, if not provided
     # or if it's a model with subclasses like Status, check again
@@ -358,19 +396,15 @@ def resolve_remote_id(
 
 def get_representative():
     """Get or create an actor representing the instance
-    to sign requests to 'secure mastodon' servers"""
-    username = f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}"
-    email = "bookwyrm@localhost"
-    try:
-        user = models.User.objects.get(username=username)
-    except models.User.DoesNotExist:
-        user = models.User.objects.create_user(
-            username=username,
-            email=email,
-            local=True,
-            localname=INSTANCE_ACTOR_USERNAME,
-        )
-    return user
+    to sign outgoing HTTP GET requests"""
+    return models.User.objects.get_or_create(
+        username=f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}",
+        defaults={
+            "email": "bookwyrm@localhost",
+            "local": True,
+            "localname": INSTANCE_ACTOR_USERNAME,
+        },
+    )[0]
 
 
 def get_activitypub_data(url):
@@ -389,6 +423,7 @@ def get_activitypub_data(url):
                 "Date": now,
                 "Signature": make_signature("get", sender, url, now),
             },
+            timeout=15,
         )
     except requests.RequestException:
         raise ConnectorException()

bookwyrm/activitypub/book.py

@@ -1,6 +1,6 @@
 """ book and author data """
 from dataclasses import dataclass, field
-from typing import List
+from typing import Optional
 
 from .base_activity import ActivityObject
 from .image import Document
@@ -11,19 +11,17 @@ from .image import Document
 class BookData(ActivityObject):
     """shared fields for all book data and authors"""
 
-    openlibraryKey: str = None
-    inventaireId: str = None
-    librarythingKey: str = None
-    goodreadsKey: str = None
-    bnfId: str = None
-    viaf: str = None
-    wikidata: str = None
-    asin: str = None
-    aasin: str = None
-    isfdb: str = None
-    lastEditedBy: str = None
-    links: List[str] = field(default_factory=lambda: [])
-    fileLinks: List[str] = field(default_factory=lambda: [])
+    openlibraryKey: Optional[str] = None
+    inventaireId: Optional[str] = None
+    librarythingKey: Optional[str] = None
+    goodreadsKey: Optional[str] = None
+    bnfId: Optional[str] = None
+    viaf: Optional[str] = None
+    wikidata: Optional[str] = None
+    asin: Optional[str] = None
+    aasin: Optional[str] = None
+    isfdb: Optional[str] = None
+    lastEditedBy: Optional[str] = None
 
 
 # pylint: disable=invalid-name
@@ -35,17 +33,19 @@ class Book(BookData):
     sortTitle: str = None
     subtitle: str = None
     description: str = ""
-    languages: List[str] = field(default_factory=lambda: [])
+    languages: list[str] = field(default_factory=list)
     series: str = ""
     seriesNumber: str = ""
-    subjects: List[str] = field(default_factory=lambda: [])
-    subjectPlaces: List[str] = field(default_factory=lambda: [])
+    subjects: list[str] = field(default_factory=list)
+    subjectPlaces: list[str] = field(default_factory=list)
 
-    authors: List[str] = field(default_factory=lambda: [])
+    authors: list[str] = field(default_factory=list)
     firstPublishedDate: str = ""
     publishedDate: str = ""
 
-    cover: Document = None
+    fileLinks: list[str] = field(default_factory=list)
+
+    cover: Optional[Document] = None
     type: str = "Book"
 
 
@@ -58,22 +58,21 @@ class Edition(Book):
     isbn10: str = ""
     isbn13: str = ""
     oclcNumber: str = ""
-    pages: int = None
+    pages: Optional[int] = None
     physicalFormat: str = ""
     physicalFormatDetail: str = ""
-    publishers: List[str] = field(default_factory=lambda: [])
+    publishers: list[str] = field(default_factory=list)
     editionRank: int = 0
 
     type: str = "Edition"
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Work(Book):
     """work instance of a book object"""
 
     lccn: str = ""
-    editions: List[str] = field(default_factory=lambda: [])
+    editions: list[str] = field(default_factory=list)
     type: str = "Work"
 
 
@@ -83,12 +82,12 @@ class Author(BookData):
     """author of a book"""
 
     name: str
-    isni: str = None
-    viafId: str = None
-    gutenbergId: str = None
-    born: str = None
-    died: str = None
-    aliases: List[str] = field(default_factory=lambda: [])
+    isni: Optional[str] = None
+    viafId: Optional[str] = None
+    gutenbergId: Optional[str] = None
+    born: Optional[str] = None
+    died: Optional[str] = None
+    aliases: list[str] = field(default_factory=list)
     bio: str = ""
     wikipediaLink: str = ""
     type: str = "Author"

bookwyrm/activitypub/ordered_collection.py

@@ -18,7 +18,6 @@ class OrderedCollection(ActivityObject):
     type: str = "OrderedCollection"
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class OrderedCollectionPrivate(OrderedCollection):
     """an ordered collection with privacy settings"""

bookwyrm/activitypub/person.py

@@ -1,5 +1,5 @@
 """ actor serializer """
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import Dict
 
 from .base_activity import ActivityObject
@@ -35,9 +35,11 @@ class Person(ActivityObject):
     endpoints: Dict = None
     name: str = None
     summary: str = None
-    icon: Image = field(default_factory=lambda: {})
+    icon: Image = None
     bookwyrmUser: bool = False
     manuallyApprovesFollowers: str = False
     discoverable: str = False
     hideFollows: str = False
+    movedTo: str = None
+    alsoKnownAs: dict[str] = None
     type: str = "Person"

bookwyrm/activitypub/verbs.py

@@ -22,7 +22,6 @@ class Verb(ActivityObject):
             self.object.to_model(allow_external_connections=allow_external_connections)
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Create(Verb):
     """Create activity"""
@@ -33,7 +32,6 @@ class Create(Verb):
     type: str = "Create"
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Delete(Verb):
     """Create activity"""
@@ -63,7 +61,6 @@ class Delete(Verb):
         # if we can't find it, we don't need to delete it because we don't have it
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Update(Verb):
     """Update activity"""
@@ -171,9 +168,19 @@ class Reject(Verb):
     type: str = "Reject"
 
     def action(self, allow_external_connections=True):
-        """reject a follow request"""
-        obj = self.object.to_model(save=False, allow_create=False)
+        """reject a follow or follow request"""
+
+        for model_name in ["UserFollowRequest", "UserFollows", None]:
+            model = apps.get_model(f"bookwyrm.{model_name}") if model_name else None
+            if obj := self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            ):
+                # Reject the first model that can be built.
                 obj.reject()
+                break
 
 
 @dataclass(init=False)
@@ -217,7 +224,6 @@ class Like(Verb):
         self.to_model(allow_external_connections=allow_external_connections)
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Announce(Verb):
     """boosting a status"""
@@ -231,3 +237,30 @@ class Announce(Verb):
     def action(self, allow_external_connections=True):
         """boost"""
         self.to_model(allow_external_connections=allow_external_connections)
+
+
+@dataclass(init=False)
+class Move(Verb):
+    """a user moving an object"""
+
+    object: str
+    type: str = "Move"
+    origin: str = None
+    target: str = None
+
+    def action(self, allow_external_connections=True):
+        """move"""
+
+        object_is_user = resolve_remote_id(remote_id=self.object, model="User")
+
+        if object_is_user:
+            model = apps.get_model("bookwyrm.MoveUser")
+
+            self.to_model(
+                model=model,
+                save=True,
+                allow_external_connections=allow_external_connections,
+            )
+        else:
+            # we might do something with this to move other objects at some point
+            pass

bookwyrm/activitystreams.py

@@ -8,7 +8,7 @@ from opentelemetry import trace
 
 from bookwyrm import models
 from bookwyrm.redis_store import RedisStore, r
-from bookwyrm.tasks import app, LOW, MEDIUM, HIGH
+from bookwyrm.tasks import app, STREAMS, IMPORT_TRIGGERED
 from bookwyrm.telemetry import open_telemetry
 
 
@@ -32,7 +32,7 @@ class ActivityStream(RedisStore):
         stream_id = self.stream_id(user_id)
         return f"{stream_id}-unread-by-type"
 
-    def get_rank(self, obj):  # pylint: disable=no-self-use
+    def get_rank(self, obj):
         """statuses are sorted by date published"""
         return obj.published_date.timestamp()
 
@@ -112,7 +112,7 @@ class ActivityStream(RedisStore):
         trace.get_current_span().set_attribute("status_privacy", status.privacy)
         trace.get_current_span().set_attribute(
             "status_reply_parent_privacy",
-            status.reply_parent.privacy if status.reply_parent else None,
+            status.reply_parent.privacy if status.reply_parent else status.privacy,
         )
         # direct messages don't appear in feeds, direct comments/reviews/etc do
         if status.privacy == "direct" and status.status_type == "Note":
@@ -139,14 +139,14 @@ class ActivityStream(RedisStore):
                 | (
                     Q(following=status.user) & Q(following=status.reply_parent.user)
                 )  # if the user is following both authors
-            ).distinct()
+            )
 
         # only visible to the poster's followers and tagged users
         elif status.privacy == "followers":
             audience = audience.filter(
                 Q(following=status.user)  # if the user is following the author
             )
-        return audience.distinct()
+        return audience.distinct("id")
 
     @tracer.start_as_current_span("ActivityStream.get_audience")
     def get_audience(self, status):
@@ -156,7 +156,7 @@ class ActivityStream(RedisStore):
         status_author = models.User.objects.filter(
             is_active=True, local=True, id=status.user.id
         ).values_list("id", flat=True)
-        return list(set(list(audience) + list(status_author)))
+        return list(set(audience) | set(status_author))
 
     def get_stores_for_users(self, user_ids):
         """convert a list of user ids into redis store ids"""
@@ -183,15 +183,13 @@ class HomeStream(ActivityStream):
     def get_audience(self, status):
         trace.get_current_span().set_attribute("stream_id", self.key)
         audience = super()._get_audience(status)
-        if not audience:
-            return []
         # if the user is following the author
         audience = audience.filter(following=status.user).values_list("id", flat=True)
         # if the user is the post's author
         status_author = models.User.objects.filter(
             is_active=True, local=True, id=status.user.id
         ).values_list("id", flat=True)
-        return list(set(list(audience) + list(status_author)))
+        return list(set(audience) | set(status_author))
 
     def get_statuses_for_user(self, user):
         return models.Status.privacy_filter(
@@ -239,9 +237,7 @@ class BooksStream(ActivityStream):
         )
 
         audience = super()._get_audience(status)
-        if not audience:
-            return models.User.objects.none()
-        return audience.filter(shelfbook__book__parent_work=work).distinct()
+        return audience.filter(shelfbook__book__parent_work=work)
 
     def get_audience(self, status):
         # only show public statuses on the books feed,
@@ -329,10 +325,9 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
         remove_status_task.delay(instance.id)
         return
 
-    # To avoid creating a zillion unnecessary tasks caused by re-saving the model,
-    # check if it's actually ready to send before we go. We're trusting this was
-    # set correctly by the inbox or view
-    if not instance.ready:
+    # We don't want to create multiple add_status_tasks for each status, and because
+    # the transactions are atomic, on_commit won't run until the status is ready to add.
+    if not created:
         return
 
     # when creating new things, gotta wait on the transaction
@@ -343,7 +338,11 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
 
 def add_status_on_create_command(sender, instance, created):
     """runs this code only after the database commit completes"""
-    priority = HIGH
+    # boosts trigger 'saves" twice, so don't bother duplicating the task
+    if sender == models.Boost and not created:
+        return
+
+    priority = STREAMS
     # check if this is an old status, de-prioritize if so
     # (this will happen if federation is very slow, or, more expectedly, on csv import)
     if instance.published_date < timezone.now() - timedelta(
@@ -353,7 +352,7 @@ def add_status_on_create_command(sender, instance, created):
         if instance.user.local:
             return
         # an out of date remote status is a low priority but should be added
-        priority = LOW
+        priority = IMPORT_TRIGGERED
 
     add_status_task.apply_async(
         args=(instance.id,),
@@ -497,7 +496,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs):
 # ---- TASKS
 
 
-@app.task(queue=LOW)
+@app.task(queue=STREAMS)
 def add_book_statuses_task(user_id, book_id):
     """add statuses related to a book on shelve"""
     user = models.User.objects.get(id=user_id)
@@ -505,7 +504,7 @@ def add_book_statuses_task(user_id, book_id):
     BooksStream().add_book_statuses(user, book)
 
 
-@app.task(queue=LOW)
+@app.task(queue=STREAMS)
 def remove_book_statuses_task(user_id, book_id):
     """remove statuses about a book from a user's books feed"""
     user = models.User.objects.get(id=user_id)
@@ -513,7 +512,7 @@ def remove_book_statuses_task(user_id, book_id):
     BooksStream().remove_book_statuses(user, book)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def populate_stream_task(stream, user_id):
     """background task for populating an empty activitystream"""
     user = models.User.objects.get(id=user_id)
@@ -521,7 +520,7 @@ def populate_stream_task(stream, user_id):
     stream.populate_streams(user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def remove_status_task(status_ids):
     """remove a status from any stream it might be in"""
     # this can take an id or a list of ids
@@ -536,7 +535,7 @@ def remove_status_task(status_ids):
             )
 
 
-@app.task(queue=HIGH)
+@app.task(queue=STREAMS)
 def add_status_task(status_id, increment_unread=False):
     """add a status to any stream it should be in"""
     status = models.Status.objects.select_subclasses().get(id=status_id)
@@ -548,7 +547,7 @@ def add_status_task(status_id, increment_unread=False):
         stream.add_status(status, increment_unread=increment_unread)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
     """remove all statuses by a user from a viewer's stream"""
     stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -558,7 +557,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
         stream.remove_user_statuses(viewer, user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def add_user_statuses_task(viewer_id, user_id, stream_list=None):
     """add all statuses by a user to a viewer's stream"""
     stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -568,7 +567,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None):
         stream.add_user_statuses(viewer, user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def handle_boost_task(boost_id):
     """remove the original post and other, earlier boosts"""
     instance = models.Status.objects.get(id=boost_id)

bookwyrm/apps.py

@@ -1,4 +1,5 @@
 """Do further startup configuration and initialization"""
+
 import os
 import urllib
 import logging
@@ -14,16 +15,16 @@ def download_file(url, destination):
     """Downloads a file to the given path"""
     try:
         # Ensure our destination directory exists
-        os.makedirs(os.path.dirname(destination))
+        os.makedirs(os.path.dirname(destination), exist_ok=True)
         with urllib.request.urlopen(url) as stream:
             with open(destination, "b+w") as outfile:
                 outfile.write(stream.read())
-    except (urllib.error.HTTPError, urllib.error.URLError):
-        logger.info("Failed to download file %s", url)
-    except OSError:
-        logger.info("Couldn't open font file %s for writing", destination)
-    except:  # pylint: disable=bare-except
-        logger.info("Unknown error in file download")
+    except (urllib.error.HTTPError, urllib.error.URLError) as err:
+        logger.error("Failed to download file %s: %s", url, err)
+    except OSError as err:
+        logger.error("Couldn't open font file %s for writing: %s", destination, err)
+    except Exception as err:  # pylint:disable=broad-except
+        logger.error("Unknown error in file download: %s", err)
 
 
 class BookwyrmConfig(AppConfig):
@@ -32,7 +33,6 @@ class BookwyrmConfig(AppConfig):
     name = "bookwyrm"
     verbose_name = "BookWyrm"
 
-    # pylint: disable=no-self-use
     def ready(self):
         """set up OTLP and preview image files, if desired"""
         if settings.OTEL_EXPORTER_OTLP_ENDPOINT or settings.OTEL_EXPORTER_CONSOLE:

bookwyrm/book_search.py

@@ -1,35 +1,68 @@
 """ using a bookwyrm instance as a source of book data """
+from __future__ import annotations
 from dataclasses import asdict, dataclass
 from functools import reduce
 import operator
+from typing import Optional, Union, Any, Literal, overload
 
 from django.contrib.postgres.search import SearchRank, SearchQuery
 from django.db.models import F, Q
+from django.db.models.query import QuerySet
 
 from bookwyrm import models
 from bookwyrm import connectors
 from bookwyrm.settings import MEDIA_FULL_URL
 
 
-# pylint: disable=arguments-differ
-def search(query, min_confidence=0, filters=None, return_first=False):
+@overload
+def search(
+    query: str,
+    *,
+    min_confidence: float = 0,
+    filters: Optional[list[Any]] = None,
+    return_first: Literal[False],
+) -> QuerySet[models.Edition]:
+    ...
+
+
+@overload
+def search(
+    query: str,
+    *,
+    min_confidence: float = 0,
+    filters: Optional[list[Any]] = None,
+    return_first: Literal[True],
+) -> Optional[models.Edition]:
+    ...
+
+
+def search(
+    query: str,
+    *,
+    min_confidence: float = 0,
+    filters: Optional[list[Any]] = None,
+    return_first: bool = False,
+    books: Optional[QuerySet[models.Edition]] = None,
+) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
     """search your local database"""
     filters = filters or []
     if not query:
-        return []
+        return None if return_first else []
     query = query.strip()
 
     results = None
     # first, try searching unique identifiers
     # unique identifiers never have spaces, title/author usually do
     if not " " in query:
-        results = search_identifiers(query, *filters, return_first=return_first)
+        results = search_identifiers(
+            query, *filters, return_first=return_first, books=books
+        )
 
     # if there were no identifier results...
     if not results:
         # then try searching title/author
         results = search_title_author(
-            query, min_confidence, *filters, return_first=return_first
+            query, min_confidence, *filters, return_first=return_first, books=books
         )
     return results
 
@@ -66,8 +99,18 @@ def format_search_result(search_result):
     ).json()
 
 
-def search_identifiers(query, *filters, return_first=False):
-    """tries remote_id, isbn; defined as dedupe fields on the model"""
+def search_identifiers(
+    query,
+    *filters,
+    return_first=False,
+    books=None,
+) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
+    """search Editions by deduplication fields
+
+    Best for cases when we can assume someone is searching for an exact match on
+    commonly unique data identifiers like isbn or specific library ids.
+    """
+    books = books or models.Edition.objects
     if connectors.maybe_isbn(query):
         # Oh did you think the 'S' in ISBN stood for 'standard'?
         normalized_isbn = query.strip().upper().rjust(10, "0")
@@ -78,7 +121,7 @@ def search_identifiers(query, *filters, return_first=False):
         for f in models.Edition._meta.get_fields()
         if hasattr(f, "deduplication_field") and f.deduplication_field
     ]
-    results = models.Edition.objects.filter(
+    results = books.filter(
         *filters, reduce(operator.or_, (Q(**f) for f in or_filters))
     ).distinct()
 
@@ -87,11 +130,18 @@ def search_identifiers(query, *filters, return_first=False):
     return results
 
 
-def search_title_author(query, min_confidence, *filters, return_first=False):
+def search_title_author(
+    query,
+    min_confidence,
+    *filters,
+    return_first=False,
+    books=None,
+) -> QuerySet[models.Edition]:
     """searches for title and author"""
+    books = books or models.Edition.objects
     query = SearchQuery(query, config="simple") | SearchQuery(query, config="english")
     results = (
-        models.Edition.objects.filter(*filters, search_vector=query)
+        books.filter(*filters, search_vector=query)
         .annotate(rank=SearchRank(F("search_vector"), query))
         .filter(rank__gt=min_confidence)
         .order_by("-rank")
@@ -102,7 +152,7 @@ def search_title_author(query, min_confidence, *filters, return_first=False):
 
     # filter out multiple editions of the same work
     list_results = []
-    for work_id in set(editions_of_work[:30]):
+    for work_id in editions_of_work[:30]:
         result = (
             results.filter(parent_work=work_id)
             .order_by("-rank", "-edition_rank")
@@ -122,11 +172,11 @@ class SearchResult:
     title: str
     key: str
     connector: object
-    view_link: str = None
-    author: str = None
-    year: str = None
-    cover: str = None
-    confidence: int = 1
+    view_link: Optional[str] = None
+    author: Optional[str] = None
+    year: Optional[str] = None
+    cover: Optional[str] = None
+    confidence: float = 1.0
 
     def __repr__(self):
         # pylint: disable=consider-using-f-string

bookwyrm/connectors/abstract_connector.py

@@ -1,7 +1,11 @@
 """ functionality outline for a book data connector """
+from __future__ import annotations
 from abc import ABC, abstractmethod
+from typing import Optional, TypedDict, Any, Callable, Union, Iterator
 from urllib.parse import quote_plus
-import imghdr
+
+# pylint: disable-next=deprecated-module
+import imghdr  # Deprecated in 3.11 for removal in 3.13; no good alternative yet
 import logging
 import re
 import asyncio
@@ -16,33 +20,38 @@ from bookwyrm import activitypub, models, settings
 from bookwyrm.settings import USER_AGENT
 from .connector_manager import load_more_data, ConnectorException, raise_not_valid_url
 from .format_mappings import format_mappings
-
+from ..book_search import SearchResult
 
 logger = logging.getLogger(__name__)
 
+JsonDict = dict[str, Any]
+
+
+class ConnectorResults(TypedDict):
+    """TypedDict for results returned by connector"""
+
+    connector: AbstractMinimalConnector
+    results: list[SearchResult]
+
 
 class AbstractMinimalConnector(ABC):
     """just the bare bones, for other bookwyrm instances"""
 
-    def __init__(self, identifier):
+    def __init__(self, identifier: str):
         # load connector settings
         info = models.Connector.objects.get(identifier=identifier)
         self.connector = info
 
         # the things in the connector model to copy over
-        self_fields = [
-            "base_url",
-            "books_url",
-            "covers_url",
-            "search_url",
-            "isbn_search_url",
-            "name",
-            "identifier",
-        ]
-        for field in self_fields:
-            setattr(self, field, getattr(info, field))
+        self.base_url = info.base_url
+        self.books_url = info.books_url
+        self.covers_url = info.covers_url
+        self.search_url = info.search_url
+        self.isbn_search_url = info.isbn_search_url
+        self.name = info.name
+        self.identifier = info.identifier
 
-    def get_search_url(self, query):
+    def get_search_url(self, query: str) -> str:
         """format the query url"""
         # Check if the query resembles an ISBN
         if maybe_isbn(query) and self.isbn_search_url and self.isbn_search_url != "":
@@ -54,13 +63,21 @@ class AbstractMinimalConnector(ABC):
         # searched as free text. This, instead, only searches isbn if it's isbn-y
         return f"{self.search_url}{quote_plus(query)}"
 
-    def process_search_response(self, query, data, min_confidence):
+    def process_search_response(
+        self, query: str, data: Any, min_confidence: float
+    ) -> list[SearchResult]:
         """Format the search results based on the format of the query"""
         if maybe_isbn(query):
             return list(self.parse_isbn_search_data(data))[:10]
         return list(self.parse_search_data(data, min_confidence))[:10]
 
-    async def get_results(self, session, url, min_confidence, query):
+    async def get_results(
+        self,
+        session: aiohttp.ClientSession,
+        url: str,
+        min_confidence: float,
+        query: str,
+    ) -> Optional[ConnectorResults]:
         """try this specific connector"""
         # pylint: disable=line-too-long
         headers = {
@@ -69,60 +86,68 @@ class AbstractMinimalConnector(ABC):
             ),
             "User-Agent": USER_AGENT,
         }
-        params = {"min_confidence": min_confidence}
+        params = {"min_confidence": str(min_confidence)}
         try:
             async with session.get(url, headers=headers, params=params) as response:
                 if not response.ok:
                     logger.info("Unable to connect to %s: %s", url, response.reason)
-                    return
+                    return None
 
                 try:
                     raw_data = await response.json()
                 except aiohttp.client_exceptions.ContentTypeError as err:
                     logger.exception(err)
-                    return
+                    return None
 
-                return {
-                    "connector": self,
-                    "results": self.process_search_response(
+                return ConnectorResults(
+                    connector=self,
+                    results=self.process_search_response(
                         query, raw_data, min_confidence
                     ),
-                }
+                )
         except asyncio.TimeoutError:
             logger.info("Connection timed out for url: %s", url)
         except aiohttp.ClientError as err:
             logger.info(err)
+        return None
 
     @abstractmethod
-    def get_or_create_book(self, remote_id):
+    def get_or_create_book(self, remote_id: str) -> Optional[models.Book]:
         """pull up a book record by whatever means possible"""
 
     @abstractmethod
-    def parse_search_data(self, data, min_confidence):
+    def parse_search_data(
+        self, data: Any, min_confidence: float
+    ) -> Iterator[SearchResult]:
         """turn the result json from a search into a list"""
 
     @abstractmethod
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(self, data: Any) -> Iterator[SearchResult]:
         """turn the result json from a search into a list"""
 
 
 class AbstractConnector(AbstractMinimalConnector):
     """generic book data connector"""
 
-    def __init__(self, identifier):
+    generated_remote_link_field = ""
+
+    def __init__(self, identifier: str):
         super().__init__(identifier)
         # fields we want to look for in book data to copy over
         # title we handle separately.
-        self.book_mappings = []
+        self.book_mappings: list[Mapping] = []
+        self.author_mappings: list[Mapping] = []
 
-    def get_or_create_book(self, remote_id):
+    def get_or_create_book(self, remote_id: str) -> Optional[models.Book]:
         """translate arbitrary json into an Activitypub dataclass"""
         # first, check if we have the origin_id saved
         existing = models.Edition.find_existing_by_remote_id(
             remote_id
         ) or models.Work.find_existing_by_remote_id(remote_id)
         if existing:
-            if hasattr(existing, "default_edition"):
+            if hasattr(existing, "default_edition") and isinstance(
+                existing.default_edition, models.Edition
+            ):
                 return existing.default_edition
             return existing
 
@@ -154,6 +179,9 @@ class AbstractConnector(AbstractMinimalConnector):
             )
             # this will dedupe automatically
             work = work_activity.to_model(model=models.Work, overwrite=False)
+            if not work:
+                return None
+
             for author in self.get_authors_from_data(work_data):
                 work.authors.add(author)
 
@@ -161,12 +189,21 @@ class AbstractConnector(AbstractMinimalConnector):
         load_more_data.delay(self.connector.id, work.id)
         return edition
 
-    def get_book_data(self, remote_id):  # pylint: disable=no-self-use
+    def get_book_data(self, remote_id: str) -> JsonDict:  # pylint: disable=no-self-use
         """this allows connectors to override the default behavior"""
         return get_data(remote_id)
 
-    def create_edition_from_data(self, work, edition_data, instance=None):
+    def create_edition_from_data(
+        self,
+        work: models.Work,
+        edition_data: Union[str, JsonDict],
+        instance: Optional[models.Edition] = None,
+    ) -> Optional[models.Edition]:
         """if we already have the work, we're ready"""
+        if isinstance(edition_data, str):
+            # We don't expect a string here
+            return None
+
         mapped_data = dict_from_mappings(edition_data, self.book_mappings)
         mapped_data["work"] = work.remote_id
         edition_activity = activitypub.Edition(**mapped_data)
@@ -174,6 +211,9 @@ class AbstractConnector(AbstractMinimalConnector):
             model=models.Edition, overwrite=False, instance=instance
         )
 
+        if not edition:
+            return None
+
         # if we're updating an existing instance, we don't need to load authors
         if instance:
             return edition
@@ -190,7 +230,9 @@ class AbstractConnector(AbstractMinimalConnector):
 
         return edition
 
-    def get_or_create_author(self, remote_id, instance=None):
+    def get_or_create_author(
+        self, remote_id: str, instance: Optional[models.Author] = None
+    ) -> Optional[models.Author]:
         """load that author"""
         if not instance:
             existing = models.Author.find_existing_by_remote_id(remote_id)
@@ -210,46 +252,51 @@ class AbstractConnector(AbstractMinimalConnector):
             model=models.Author, overwrite=False, instance=instance
         )
 
-    def get_remote_id_from_model(self, obj):
+    def get_remote_id_from_model(self, obj: models.BookDataModel) -> Optional[str]:
         """given the data stored, how can we look this up"""
-        return getattr(obj, getattr(self, "generated_remote_link_field"))
+        remote_id: Optional[str] = getattr(obj, self.generated_remote_link_field)
+        return remote_id
 
-    def update_author_from_remote(self, obj):
+    def update_author_from_remote(self, obj: models.Author) -> Optional[models.Author]:
         """load the remote data from this connector and add it to an existing author"""
         remote_id = self.get_remote_id_from_model(obj)
+        if not remote_id:
+            return None
         return self.get_or_create_author(remote_id, instance=obj)
 
-    def update_book_from_remote(self, obj):
+    def update_book_from_remote(self, obj: models.Edition) -> Optional[models.Edition]:
         """load the remote data from this connector and add it to an existing book"""
         remote_id = self.get_remote_id_from_model(obj)
+        if not remote_id:
+            return None
         data = self.get_book_data(remote_id)
         return self.create_edition_from_data(obj.parent_work, data, instance=obj)
 
     @abstractmethod
-    def is_work_data(self, data):
+    def is_work_data(self, data: JsonDict) -> bool:
         """differentiate works and editions"""
 
     @abstractmethod
-    def get_edition_from_work_data(self, data):
+    def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
         """every work needs at least one edition"""
 
     @abstractmethod
-    def get_work_from_edition_data(self, data):
+    def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
         """every edition needs a work"""
 
     @abstractmethod
-    def get_authors_from_data(self, data):
+    def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
         """load author data"""
 
     @abstractmethod
-    def expand_book_data(self, book):
+    def expand_book_data(self, book: models.Book) -> None:
         """get more info on a book"""
 
 
-def dict_from_mappings(data, mappings):
+def dict_from_mappings(data: JsonDict, mappings: list[Mapping]) -> JsonDict:
     """create a dict in Activitypub format, using mappings supplies by
     the subclass"""
-    result = {}
+    result: JsonDict = {}
     for mapping in mappings:
         # sometimes there are multiple mappings for one field, don't
         # overwrite earlier writes in that case
@@ -259,7 +306,11 @@ def dict_from_mappings(data, mappings):
     return result
 
 
-def get_data(url, params=None, timeout=settings.QUERY_TIMEOUT):
+def get_data(
+    url: str,
+    params: Optional[dict[str, str]] = None,
+    timeout: int = settings.QUERY_TIMEOUT,
+) -> JsonDict:
     """wrapper for request.get"""
     # check if the url is blocked
     raise_not_valid_url(url)
@@ -292,10 +343,15 @@ def get_data(url, params=None, timeout=settings.QUERY_TIMEOUT):
         logger.info(err)
         raise ConnectorException(err)
 
+    if not isinstance(data, dict):
+        raise ConnectorException("Unexpected data format")
+
     return data
 
 
-def get_image(url, timeout=10):
+def get_image(
+    url: str, timeout: int = 10
+) -> Union[tuple[ContentFile[bytes], str], tuple[None, None]]:
     """wrapper for requesting an image"""
     raise_not_valid_url(url)
     try:
@@ -325,14 +381,19 @@ def get_image(url, timeout=10):
 class Mapping:
     """associate a local database field with a field in an external dataset"""
 
-    def __init__(self, local_field, remote_field=None, formatter=None):
+    def __init__(
+        self,
+        local_field: str,
+        remote_field: Optional[str] = None,
+        formatter: Optional[Callable[[Any], Any]] = None,
+    ):
         noop = lambda x: x
 
         self.local_field = local_field
         self.remote_field = remote_field or local_field
         self.formatter = formatter or noop
 
-    def get_value(self, data):
+    def get_value(self, data: JsonDict) -> Optional[Any]:
         """pull a field from incoming json and return the formatted version"""
         value = data.get(self.remote_field)
         if not value:
@@ -343,7 +404,7 @@ class Mapping:
             return None
 
 
-def infer_physical_format(format_text):
+def infer_physical_format(format_text: str) -> Optional[str]:
     """try to figure out what the standardized format is from the free value"""
     format_text = format_text.lower()
     if format_text in format_mappings:
@@ -356,7 +417,7 @@ def infer_physical_format(format_text):
     return matches[0]
 
 
-def unique_physical_format(format_text):
+def unique_physical_format(format_text: str) -> Optional[str]:
     """only store the format if it isn't directly in the format mappings"""
     format_text = format_text.lower()
     if format_text in format_mappings:
@@ -365,7 +426,7 @@ def unique_physical_format(format_text):
     return format_text
 
 
-def maybe_isbn(query):
+def maybe_isbn(query: str) -> bool:
     """check if a query looks like an isbn"""
     isbn = re.sub(r"[\W_]", "", query)  # removes filler characters
     # ISBNs must be numeric except an ISBN10 checkdigit can be 'X'

bookwyrm/connectors/bookwyrm_connector.py

@@ -1,4 +1,7 @@
 """ using another bookwyrm instance as a source of book data """
+from __future__ import annotations
+from typing import Any, Iterator
+
 from bookwyrm import activitypub, models
 from bookwyrm.book_search import SearchResult
 from .abstract_connector import AbstractMinimalConnector
@@ -7,15 +10,19 @@ from .abstract_connector import AbstractMinimalConnector
 class Connector(AbstractMinimalConnector):
     """this is basically just for search"""
 
-    def get_or_create_book(self, remote_id):
+    def get_or_create_book(self, remote_id: str) -> models.Edition:
         return activitypub.resolve_remote_id(remote_id, model=models.Edition)
 
-    def parse_search_data(self, data, min_confidence):
+    def parse_search_data(
+        self, data: list[dict[str, Any]], min_confidence: float
+    ) -> Iterator[SearchResult]:
         for search_result in data:
             search_result["connector"] = self
             yield SearchResult(**search_result)
 
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(
+        self, data: list[dict[str, Any]]
+    ) -> Iterator[SearchResult]:
         for search_result in data:
             search_result["connector"] = self
             yield SearchResult(**search_result)

bookwyrm/connectors/connector_manager.py

@@ -1,8 +1,11 @@
 """ interface with whatever connectors the app has """
+from __future__ import annotations
 import asyncio
 import importlib
 import ipaddress
 import logging
+from asyncio import Future
+from typing import Iterator, Any, Optional, Union, overload, Literal
 from urllib.parse import urlparse
 
 import aiohttp
@@ -12,8 +15,10 @@ from django.db.models import signals
 from requests import HTTPError
 
 from bookwyrm import book_search, models
+from bookwyrm.book_search import SearchResult
+from bookwyrm.connectors import abstract_connector
 from bookwyrm.settings import SEARCH_TIMEOUT
-from bookwyrm.tasks import app, LOW
+from bookwyrm.tasks import app, CONNECTORS
 
 logger = logging.getLogger(__name__)
 
@@ -22,11 +27,15 @@ class ConnectorException(HTTPError):
     """when the connector can't do what was asked"""
 
 
-async def async_connector_search(query, items, min_confidence):
+async def async_connector_search(
+    query: str,
+    items: list[tuple[str, abstract_connector.AbstractConnector]],
+    min_confidence: float,
+) -> list[Optional[abstract_connector.ConnectorResults]]:
     """Try a number of requests simultaneously"""
     timeout = aiohttp.ClientTimeout(total=SEARCH_TIMEOUT)
     async with aiohttp.ClientSession(timeout=timeout) as session:
-        tasks = []
+        tasks: list[Future[Optional[abstract_connector.ConnectorResults]]] = []
         for url, connector in items:
             tasks.append(
                 asyncio.ensure_future(
@@ -35,14 +44,29 @@ async def async_connector_search(query, items, min_confidence):
             )
 
         results = await asyncio.gather(*tasks)
-        return results
+        return list(results)
 
 
-def search(query, min_confidence=0.1, return_first=False):
+@overload
+def search(
+    query: str, *, min_confidence: float = 0.1, return_first: Literal[False]
+) -> list[abstract_connector.ConnectorResults]:
+    ...
+
+
+@overload
+def search(
+    query: str, *, min_confidence: float = 0.1, return_first: Literal[True]
+) -> Optional[SearchResult]:
+    ...
+
+
+def search(
+    query: str, *, min_confidence: float = 0.1, return_first: bool = False
+) -> Union[list[abstract_connector.ConnectorResults], Optional[SearchResult]]:
     """find books based on arbitrary keywords"""
     if not query:
-        return []
-    results = []
+        return None if return_first else []
 
     items = []
     for connector in get_connectors():
@@ -57,8 +81,12 @@ def search(query, min_confidence=0.1, return_first=False):
         items.append((url, connector))
 
     # load as many results as we can
-    results = asyncio.run(async_connector_search(query, items, min_confidence))
-    results = [r for r in results if r]
+    # failed requests will return None, so filter those out
+    results = [
+        r
+        for r in asyncio.run(async_connector_search(query, items, min_confidence))
+        if r
+    ]
 
     if return_first:
         # find the best result from all the responses and return that
@@ -66,11 +94,12 @@ def search(query, min_confidence=0.1, return_first=False):
         all_results = sorted(all_results, key=lambda r: r.confidence, reverse=True)
         return all_results[0] if all_results else None
 
-    # failed requests will return None, so filter those out
     return results
 
 
-def first_search_result(query, min_confidence=0.1):
+def first_search_result(
+    query: str, min_confidence: float = 0.1
+) -> Union[models.Edition, SearchResult, None]:
     """search until you find a result that fits"""
     # try local search first
     result = book_search.search(query, min_confidence=min_confidence, return_first=True)
@@ -80,18 +109,20 @@ def first_search_result(query, min_confidence=0.1):
     return search(query, min_confidence=min_confidence, return_first=True) or None
 
 
-def get_connectors():
+def get_connectors() -> Iterator[abstract_connector.AbstractConnector]:
     """load all connectors"""
     for info in models.Connector.objects.filter(active=True).order_by("priority").all():
         yield load_connector(info)
 
 
-def get_or_create_connector(remote_id):
+def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnector:
     """get the connector related to the object's server"""
     url = urlparse(remote_id)
-    identifier = url.netloc
+    identifier = url.hostname
     if not identifier:
-        raise ValueError("Invalid remote id")
+        raise ValueError(f"Invalid remote id: {remote_id}")
+
+    base_url = f"{url.scheme}://{url.netloc}"
 
     try:
         connector_info = models.Connector.objects.get(identifier=identifier)
@@ -99,58 +130,75 @@ def get_or_create_connector(remote_id):
         connector_info = models.Connector.objects.create(
             identifier=identifier,
             connector_file="bookwyrm_connector",
-            base_url=f"https://{identifier}",
-            books_url=f"https://{identifier}/book",
-            covers_url=f"https://{identifier}/images/covers",
-            search_url=f"https://{identifier}/search?q=",
+            base_url=base_url,
+            books_url=f"{base_url}/book",
+            covers_url=f"{base_url}/images/covers",
+            search_url=f"{base_url}/search?q=",
             priority=2,
         )
 
     return load_connector(connector_info)
 
 
-@app.task(queue=LOW)
-def load_more_data(connector_id, book_id):
+@app.task(queue=CONNECTORS)
+def load_more_data(connector_id: str, book_id: str) -> None:
     """background the work of getting all 10,000 editions of LoTR"""
     connector_info = models.Connector.objects.get(id=connector_id)
     connector = load_connector(connector_info)
-    book = models.Book.objects.select_subclasses().get(id=book_id)
+    book = models.Book.objects.select_subclasses().get(  # type: ignore[no-untyped-call]
+        id=book_id
+    )
     connector.expand_book_data(book)
 
 
-@app.task(queue=LOW)
-def create_edition_task(connector_id, work_id, data):
+@app.task(queue=CONNECTORS)
+def create_edition_task(
+    connector_id: int, work_id: int, data: Union[str, abstract_connector.JsonDict]
+) -> None:
     """separate task for each of the 10,000 editions of LoTR"""
     connector_info = models.Connector.objects.get(id=connector_id)
     connector = load_connector(connector_info)
-    work = models.Work.objects.select_subclasses().get(id=work_id)
+    work = models.Work.objects.select_subclasses().get(  # type: ignore[no-untyped-call]
+        id=work_id
+    )
     connector.create_edition_from_data(work, data)
 
 
-def load_connector(connector_info):
+def load_connector(
+    connector_info: models.Connector,
+) -> abstract_connector.AbstractConnector:
     """instantiate the connector class"""
     connector = importlib.import_module(
         f"bookwyrm.connectors.{connector_info.connector_file}"
     )
-    return connector.Connector(connector_info.identifier)
+    return connector.Connector(connector_info.identifier)  # type: ignore[no-any-return]
 
 
 @receiver(signals.post_save, sender="bookwyrm.FederatedServer")
 # pylint: disable=unused-argument
-def create_connector(sender, instance, created, *args, **kwargs):
+def create_connector(
+    sender: Any,
+    instance: models.FederatedServer,
+    created: Any,
+    *args: Any,
+    **kwargs: Any,
+) -> None:
     """create a connector to an external bookwyrm server"""
     if instance.application_type == "bookwyrm":
         get_or_create_connector(f"https://{instance.server_name}")
 
 
-def raise_not_valid_url(url):
+def raise_not_valid_url(url: str) -> None:
     """do some basic reality checks on the url"""
     parsed = urlparse(url)
     if not parsed.scheme in ["http", "https"]:
         raise ConnectorException("Invalid scheme: ", url)
 
+    if not parsed.hostname:
+        raise ConnectorException("Hostname missing: ", url)
+
     try:
-        ipaddress.ip_address(parsed.netloc)
+        ipaddress.ip_address(parsed.hostname)
         raise ConnectorException("Provided url is an IP address: ", url)
     except ValueError:
         # it's not an IP address, which is good

bookwyrm/connectors/inventaire.py

@@ -1,9 +1,10 @@
 """ inventaire data connector """
 import re
+from typing import Any, Union, Optional, Iterator, Iterable
 
 from bookwyrm import models
 from bookwyrm.book_search import SearchResult
-from .abstract_connector import AbstractConnector, Mapping
+from .abstract_connector import AbstractConnector, Mapping, JsonDict
 from .abstract_connector import get_data
 from .connector_manager import ConnectorException, create_edition_task
 
@@ -13,7 +14,7 @@ class Connector(AbstractConnector):
 
     generated_remote_link_field = "inventaire_id"
 
-    def __init__(self, identifier):
+    def __init__(self, identifier: str):
         super().__init__(identifier)
 
         get_first = lambda a: a[0]
@@ -60,13 +61,13 @@ class Connector(AbstractConnector):
             Mapping("died", remote_field="wdt:P570", formatter=get_first),
         ] + shared_mappings
 
-    def get_remote_id(self, value):
+    def get_remote_id(self, value: str) -> str:
         """convert an id/uri into a url"""
         return f"{self.books_url}?action=by-uris&uris={value}"
 
-    def get_book_data(self, remote_id):
+    def get_book_data(self, remote_id: str) -> JsonDict:
         data = get_data(remote_id)
-        extracted = list(data.get("entities").values())
+        extracted = list(data.get("entities", {}).values())
         try:
             data = extracted[0]
         except (KeyError, IndexError):
@@ -74,10 +75,16 @@ class Connector(AbstractConnector):
         # flatten the data so that images, uri, and claims are on the same level
         return {
             **data.get("claims", {}),
-            **{k: data.get(k) for k in ["uri", "image", "labels", "sitelinks", "type"]},
+            **{
+                k: data.get(k)
+                for k in ["uri", "image", "labels", "sitelinks", "type"]
+                if k in data
+            },
         }
 
-    def parse_search_data(self, data, min_confidence):
+    def parse_search_data(
+        self, data: JsonDict, min_confidence: float
+    ) -> Iterator[SearchResult]:
         for search_result in data.get("results", []):
             images = search_result.get("image")
             cover = f"{self.covers_url}/img/entities/{images[0]}" if images else None
@@ -96,7 +103,7 @@ class Connector(AbstractConnector):
                 connector=self,
             )
 
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
         """got some data"""
         results = data.get("entities")
         if not results:
@@ -114,35 +121,44 @@ class Connector(AbstractConnector):
                 connector=self,
             )
 
-    def is_work_data(self, data):
+    def is_work_data(self, data: JsonDict) -> bool:
         return data.get("type") == "work"
 
-    def load_edition_data(self, work_uri):
+    def load_edition_data(self, work_uri: str) -> JsonDict:
         """get a list of editions for a work"""
         # pylint: disable=line-too-long
         url = f"{self.books_url}?action=reverse-claims&property=wdt:P629&value={work_uri}&sort=true"
         return get_data(url)
 
-    def get_edition_from_work_data(self, data):
-        data = self.load_edition_data(data.get("uri"))
+    def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
+        work_uri = data.get("uri")
+        if not work_uri:
+            raise ConnectorException("Invalid URI")
+        data = self.load_edition_data(work_uri)
         try:
             uri = data.get("uris", [])[0]
         except IndexError:
             raise ConnectorException("Invalid book data")
         return self.get_book_data(self.get_remote_id(uri))
 
-    def get_work_from_edition_data(self, data):
-        uri = data.get("wdt:P629", [None])[0]
+    def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
+        try:
+            uri = data.get("wdt:P629", [])[0]
+        except IndexError:
+            raise ConnectorException("Invalid book data")
+
         if not uri:
             raise ConnectorException("Invalid book data")
         return self.get_book_data(self.get_remote_id(uri))
 
-    def get_authors_from_data(self, data):
+    def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
         authors = data.get("wdt:P50", [])
         for author in authors:
-            yield self.get_or_create_author(self.get_remote_id(author))
+            model = self.get_or_create_author(self.get_remote_id(author))
+            if model:
+                yield model
 
-    def expand_book_data(self, book):
+    def expand_book_data(self, book: models.Book) -> None:
         work = book
         # go from the edition to the work, if necessary
         if isinstance(book, models.Edition):
@@ -154,11 +170,16 @@ class Connector(AbstractConnector):
             # who knows, man
             return
 
-        for edition_uri in edition_options.get("uris"):
+        for edition_uri in edition_options.get("uris", []):
             remote_id = self.get_remote_id(edition_uri)
             create_edition_task.delay(self.connector.id, work.id, remote_id)
 
-    def create_edition_from_data(self, work, edition_data, instance=None):
+    def create_edition_from_data(
+        self,
+        work: models.Work,
+        edition_data: Union[str, JsonDict],
+        instance: Optional[models.Edition] = None,
+    ) -> Optional[models.Edition]:
         """pass in the url as data and then call the version in abstract connector"""
         if isinstance(edition_data, str):
             try:
@@ -168,22 +189,26 @@ class Connector(AbstractConnector):
                 return None
         return super().create_edition_from_data(work, edition_data, instance=instance)
 
-    def get_cover_url(self, cover_blob, *_):
+    def get_cover_url(
+        self, cover_blob: Union[list[JsonDict], JsonDict], *_: Any
+    ) -> Optional[str]:
         """format the relative cover url into an absolute one:
         {"url": "/img/entities/e794783f01b9d4f897a1ea9820b96e00d346994f"}
         """
         # covers may or may not be a list
-        if isinstance(cover_blob, list) and len(cover_blob) > 0:
+        if isinstance(cover_blob, list):
+            if len(cover_blob) == 0:
+                return None
             cover_blob = cover_blob[0]
         cover_id = cover_blob.get("url")
-        if not cover_id:
+        if not isinstance(cover_id, str):
             return None
         # cover may or may not be an absolute url already
         if re.match(r"^http", cover_id):
             return cover_id
         return f"{self.covers_url}{cover_id}"
 
-    def resolve_keys(self, keys):
+    def resolve_keys(self, keys: Iterable[str]) -> list[str]:
         """cool, it's "wd:Q3156592" now what the heck does that mean"""
         results = []
         for uri in keys:
@@ -191,10 +216,10 @@ class Connector(AbstractConnector):
                 data = self.get_book_data(self.get_remote_id(uri))
             except ConnectorException:
                 continue
-            results.append(get_language_code(data.get("labels")))
+            results.append(get_language_code(data.get("labels", {})))
         return results
 
-    def get_description(self, links):
+    def get_description(self, links: JsonDict) -> str:
         """grab an extracted excerpt from wikipedia"""
         link = links.get("enwiki")
         if not link:
@@ -204,15 +229,15 @@ class Connector(AbstractConnector):
             data = get_data(url)
         except ConnectorException:
             return ""
-        return data.get("extract")
+        return str(data.get("extract", ""))
 
-    def get_remote_id_from_model(self, obj):
+    def get_remote_id_from_model(self, obj: models.BookDataModel) -> str:
         """use get_remote_id to figure out the link from a model obj"""
         remote_id_value = obj.inventaire_id
         return self.get_remote_id(remote_id_value)
 
 
-def get_language_code(options, code="en"):
+def get_language_code(options: JsonDict, code: str = "en") -> Any:
     """when there are a bunch of translation but we need a single field"""
     result = options.get(code)
     if result:

bookwyrm/connectors/openlibrary.py

@@ -1,9 +1,13 @@
 """ openlibrary data connector """
 import re
+from typing import Any, Optional, Union, Iterator, Iterable
+
+from markdown import markdown
 
 from bookwyrm import models
 from bookwyrm.book_search import SearchResult
-from .abstract_connector import AbstractConnector, Mapping
+from bookwyrm.utils.sanitizer import clean
+from .abstract_connector import AbstractConnector, Mapping, JsonDict
 from .abstract_connector import get_data, infer_physical_format, unique_physical_format
 from .connector_manager import ConnectorException, create_edition_task
 from .openlibrary_languages import languages
@@ -14,7 +18,7 @@ class Connector(AbstractConnector):
 
     generated_remote_link_field = "openlibrary_link"
 
-    def __init__(self, identifier):
+    def __init__(self, identifier: str):
         super().__init__(identifier)
 
         get_first = lambda a, *args: a[0]
@@ -94,14 +98,14 @@ class Connector(AbstractConnector):
             Mapping("inventaire_id", remote_field="links", formatter=get_inventaire_id),
         ]
 
-    def get_book_data(self, remote_id):
+    def get_book_data(self, remote_id: str) -> JsonDict:
         data = get_data(remote_id)
         if data.get("type", {}).get("key") == "/type/redirect":
-            remote_id = self.base_url + data.get("location")
+            remote_id = self.base_url + data.get("location", "")
             return get_data(remote_id)
         return data
 
-    def get_remote_id_from_data(self, data):
+    def get_remote_id_from_data(self, data: JsonDict) -> str:
         """format a url from an openlibrary id field"""
         try:
             key = data["key"]
@@ -109,10 +113,10 @@ class Connector(AbstractConnector):
             raise ConnectorException("Invalid book data")
         return f"{self.books_url}{key}"
 
-    def is_work_data(self, data):
+    def is_work_data(self, data: JsonDict) -> bool:
         return bool(re.match(r"^[\/\w]+OL\d+W$", data["key"]))
 
-    def get_edition_from_work_data(self, data):
+    def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
         try:
             key = data["key"]
         except KeyError:
@@ -124,7 +128,7 @@ class Connector(AbstractConnector):
             raise ConnectorException("No editions for work")
         return edition
 
-    def get_work_from_edition_data(self, data):
+    def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
         try:
             key = data["works"][0]["key"]
         except (IndexError, KeyError):
@@ -132,7 +136,7 @@ class Connector(AbstractConnector):
         url = f"{self.books_url}{key}"
         return self.get_book_data(url)
 
-    def get_authors_from_data(self, data):
+    def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
         """parse author json and load or create authors"""
         for author_blob in data.get("authors", []):
             author_blob = author_blob.get("author", author_blob)
@@ -144,7 +148,7 @@ class Connector(AbstractConnector):
                 continue
             yield author
 
-    def get_cover_url(self, cover_blob, size="L"):
+    def get_cover_url(self, cover_blob: list[str], size: str = "L") -> Optional[str]:
         """ask openlibrary for the cover"""
         if not cover_blob:
             return None
@@ -152,8 +156,10 @@ class Connector(AbstractConnector):
         image_name = f"{cover_id}-{size}.jpg"
         return f"{self.covers_url}/b/id/{image_name}"
 
-    def parse_search_data(self, data, min_confidence):
-        for idx, search_result in enumerate(data.get("docs")):
+    def parse_search_data(
+        self, data: JsonDict, min_confidence: float
+    ) -> Iterator[SearchResult]:
+        for idx, search_result in enumerate(data.get("docs", [])):
             # build the remote id from the openlibrary key
             key = self.books_url + search_result["key"]
             author = search_result.get("author_name") or ["Unknown"]
@@ -174,7 +180,7 @@ class Connector(AbstractConnector):
                 confidence=confidence,
             )
 
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
         for search_result in list(data.values()):
             # build the remote id from the openlibrary key
             key = self.books_url + search_result["key"]
@@ -188,12 +194,12 @@ class Connector(AbstractConnector):
                 year=search_result.get("publish_date"),
             )
 
-    def load_edition_data(self, olkey):
+    def load_edition_data(self, olkey: str) -> JsonDict:
         """query openlibrary for editions of a work"""
         url = f"{self.books_url}/works/{olkey}/editions"
         return self.get_book_data(url)
 
-    def expand_book_data(self, book):
+    def expand_book_data(self, book: models.Book) -> None:
         work = book
         # go from the edition to the work, if necessary
         if isinstance(book, models.Edition):
@@ -206,14 +212,14 @@ class Connector(AbstractConnector):
             # who knows, man
             return
 
-        for edition_data in edition_options.get("entries"):
+        for edition_data in edition_options.get("entries", []):
             # does this edition have ANY interesting data?
             if ignore_edition(edition_data):
                 continue
             create_edition_task.delay(self.connector.id, work.id, edition_data)
 
 
-def ignore_edition(edition_data):
+def ignore_edition(edition_data: JsonDict) -> bool:
     """don't load a million editions that have no metadata"""
     # an isbn, we love to see it
     if edition_data.get("isbn_13") or edition_data.get("isbn_10"):
@@ -232,19 +238,30 @@ def ignore_edition(edition_data):
     return True
 
 
-def get_description(description_blob):
+def get_description(description_blob: Union[JsonDict, str]) -> str:
     """descriptions can be a string or a dict"""
     if isinstance(description_blob, dict):
-        return description_blob.get("value")
-    return description_blob
+        description = markdown(description_blob.get("value", ""))
+    else:
+        description = markdown(description_blob)
+
+    if (
+        description.startswith("<p>")
+        and description.endswith("</p>")
+        and description.count("<p>") == 1
+    ):
+        # If there is just one <p> tag and it is around the text remove it
+        return description[len("<p>") : -len("</p>")].strip()
+
+    return clean(description)
 
 
-def get_openlibrary_key(key):
+def get_openlibrary_key(key: str) -> str:
     """convert /books/OL27320736M into OL27320736M"""
     return key.split("/")[-1]
 
 
-def get_languages(language_blob):
+def get_languages(language_blob: Iterable[JsonDict]) -> list[Optional[str]]:
     """/language/eng -> English"""
     langs = []
     for lang in language_blob:
@@ -252,14 +269,14 @@ def get_languages(language_blob):
     return langs
 
 
-def get_dict_field(blob, field_name):
+def get_dict_field(blob: Optional[JsonDict], field_name: str) -> Optional[Any]:
     """extract the isni from the remote id data for the author"""
     if not blob or not isinstance(blob, dict):
         return None
     return blob.get(field_name)
 
 
-def get_wikipedia_link(links):
+def get_wikipedia_link(links: list[Any]) -> Optional[str]:
     """extract wikipedia links"""
     if not isinstance(links, list):
         return None
@@ -272,7 +289,7 @@ def get_wikipedia_link(links):
     return None
 
 
-def get_inventaire_id(links):
+def get_inventaire_id(links: list[Any]) -> Optional[str]:
     """extract and format inventaire ids"""
     if not isinstance(links, list):
         return None
@@ -282,11 +299,13 @@ def get_inventaire_id(links):
             continue
         if link.get("title") == "inventaire.io":
             iv_link = link.get("url")
+            if not isinstance(iv_link, str):
+                return None
             return iv_link.split("/")[-1]
     return None
 
 
-def pick_default_edition(options):
+def pick_default_edition(options: list[JsonDict]) -> Optional[JsonDict]:
     """favor physical copies with covers in english"""
     if not options:
         return None

bookwyrm/context_processors.py

@@ -2,7 +2,7 @@
 from bookwyrm import models, settings
 
 
-def site_settings(request):  # pylint: disable=unused-argument
+def site_settings(request):
     """include the custom info about the site"""
     request_protocol = "https://"
     if not request.is_secure():

bookwyrm/emailing.py

@@ -3,8 +3,8 @@ from django.core.mail import EmailMultiAlternatives
 from django.template.loader import get_template
 
 from bookwyrm import models, settings
-from bookwyrm.tasks import app, HIGH
-from bookwyrm.settings import DOMAIN
+from bookwyrm.tasks import app, EMAIL
+from bookwyrm.settings import DOMAIN, BASE_URL
 
 
 def email_data():
@@ -14,6 +14,7 @@ def email_data():
         "site_name": site.name,
         "logo": site.logo_small_url,
         "domain": DOMAIN,
+        "base_url": BASE_URL,
         "user": None,
     }
 
@@ -75,7 +76,7 @@ def format_email(email_name, data):
     return (subject, html_content, text_content)
 
 
-@app.task(queue=HIGH)
+@app.task(queue=EMAIL)
 def send_email(recipient, subject, html_content, text_content):
     """use a task to send the email"""
     email = EmailMultiAlternatives(

bookwyrm/forms/author.py

@@ -15,6 +15,7 @@ class AuthorForm(CustomForm):
             "aliases",
             "bio",
             "wikipedia_link",
+            "wikidata",
             "website",
             "born",
             "died",
@@ -32,6 +33,7 @@ class AuthorForm(CustomForm):
             "wikipedia_link": forms.TextInput(
                 attrs={"aria-describedby": "desc_wikipedia_link"}
             ),
+            "wikidata": forms.TextInput(attrs={"aria-describedby": "desc_wikidata"}),
             "website": forms.TextInput(attrs={"aria-describedby": "desc_website"}),
             "born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}),
             "died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}),

bookwyrm/forms/books.py

@@ -1,8 +1,9 @@
 """ using django model forms """
 from django import forms
 
+from file_resubmit.widgets import ResubmitImageWidget
+
 from bookwyrm import models
-from bookwyrm.models.fields import ClearableFileInputWithWarning
 from .custom_form import CustomForm
 from .widgets import ArrayWidget, SelectDateWidget, Select
 
@@ -20,6 +21,7 @@ class EditionForm(CustomForm):
         model = models.Edition
         fields = [
             "title",
+            "sort_title",
             "subtitle",
             "description",
             "series",
@@ -45,6 +47,9 @@ class EditionForm(CustomForm):
         ]
         widgets = {
             "title": forms.TextInput(attrs={"aria-describedby": "desc_title"}),
+            "sort_title": forms.TextInput(
+                attrs={"aria-describedby": "desc_sort_title"}
+            ),
             "subtitle": forms.TextInput(attrs={"aria-describedby": "desc_subtitle"}),
             "description": forms.Textarea(
                 attrs={"aria-describedby": "desc_description"}
@@ -66,9 +71,7 @@ class EditionForm(CustomForm):
             "published_date": SelectDateWidget(
                 attrs={"aria-describedby": "desc_published_date"}
             ),
-            "cover": ClearableFileInputWithWarning(
-                attrs={"aria-describedby": "desc_cover"}
-            ),
+            "cover": ResubmitImageWidget(attrs={"aria-describedby": "desc_cover"}),
             "physical_format": Select(
                 attrs={"aria-describedby": "desc_physical_format"}
             ),
@@ -107,6 +110,7 @@ class EditionFromWorkForm(CustomForm):
         model = models.Work
         fields = [
             "title",
+            "sort_title",
             "subtitle",
             "authors",
             "description",

bookwyrm/forms/custom_form.py

@@ -15,9 +15,9 @@ class StyledForm(ModelForm):
         css_classes["number"] = "input"
         css_classes["checkbox"] = "checkbox"
         css_classes["textarea"] = "textarea"
-        # pylint: disable=super-with-arguments
         super().__init__(*args, **kwargs)
         for visible in self.visible_fields():
+            input_type = ""
             if hasattr(visible.field.widget, "input_type"):
                 input_type = visible.field.widget.input_type
             if isinstance(visible.field.widget, Textarea):

bookwyrm/forms/edit_user.py

@@ -18,6 +18,7 @@ class EditUserForm(CustomForm):
             "email",
             "summary",
             "show_goal",
+            "show_ratings",
             "show_suggested_users",
             "manually_approves_followers",
             "default_post_privacy",
@@ -70,6 +71,22 @@ class DeleteUserForm(CustomForm):
         fields = ["password"]
 
 
+class MoveUserForm(CustomForm):
+    target = forms.CharField(widget=forms.TextInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+
+
+class AliasUserForm(CustomForm):
+    username = forms.CharField(widget=forms.TextInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+
+
 class ChangePasswordForm(CustomForm):
     current_password = forms.CharField(widget=forms.PasswordInput)
     confirm_password = forms.CharField(widget=forms.PasswordInput)

bookwyrm/forms/forms.py

@@ -25,6 +25,10 @@ class ImportForm(forms.Form):
     csv_file = forms.FileField()
 
 
+class ImportUserForm(forms.Form):
+    archive_file = forms.FileField()
+
+
 class ShelfForm(CustomForm):
     class Meta:
         model = models.Shelf

bookwyrm/forms/landing.py

@@ -34,7 +34,6 @@ class LoginForm(CustomForm):
 
     def add_invalid_password_error(self):
         """We don't want to be too specific about this"""
-        # pylint: disable=attribute-defined-outside-init
         self.non_field_errors = _("Username or password are incorrect")
 
 

bookwyrm/forms/links.py

@@ -1,4 +1,5 @@
 """ using django model forms """
+
 from urllib.parse import urlparse
 
 from django.utils.translation import gettext_lazy as _
@@ -25,7 +26,7 @@ class FileLinkForm(CustomForm):
         url = cleaned_data.get("url")
         filetype = cleaned_data.get("filetype")
         book = cleaned_data.get("book")
-        domain = urlparse(url).netloc
+        domain = urlparse(url).hostname
         if models.LinkDomain.objects.filter(domain=domain).exists():
             status = models.LinkDomain.objects.get(domain=domain).status
             if status == "blocked":
@@ -37,10 +38,9 @@ class FileLinkForm(CustomForm):
                     ),
                 )
         if (
-            not self.instance
-            and models.FileLink.objects.filter(
-                url=url, book=book, filetype=filetype
-            ).exists()
+            models.FileLink.objects.filter(url=url, book=book, filetype=filetype)
+            .exclude(pk=self.instance)
+            .exists()
         ):
             # pylint: disable=line-too-long
             self.add_error(

bookwyrm/forms/lists.py

@@ -24,7 +24,7 @@ class SortListForm(forms.Form):
     sort_by = ChoiceField(
         choices=(
             ("order", _("List Order")),
-            ("title", _("Book Title")),
+            ("sort_title", _("Book Title")),
             ("rating", _("Rating")),
         ),
         label=_("Sort By"),

bookwyrm/forms/widgets.py

@@ -5,8 +5,6 @@ from django import forms
 class ArrayWidget(forms.widgets.TextInput):
     """Inputs for postgres array fields"""
 
-    # pylint: disable=unused-argument
-    # pylint: disable=no-self-use
     def value_from_datadict(self, data, files, name):
         """get all values for this name"""
         return [i for i in data.getlist(name) if i]

bookwyrm/importers/__init__.py

@@ -1,6 +1,7 @@
 """ import classes """
 
 from .importer import Importer
+from .bookwyrm_import import BookwyrmImporter, BookwyrmBooksImporter
 from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter

bookwyrm/importers/bookwyrm_import.py

@@ -0,0 +1,39 @@
+"""Import data from Bookwyrm export files"""
+from django.http import QueryDict
+
+from bookwyrm.models import User
+from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
+from . import Importer
+
+
+class BookwyrmImporter:
+    """Import a Bookwyrm User export file.
+    This is kind of a combination of an importer and a connector.
+    """
+
+    # pylint: disable=no-self-use
+    def process_import(
+        self, user: User, archive_file: bytes, settings: QueryDict
+    ) -> BookwyrmImportJob:
+        """import user data from a Bookwyrm export file"""
+
+        required = [k for k in settings if settings.get(k) == "on"]
+
+        job = BookwyrmImportJob.objects.create(
+            user=user, archive_file=archive_file, required=required
+        )
+        return job
+
+
+class BookwyrmBooksImporter(Importer):
+    """
+    Handle reading a csv from BookWyrm.
+    Goodreads is the default importer, we basically just use the same structure
+    But BookWyrm has additional attributes in the csv
+    """
+
+    service = "BookWyrm"
+    row_mappings_guesses = Importer.row_mappings_guesses + [
+        ("shelf_name", ["shelf_name"]),
+        ("review_published", ["review_published"]),
+    ]

bookwyrm/importers/calibre_import.py

@@ -1,4 +1,6 @@
 """ handle reading a csv from calibre """
+from typing import Any, Optional
+
 from bookwyrm.models import Shelf
 
 from . import Importer
@@ -9,20 +11,15 @@ class CalibreImporter(Importer):
 
     service = "Calibre"
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         # Add timestamp to row_mappings_guesses for date_added to avoid
         # integrity error
-        row_mappings_guesses = []
-
-        for field, mapping in self.row_mappings_guesses:
-            if field in ("date_added",):
-                row_mappings_guesses.append((field, mapping + ["timestamp"]))
-            else:
-                row_mappings_guesses.append((field, mapping))
-
-        self.row_mappings_guesses = row_mappings_guesses
+        self.row_mappings_guesses = [
+            (field, mapping + (["timestamp"] if field == "date_added" else []))
+            for field, mapping in self.row_mappings_guesses
+        ]
         super().__init__(*args, **kwargs)
 
-    def get_shelf(self, normalized_row):
+    def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         # Calibre export does not indicate which shelf to use. Use a default one for now
         return Shelf.TO_READ

bookwyrm/importers/importer.py

@@ -1,8 +1,10 @@
 """ handle reading a csv from an external service, defaults are from Goodreads """
 import csv
 from datetime import timedelta
+from typing import Iterable, Optional
+
 from django.utils import timezone
-from bookwyrm.models import ImportJob, ImportItem, SiteSettings
+from bookwyrm.models import ImportJob, ImportItem, SiteSettings, User
 
 
 class Importer:
@@ -16,17 +18,26 @@ class Importer:
     row_mappings_guesses = [
         ("id", ["id", "book id"]),
         ("title", ["title"]),
-        ("authors", ["author", "authors", "primary author"]),
-        ("isbn_10", ["isbn10", "isbn", "isbn/uid"]),
-        ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]),
+        ("authors", ["author_text", "author", "authors", "primary author"]),
+        ("isbn_10", ["isbn_10", "isbn10", "isbn", "isbn/uid"]),
+        ("isbn_13", ["isbn_13", "isbn13", "isbn", "isbns", "isbn/uid"]),
         ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]),
-        ("review_name", ["review name"]),
-        ("review_body", ["my review", "review"]),
+        ("review_name", ["review_name", "review name"]),
+        ("review_body", ["review_content", "my review", "review"]),
         ("rating", ["my rating", "rating", "star rating"]),
-        ("date_added", ["date added", "entry date", "added"]),
-        ("date_started", ["date started", "started"]),
-        ("date_finished", ["date finished", "last date read", "date read", "finished"]),
+        (
+            "date_added",
+            ["shelf_date", "date_added", "date added", "entry date", "added"],
+        ),
+        ("date_started", ["start_date", "date started", "started"]),
+        (
+            "date_finished",
+            ["finish_date", "date finished", "last date read", "date read", "finished"],
+        ),
     ]
+
+    # TODO: stopped
+
     date_fields = ["date_added", "date_started", "date_finished"]
     shelf_mapping_guesses = {
         "to-read": ["to-read", "want to read"],
@@ -34,20 +45,33 @@ class Importer:
         "reading": ["currently-reading", "reading", "currently reading"],
     }
 
-    # pylint: disable=too-many-locals
-    def create_job(self, user, csv_file, include_reviews, privacy):
+    # pylint: disable=too-many-arguments
+    def create_job(
+        self,
+        user: User,
+        csv_file: Iterable[str],
+        include_reviews: bool,
+        privacy: str,
+        create_shelves: bool = True,
+    ) -> ImportJob:
         """check over a csv and creates a database entry for the job"""
         csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
         rows = list(csv_reader)
         if len(rows) < 1:
             raise ValueError("CSV file is empty")
-        rows = enumerate(rows)
+
+        mappings = (
+            self.create_row_mappings(list(fieldnames))
+            if (fieldnames := csv_reader.fieldnames)
+            else {}
+        )
 
         job = ImportJob.objects.create(
             user=user,
             include_reviews=include_reviews,
+            create_shelves=create_shelves,
             privacy=privacy,
-            mappings=self.create_row_mappings(csv_reader.fieldnames),
+            mappings=mappings,
             source=self.service,
         )
 
@@ -55,16 +79,20 @@ class Importer:
         if enforce_limit and allowed_imports <= 0:
             job.complete_job()
             return job
-        for index, entry in rows:
+        for index, entry in enumerate(rows):
             if enforce_limit and index >= allowed_imports:
                 break
             self.create_item(job, index, entry)
         return job
 
-    def update_legacy_job(self, job):
+    def update_legacy_job(self, job: ImportJob) -> None:
         """patch up a job that was in the old format"""
         items = job.items
-        headers = list(items.first().data.keys())
+        first_item = items.first()
+        if first_item is None:
+            return
+
+        headers = list(first_item.data.keys())
         job.mappings = self.create_row_mappings(headers)
         job.updated_date = timezone.now()
         job.save()
@@ -75,24 +103,24 @@ class Importer:
             item.normalized_data = normalized
             item.save()
 
-    def create_row_mappings(self, headers):
+    def create_row_mappings(self, headers: list[str]) -> dict[str, Optional[str]]:
         """guess what the headers mean"""
         mappings = {}
         for (key, guesses) in self.row_mappings_guesses:
-            value = [h for h in headers if h.lower() in guesses]
-            value = value[0] if len(value) else None
+            values = [h for h in headers if h.lower() in guesses]
+            value = values[0] if len(values) else None
             if value:
                 headers.remove(value)
             mappings[key] = value
         return mappings
 
-    def create_item(self, job, index, data):
+    def create_item(self, job: ImportJob, index: int, data: dict[str, str]) -> None:
         """creates and saves an import item"""
         normalized = self.normalize_row(data, job.mappings)
         normalized["shelf"] = self.get_shelf(normalized)
         ImportItem(job=job, index=index, data=data, normalized_data=normalized).save()
 
-    def get_shelf(self, normalized_row):
+    def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         """determine which shelf to use"""
         shelf_name = normalized_row.get("shelf")
         if not shelf_name:
@@ -101,13 +129,17 @@ class Importer:
         shelf = [
             s for (s, gs) in self.shelf_mapping_guesses.items() if shelf_name in gs
         ]
-        return shelf[0] if shelf else None
+        return shelf[0] if shelf else normalized_row.get("shelf") or None
 
-    def normalize_row(self, entry, mappings):  # pylint: disable=no-self-use
+    # pylint: disable=no-self-use
+    def normalize_row(
+        self, entry: dict[str, str], mappings: dict[str, Optional[str]]
+    ) -> dict[str, Optional[str]]:
         """use the dataclass to create the formatted row of data"""
-        return {k: entry.get(v) for k, v in mappings.items()}
+        return {k: entry.get(v) if v else None for k, v in mappings.items()}
 
-    def get_import_limit(self, user):  # pylint: disable=no-self-use
+    # pylint: disable=no-self-use
+    def get_import_limit(self, user: User) -> tuple[int, int]:
         """check if import limit is set and return how many imports are left"""
         site_settings = SiteSettings.objects.get()
         import_size_limit = site_settings.import_size_limit
@@ -125,11 +157,14 @@ class Importer:
             allowed_imports = import_size_limit - imported_books
         return enforce_limit, allowed_imports
 
-    def create_retry_job(self, user, original_job, items):
+    def create_retry_job(
+        self, user: User, original_job: ImportJob, items: list[ImportItem]
+    ) -> ImportJob:
         """retry items that didn't import"""
         job = ImportJob.objects.create(
             user=user,
             include_reviews=original_job.include_reviews,
+            create_shelves=original_job.create_shelves,
             privacy=original_job.privacy,
             source=original_job.source,
             # TODO: allow users to adjust mappings

bookwyrm/importers/librarything_import.py

@@ -1,11 +1,16 @@
 """ handle reading a tsv from librarything """
 import re
+from typing import Optional
 
 from bookwyrm.models import Shelf
 
 from . import Importer
 
 
+def _remove_brackets(value: Optional[str]) -> Optional[str]:
+    return re.sub(r"\[|\]", "", value) if value else None
+
+
 class LibrarythingImporter(Importer):
     """csv downloads from librarything"""
 
@@ -13,16 +18,19 @@ class LibrarythingImporter(Importer):
     delimiter = "\t"
     encoding = "ISO-8859-1"
 
-    def normalize_row(self, entry, mappings):  # pylint: disable=no-self-use
+    def normalize_row(
+        self, entry: dict[str, str], mappings: dict[str, Optional[str]]
+    ) -> dict[str, Optional[str]]:
         """use the dataclass to create the formatted row of data"""
-        remove_brackets = lambda v: re.sub(r"\[|\]", "", v) if v else None
-        normalized = {k: remove_brackets(entry.get(v)) for k, v in mappings.items()}
-        isbn_13 = normalized.get("isbn_13")
-        isbn_13 = isbn_13.split(", ") if isbn_13 else []
+        normalized = {
+            k: _remove_brackets(entry.get(v) if v else None)
+            for k, v in mappings.items()
+        }
+        isbn_13 = value.split(", ") if (value := normalized.get("isbn_13")) else []
         normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 1 else None
         return normalized
 
-    def get_shelf(self, normalized_row):
+    def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         if normalized_row["date_finished"]:
             return Shelf.READ_FINISHED
         if normalized_row["date_started"]:

bookwyrm/importers/openlibrary_import.py

@@ -1,4 +1,6 @@
 """ handle reading a csv from openlibrary"""
+from typing import Any
+
 from . import Importer
 
 
@@ -7,7 +9,7 @@ class OpenLibraryImporter(Importer):
 
     service = "OpenLibrary"
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         self.row_mappings_guesses.append(("openlibrary_key", ["edition id"]))
         self.row_mappings_guesses.append(("openlibrary_work_key", ["work id"]))
         super().__init__(*args, **kwargs)

bookwyrm/isbn/isbn.py

@@ -0,0 +1,128 @@
+""" Use the range message from isbn-international to hyphenate ISBNs """
+import os
+from typing import Optional
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+
+import requests
+
+from bookwyrm import settings
+
+
+def _get_rules(element: Element) -> list[Element]:
+    if (rules_el := element.find("Rules")) is not None:
+        return rules_el.findall("Rule")
+    return []
+
+
+class IsbnHyphenator:
+    """Class to manage the range message xml file and use it to hyphenate ISBNs"""
+
+    __range_message_url = "https://www.isbn-international.org/export_rangemessage.xml"
+    __range_file_path = os.path.join(
+        settings.BASE_DIR, "bookwyrm", "isbn", "RangeMessage.xml"
+    )
+    __element_tree = None
+
+    def update_range_message(self) -> None:
+        """Download the range message xml file and save it locally"""
+        response = requests.get(self.__range_message_url, timeout=15)
+        with open(self.__range_file_path, "w", encoding="utf-8") as file:
+            file.write(response.text)
+        self.__element_tree = None
+
+    def hyphenate(self, isbn_13: Optional[str]) -> Optional[str]:
+        """hyphenate the given ISBN-13 number using the range message"""
+        if isbn_13 is None:
+            return None
+
+        if self.__element_tree is None:
+            self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+        gs1_prefix = isbn_13[:3]
+        try:
+            reg_group = self.__find_reg_group(isbn_13, gs1_prefix)
+        except ValueError:
+            # if the reg groups are invalid, just return the original isbn
+            return isbn_13
+
+        if reg_group is None:
+            return isbn_13  # failed to hyphenate
+
+        registrant = self.__find_registrant(isbn_13, gs1_prefix, reg_group)
+        if registrant is None:
+            return isbn_13  # failed to hyphenate
+
+        publication = isbn_13[len(gs1_prefix) + len(reg_group) + len(registrant) : -1]
+        check_digit = isbn_13[-1:]
+        return "-".join((gs1_prefix, reg_group, registrant, publication, check_digit))
+
+    def __find_reg_group(self, isbn_13: str, gs1_prefix: str) -> Optional[str]:
+        if self.__element_tree is None:
+            self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+        ucc_prefixes_el = self.__element_tree.find("EAN.UCCPrefixes")
+        if ucc_prefixes_el is None:
+            return None
+
+        for ean_ucc_el in ucc_prefixes_el.findall("EAN.UCC"):
+            if (
+                prefix_el := ean_ucc_el.find("Prefix")
+            ) is not None and prefix_el.text == gs1_prefix:
+                for rule_el in _get_rules(ean_ucc_el):
+                    length_el = rule_el.find("Length")
+                    if length_el is None:
+                        continue
+                    length = int(text) if (text := length_el.text) else 0
+                    if length == 0:
+                        continue
+
+                    range_el = rule_el.find("Range")
+                    if range_el is None or range_el.text is None:
+                        continue
+
+                    reg_grp_range = [int(x[:length]) for x in range_el.text.split("-")]
+                    reg_group = isbn_13[len(gs1_prefix) : len(gs1_prefix) + length]
+                    if reg_grp_range[0] <= int(reg_group) <= reg_grp_range[1]:
+                        return reg_group
+                return None
+        return None
+
+    def __find_registrant(
+        self, isbn_13: str, gs1_prefix: str, reg_group: str
+    ) -> Optional[str]:
+        from_ind = len(gs1_prefix) + len(reg_group)
+
+        if self.__element_tree is None:
+            self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+        reg_groups_el = self.__element_tree.find("RegistrationGroups")
+        if reg_groups_el is None:
+            return None
+
+        for group_el in reg_groups_el.findall("Group"):
+            if (
+                prefix_el := group_el.find("Prefix")
+            ) is not None and prefix_el.text == "-".join((gs1_prefix, reg_group)):
+                for rule_el in _get_rules(group_el):
+                    length_el = rule_el.find("Length")
+                    if length_el is None:
+                        continue
+                    length = int(text) if (text := length_el.text) else 0
+                    if length == 0:
+                        continue
+
+                    range_el = rule_el.find("Range")
+                    if range_el is None or range_el.text is None:
+                        continue
+                    registrant_range = [
+                        int(x[:length]) for x in range_el.text.split("-")
+                    ]
+                    registrant = isbn_13[from_ind : from_ind + length]
+                    if registrant_range[0] <= int(registrant) <= registrant_range[1]:
+                        return registrant
+                return None
+        return None
+
+
+hyphenator_singleton = IsbnHyphenator()

bookwyrm/lists_stream.py

@@ -5,7 +5,7 @@ from django.db.models import signals, Count, Q
 
 from bookwyrm import models
 from bookwyrm.redis_store import RedisStore
-from bookwyrm.tasks import app, MEDIUM, HIGH
+from bookwyrm.tasks import app, LISTS
 
 
 class ListsStream(RedisStore):
@@ -18,7 +18,7 @@ class ListsStream(RedisStore):
             return f"{user}-lists"
         return f"{user.id}-lists"
 
-    def get_rank(self, obj):  # pylint: disable=no-self-use
+    def get_rank(self, obj):
         """lists are sorted by updated date"""
         return obj.updated_date.timestamp()
 
@@ -217,14 +217,14 @@ def add_list_on_account_create_command(user_id):
 
 
 # ---- TASKS
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def populate_lists_task(user_id):
     """background task for populating an empty list stream"""
     user = models.User.objects.get(id=user_id)
     ListsStream().populate_lists(user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def remove_list_task(list_id, re_add=False):
     """remove a list from any stream it might be in"""
     stores = models.User.objects.filter(local=True, is_active=True).values_list(
@@ -239,14 +239,14 @@ def remove_list_task(list_id, re_add=False):
         add_list_task.delay(list_id)
 
 
-@app.task(queue=HIGH)
+@app.task(queue=LISTS)
 def add_list_task(list_id):
     """add a list to any stream it should be in"""
     book_list = models.List.objects.get(id=list_id)
     ListsStream().add_list(book_list)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
     """remove all lists by a user from a viewer's stream"""
     viewer = models.User.objects.get(id=viewer_id)
@@ -254,7 +254,7 @@ def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
     ListsStream().remove_user_lists(viewer, user, exclude_privacy=exclude_privacy)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def add_user_lists_task(viewer_id, user_id):
     """add all lists by a user to a viewer's stream"""
     viewer = models.User.objects.get(id=viewer_id)

bookwyrm/management/commands/deduplicate_book_data.py

@@ -1,13 +1,14 @@
 """ PROCEED WITH CAUTION: uses deduplication fields to permanently
 merge book data objects """
+
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from bookwyrm import models
-from bookwyrm.management.merge import merge_objects
 
 
-def dedupe_model(model):
+def dedupe_model(model, dry_run=False):
     """combine duplicate editions and update related models"""
+    print(f"deduplicating {model.__name__}:")
     fields = model._meta.get_fields()
     dedupe_fields = [
         f for f in fields if hasattr(f, "deduplication_field") and f.deduplication_field
@@ -16,30 +17,42 @@ def dedupe_model(model):
         dupes = (
             model.objects.values(field.name)
             .annotate(Count(field.name))
-            .filter(**{"%s__count__gt" % field.name: 1})
+            .filter(**{f"{field.name}__count__gt": 1})
+            .exclude(**{field.name: ""})
+            .exclude(**{f"{field.name}__isnull": True})
         )
 
         for dupe in dupes:
             value = dupe[field.name]
-            if not value or value == "":
-                continue
             print("----------")
-            print(dupe)
             objs = model.objects.filter(**{field.name: value}).order_by("id")
             canonical = objs.first()
-            print("keeping", canonical.remote_id)
+            action = "would merge" if dry_run else "merging"
+            print(
+                f"{action} into {model.__name__} {canonical.remote_id} based on {field.name} {value}:"
+            )
             for obj in objs[1:]:
-                print(obj.remote_id)
-                merge_objects(canonical, obj)
+                print(f"- {obj.remote_id}")
+                absorbed_fields = obj.merge_into(canonical, dry_run=dry_run)
+                print(f"  absorbed fields: {absorbed_fields}")
 
 
 class Command(BaseCommand):
     """deduplicate allllll the book data models"""
 
     help = "merges duplicate book data"
+
+    def add_arguments(self, parser):
+        """add the arguments for this command"""
+        parser.add_argument(
+            "--dry_run",
+            action="store_true",
+            help="don't actually merge, only print what would happen",
+        )
+
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
         """run deduplications"""
-        dedupe_model(models.Edition)
-        dedupe_model(models.Work)
-        dedupe_model(models.Author)
+        dedupe_model(models.Edition, dry_run=options["dry_run"])
+        dedupe_model(models.Work, dry_run=options["dry_run"])
+        dedupe_model(models.Author, dry_run=options["dry_run"])

bookwyrm/management/commands/erase_deleted_user_data.py

@@ -0,0 +1,43 @@
+""" Erase any data stored about deleted users """
+import sys
+from django.core.management.base import BaseCommand, CommandError
+from bookwyrm import models
+from bookwyrm.models.user import erase_user_data
+
+# pylint: disable=missing-function-docstring
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Remove Two Factor Authorisation from user"
+
+    def add_arguments(self, parser):  # pylint: disable=no-self-use
+        parser.add_argument(
+            "--dryrun",
+            action="store_true",
+            help="Preview users to be cleared without altering the database",
+        )
+
+    def handle(self, *args, **options):  # pylint: disable=unused-argument
+
+        # Check for anything fishy
+        bad_state = models.User.objects.filter(is_deleted=True, is_active=True)
+        if bad_state.exists():
+            raise CommandError(
+                f"{bad_state.count()} user(s) marked as both active and deleted"
+            )
+
+        deleted_users = models.User.objects.filter(is_deleted=True)
+        self.stdout.write(f"Found {deleted_users.count()} deleted users")
+        if options["dryrun"]:
+            self.stdout.write("\n".join(u.username for u in deleted_users[:5]))
+            if deleted_users.count() > 5:
+                self.stdout.write("... and more")
+            sys.exit()
+
+        self.stdout.write("Erasing user data:")
+        for user_id in deleted_users.values_list("id", flat=True):
+            erase_user_data.delay(user_id)
+            self.stdout.write(".", ending="")
+
+        self.stdout.write("")
+        self.stdout.write("Tasks created successfully")

bookwyrm/management/commands/instance_version.py

@@ -1,54 +0,0 @@
-""" Get your admin code to allow install """
-from django.core.management.base import BaseCommand
-
-from bookwyrm import models
-from bookwyrm.settings import VERSION
-
-
-# pylint: disable=no-self-use
-class Command(BaseCommand):
-    """command-line options"""
-
-    help = "What version is this?"
-
-    def add_arguments(self, parser):
-        """specify which function to run"""
-        parser.add_argument(
-            "--current",
-            action="store_true",
-            help="Version stored in database",
-        )
-        parser.add_argument(
-            "--target",
-            action="store_true",
-            help="Version stored in settings",
-        )
-        parser.add_argument(
-            "--update",
-            action="store_true",
-            help="Update database version",
-        )
-
-    # pylint: disable=unused-argument
-    def handle(self, *args, **options):
-        """execute init"""
-        site = models.SiteSettings.objects.get()
-        current = site.version or "0.0.1"
-        target = VERSION
-        if options.get("current"):
-            print(current)
-            return
-
-        if options.get("target"):
-            print(target)
-            return
-
-        if options.get("update"):
-            site.version = target
-            site.save()
-            return
-
-        if current != target:
-            print(f"{current}/{target}")
-        else:
-            print(current)

bookwyrm/management/commands/repair_editions.py

@@ -0,0 +1,21 @@
+""" Repair editions with missing works """
+from django.core.management.base import BaseCommand
+from bookwyrm import models
+
+
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Repairs an edition that is in a broken state"
+
+    # pylint: disable=unused-argument
+    def handle(self, *args, **options):
+        """Find and repair broken editions"""
+        # Find broken editions
+        editions = models.Edition.objects.filter(parent_work__isnull=True)
+        self.stdout.write(f"Repairing {editions.count()} edition(s):")
+
+        # Do repair
+        for edition in editions:
+            edition.repair()
+            self.stdout.write(".", ending="")

bookwyrm/management/merge.py

@@ -1,50 +0,0 @@
-from django.db.models import ManyToManyField
-
-
-def update_related(canonical, obj):
-    """update all the models with fk to the object being removed"""
-    # move related models to canonical
-    related_models = [
-        (r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
-    ]
-    for (related_field, related_model) in related_models:
-        # Skip the ManyToMany fields that aren’t auto-created. These
-        # should have a corresponding OneToMany field in the model for
-        # the linking table anyway. If we update it through that model
-        # instead then we won’t lose the extra fields in the linking
-        # table.
-        related_field_obj = related_model._meta.get_field(related_field)
-        if isinstance(related_field_obj, ManyToManyField):
-            through = related_field_obj.remote_field.through
-            if not through._meta.auto_created:
-                continue
-        related_objs = related_model.objects.filter(**{related_field: obj})
-        for related_obj in related_objs:
-            print("replacing in", related_model.__name__, related_field, related_obj.id)
-            try:
-                setattr(related_obj, related_field, canonical)
-                related_obj.save()
-            except TypeError:
-                getattr(related_obj, related_field).add(canonical)
-                getattr(related_obj, related_field).remove(obj)
-
-
-def copy_data(canonical, obj):
-    """try to get the most data possible"""
-    for data_field in obj._meta.get_fields():
-        if not hasattr(data_field, "activitypub_field"):
-            continue
-        data_value = getattr(obj, data_field.name)
-        if not data_value:
-            continue
-        if not getattr(canonical, data_field.name):
-            print("setting data field", data_field.name, data_value)
-            setattr(canonical, data_field.name, data_value)
-    canonical.save()
-
-
-def merge_objects(canonical, obj):
-    copy_data(canonical, obj)
-    update_related(canonical, obj)
-    # remove the outdated entry
-    obj.delete()

bookwyrm/management/merge_command.py

@@ -1,4 +1,3 @@
-from bookwyrm.management.merge import merge_objects
 from django.core.management.base import BaseCommand
 
 
@@ -9,6 +8,11 @@ class MergeCommand(BaseCommand):
         """add the arguments for this command"""
         parser.add_argument("--canonical", type=int, required=True)
         parser.add_argument("--other", type=int, required=True)
+        parser.add_argument(
+            "--dry_run",
+            action="store_true",
+            help="don't actually merge, only print what would happen",
+        )
 
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
@@ -26,4 +30,8 @@ class MergeCommand(BaseCommand):
             print("other book doesn’t exist!")
             return
 
-        merge_objects(canonical, other)
+        absorbed_fields = other.merge_into(canonical, dry_run=options["dry_run"])
+
+        action = "would be" if options["dry_run"] else "has been"
+        print(f"{other.remote_id} {action} merged into {canonical.remote_id}")
+        print(f"absorbed fields: {absorbed_fields}")

bookwyrm/middleware/__init__.py

@@ -1,3 +1,4 @@
 """ look at all this nice middleware! """
 from .timezone_middleware import TimezoneMiddleware
 from .ip_middleware import IPBlocklistMiddleware
+from .file_too_big import FileTooBig

bookwyrm/middleware/file_too_big.py

@@ -0,0 +1,30 @@
+"""Middleware to display a custom 413 error page"""
+
+from django.http import HttpResponse
+from django.shortcuts import render
+from django.core.exceptions import RequestDataTooBig
+
+
+class FileTooBig:
+    """Middleware to display a custom page when a
+    RequestDataTooBig exception is thrown"""
+
+    def __init__(self, get_response):
+        """boilerplate __init__ from Django docs"""
+
+        self.get_response = get_response
+
+    def __call__(self, request):
+        """If RequestDataTooBig is thrown, render the 413 error page"""
+
+        try:
+            body = request.body  # pylint: disable=unused-variable
+
+        except RequestDataTooBig:
+
+            rendered = render(request, "413.html")
+            response = HttpResponse(rendered)
+            return response
+
+        response = self.get_response(request)
+        return response

bookwyrm/middleware/timezone_middleware.py

@@ -1,5 +1,5 @@
 """ Makes the app aware of the users timezone """
-import pytz
+import zoneinfo
 
 from django.utils import timezone
 
@@ -12,9 +12,7 @@ class TimezoneMiddleware:
 
     def __call__(self, request):
         if request.user.is_authenticated:
-            timezone.activate(pytz.timezone(request.user.preferred_timezone))
+            timezone.activate(zoneinfo.ZoneInfo(request.user.preferred_timezone))
         else:
-            timezone.activate(pytz.utc)
-        response = self.get_response(request)
             timezone.deactivate()
-        return response
+        return self.get_response(request)

bookwyrm/migrations/0171_alter_user_preferred_timezone.py

@@ -10,6 +10,7 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        # The new timezones are "Factory" and "localtime"
         migrations.AlterField(
             model_name="user",
             name="preferred_timezone",

bookwyrm/migrations/0179_populate_sort_title.py

@@ -0,0 +1,51 @@
+import re
+from itertools import chain
+
+from django.db import migrations, transaction
+from django.db.models import Q
+
+from bookwyrm.settings import LANGUAGE_ARTICLES
+
+
+def set_sort_title(edition):
+    articles = chain(
+        *(LANGUAGE_ARTICLES.get(language, ()) for language in tuple(edition.languages))
+    )
+    edition.sort_title = re.sub(
+        f'^{" |^".join(articles)} ', "", str(edition.title).lower()
+    )
+    return edition
+
+
+@transaction.atomic
+def populate_sort_title(apps, schema_editor):
+    Edition = apps.get_model("bookwyrm", "Edition")
+    db_alias = schema_editor.connection.alias
+    editions_wo_sort_title = Edition.objects.using(db_alias).filter(
+        Q(sort_title__isnull=True) | Q(sort_title__exact="")
+    )
+    batch_size = 1000
+    start = 0
+    end = batch_size
+    while True:
+        batch = editions_wo_sort_title[start:end]
+        if not batch.exists():
+            break
+        Edition.objects.bulk_update(
+            (set_sort_title(edition) for edition in batch), ["sort_title"]
+        )
+        start = end
+        end += batch_size
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0178_auto_20230328_2132"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            populate_sort_title, reverse_code=migrations.RunPython.noop
+        ),
+    ]

bookwyrm/migrations/0179_reportcomment_comment_type.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.2.18 on 2023-05-16 16:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0178_auto_20230328_2132"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="reportcomment",
+            name="action_type",
+            field=models.CharField(
+                choices=[
+                    ("comment", "Comment"),
+                    ("resolve", "Resolved report"),
+                    ("reopen", "Re-opened report"),
+                    ("message_reporter", "Messaged reporter"),
+                    ("message_offender", "Messaged reported user"),
+                    ("user_suspension", "Suspended user"),
+                    ("user_unsuspension", "Un-suspended user"),
+                    ("user_perms", "Changed user permission level"),
+                    ("user_deletion", "Deleted user account"),
+                    ("block_domain", "Blocked domain"),
+                    ("approve_domain", "Approved domain"),
+                    ("delete_item", "Deleted item"),
+                ],
+                default="comment",
+                max_length=20,
+            ),
+        ),
+        migrations.RenameModel("ReportComment", "ReportAction"),
+    ]

bookwyrm/migrations/0180_alter_reportaction_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.18 on 2023-06-21 22:01
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0179_reportcomment_comment_type"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="reportaction",
+            options={"ordering": ("created_date",)},
+        ),
+    ]

bookwyrm/migrations/0180_alter_user_preferred_language.py

@@ -0,0 +1,44 @@
+# Generated by Django 3.2.19 on 2023-07-23 19:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0179_populate_sort_title"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("eo-uy", "Esperanto (Esperanto)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("nl-nl", "Nederlands (Dutch)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0181_merge_20230806_2302.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.20 on 2023-08-06 23:02
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0180_alter_reportaction_options"),
+        ("bookwyrm", "0180_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0182_auto_20231027_1122.py

@@ -0,0 +1,130 @@
+# Generated by Django 3.2.20 on 2023-10-27 11:22
+
+import bookwyrm.models.activitypub_mixin
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0181_merge_20230806_2302"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="also_known_as",
+            field=bookwyrm.models.fields.ManyToManyField(to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name="user",
+            name="moved_to",
+            field=bookwyrm.models.fields.RemoteIdField(
+                max_length=255,
+                null=True,
+                validators=[bookwyrm.models.fields.validate_remote_id],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("BOOST", "Boost"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+        migrations.CreateModel(
+            name="Move",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                ("updated_date", models.DateTimeField(auto_now=True)),
+                (
+                    "remote_id",
+                    bookwyrm.models.fields.RemoteIdField(
+                        max_length=255,
+                        null=True,
+                        validators=[bookwyrm.models.fields.validate_remote_id],
+                    ),
+                ),
+                ("object", bookwyrm.models.fields.CharField(max_length=255)),
+                (
+                    "origin",
+                    bookwyrm.models.fields.CharField(
+                        blank=True, default="", max_length=255, null=True
+                    ),
+                ),
+                (
+                    "user",
+                    bookwyrm.models.fields.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=(bookwyrm.models.activitypub_mixin.ActivityMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name="MoveUser",
+            fields=[
+                (
+                    "move_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.move",
+                    ),
+                ),
+                (
+                    "target",
+                    bookwyrm.models.fields.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="move_target",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.move",),
+        ),
+    ]

bookwyrm/migrations/0183_auto_20231105_1607.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-11-05 16:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0182_auto_20231027_1122"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="is_deleted",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0184_auto_20231106_0421.py

@@ -0,0 +1,35 @@
+# Generated by Django 3.2.20 on 2023-11-06 04:21
+
+from django.db import migrations
+from bookwyrm.models import User
+
+
+def update_deleted_users(apps, schema_editor):
+    """Find all the users who are deleted, not just inactive, and set deleted"""
+    users = apps.get_model("bookwyrm", "User")
+    db_alias = schema_editor.connection.alias
+    users.objects.using(db_alias).filter(
+        is_active=False,
+        deactivation_reason__in=[
+            "self_deletion",
+            "moderator_deletion",
+        ],
+    ).update(is_deleted=True)
+
+    # differente rules for remote users
+    users.objects.using(db_alias).filter(is_active=False, local=False,).exclude(
+        deactivation_reason="moderator_deactivation",
+    ).update(is_deleted=True)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0183_auto_20231105_1607"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            update_deleted_users, reverse_code=migrations.RunPython.noop
+        ),
+    ]

bookwyrm/migrations/0185_alter_notification_notification_type.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.20 on 2023-11-13 22:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0184_auto_20231106_0421"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0186_auto_20231116_0048.py

@@ -0,0 +1,212 @@
+# Generated by Django 3.2.20 on 2023-11-16 00:48
+
+from django.conf import settings
+import django.contrib.postgres.fields
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0185_alter_notification_notification_type"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ParentJob",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("task_id", models.UUIDField(blank=True, null=True, unique=True)),
+                (
+                    "created_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                (
+                    "updated_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("complete", models.BooleanField(default=False)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("pending", "Pending"),
+                            ("active", "Active"),
+                            ("complete", "Complete"),
+                            ("stopped", "Stopped"),
+                            ("failed", "Failed"),
+                        ],
+                        default="pending",
+                        max_length=50,
+                        null=True,
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="user_import_time_limit",
+            field=models.IntegerField(default=48),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("USER_IMPORT", "User Import"),
+                    ("USER_EXPORT", "User Export"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+        migrations.CreateModel(
+            name="BookwyrmExportJob",
+            fields=[
+                (
+                    "parentjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+                ("export_data", models.FileField(null=True, upload_to="")),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.parentjob",),
+        ),
+        migrations.CreateModel(
+            name="BookwyrmImportJob",
+            fields=[
+                (
+                    "parentjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+                ("archive_file", models.FileField(blank=True, null=True, upload_to="")),
+                ("import_data", models.JSONField(null=True)),
+                (
+                    "required",
+                    django.contrib.postgres.fields.ArrayField(
+                        base_field=models.CharField(blank=True, max_length=50),
+                        blank=True,
+                        size=None,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.parentjob",),
+        ),
+        migrations.CreateModel(
+            name="ChildJob",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("task_id", models.UUIDField(blank=True, null=True, unique=True)),
+                (
+                    "created_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                (
+                    "updated_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("complete", models.BooleanField(default=False)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("pending", "Pending"),
+                            ("active", "Active"),
+                            ("complete", "Complete"),
+                            ("stopped", "Stopped"),
+                            ("failed", "Failed"),
+                        ],
+                        default="pending",
+                        max_length=50,
+                        null=True,
+                    ),
+                ),
+                (
+                    "parent_job",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="child_jobs",
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="notification",
+            name="related_user_export",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="bookwyrm.bookwyrmexportjob",
+            ),
+        ),
+    ]

bookwyrm/migrations/0186_invite_request_notification.py

@@ -0,0 +1,48 @@
+# Generated by Django 3.2.20 on 2023-11-14 10:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0185_alter_notification_notification_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="notification",
+            name="related_invite_requests",
+            field=models.ManyToManyField(to="bookwyrm.InviteRequest"),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE_REQUEST", "Invite Request"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0187_partial_publication_dates.py

@@ -0,0 +1,54 @@
+# Generated by Django 3.2.20 on 2023-11-09 16:57
+
+import bookwyrm.models.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0186_invite_request_notification"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="first_published_date_precision",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("DAY", "Day prec."),
+                    ("MONTH", "Month prec."),
+                    ("YEAR", "Year prec."),
+                ],
+                editable=False,
+                max_length=10,
+                null=True,
+            ),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="published_date_precision",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("DAY", "Day prec."),
+                    ("MONTH", "Month prec."),
+                    ("YEAR", "Year prec."),
+                ],
+                editable=False,
+                max_length=10,
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="book",
+            name="first_published_date",
+            field=bookwyrm.models.fields.PartialDateField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="book",
+            name="published_date",
+            field=bookwyrm.models.fields.PartialDateField(blank=True, null=True),
+        ),
+    ]

bookwyrm/migrations/0188_theme_loads.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2023-11-20 18:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0187_partial_publication_dates"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="theme",
+            name="loads",
+            field=models.BooleanField(blank=True, null=True),
+        ),
+    ]

bookwyrm/migrations/0189_alter_user_preferred_language.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.23 on 2023-12-12 23:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("eo-uy", "Esperanto (Esperanto)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("nl-nl", "Nederlands (Dutch)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("uk-ua", "Українська (Ukrainian)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0189_importjob_create_shelves.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2023-11-25 05:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importjob",
+            name="create_shelves",
+            field=models.BooleanField(default=True),
+        ),
+    ]

bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2023-11-22 10:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0186_auto_20231116_0048"),
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = []

bookwyrm/migrations/0190_alter_notification_notification_type.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.23 on 2023-11-23 19:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0189_merge_0186_auto_20231116_0048_0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("USER_IMPORT", "User Import"),
+                    ("USER_EXPORT", "User Export"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE_REQUEST", "Invite Request"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0190_book_search_updates.py

@@ -0,0 +1,16 @@
+# Generated by Django 3.2.20 on 2023-11-24 17:11
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name="author",
+            name="bookwyrm_au_search__b050a8_gin",
+        ),
+    ]

bookwyrm/migrations/0191_merge_20240102_0326.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-01-02 03:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0189_alter_user_preferred_language"),
+        ("bookwyrm", "0190_alter_notification_notification_type"),
+    ]
+
+    operations = []

bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py

@@ -0,0 +1,76 @@
+# Generated by Django 3.2.20 on 2023-11-25 00:47
+
+from importlib import import_module
+import re
+
+from django.db import migrations
+import pgtrigger.compiler
+import pgtrigger.migrations
+
+trigger_migration = import_module("bookwyrm.migrations.0077_auto_20210623_2155")
+
+# it's _very_ convenient for development that this migration be reversible
+search_vector_trigger = trigger_migration.Migration.operations[4]
+author_search_vector_trigger = trigger_migration.Migration.operations[5]
+
+
+assert re.search(r"\bCREATE TRIGGER search_vector_trigger\b", search_vector_trigger.sql)
+assert re.search(
+    r"\bCREATE TRIGGER author_search_vector_trigger\b",
+    author_search_vector_trigger.sql,
+)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("bookwyrm", "0190_book_search_updates"),
+    ]
+
+    operations = [
+        pgtrigger.migrations.AddTrigger(
+            model_name="book",
+            trigger=pgtrigger.compiler.Trigger(
+                name="update_search_vector_on_book_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="new.search_vector := setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(bookwyrm_author.name), ' '), '')), 'C') FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id ) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D');RETURN NEW;",
+                    hash="77d6399497c0a89b0bf09d296e33c396da63705c",
+                    operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"',
+                    pgid="pgtrigger_update_search_vector_on_book_edit_bec58",
+                    table="bookwyrm_book",
+                    when="BEFORE",
+                ),
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="author",
+            trigger=pgtrigger.compiler.Trigger(
+                name="reset_search_vector_on_author_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id ) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;",
+                    hash="e7bbf08711ff3724c58f4d92fb7a082ffb3d7826",
+                    operation='UPDATE OF "name"',
+                    pgid="pgtrigger_reset_search_vector_on_author_edit_a447c",
+                    table="bookwyrm_author",
+                    when="AFTER",
+                ),
+            ),
+        ),
+        migrations.RunSQL(
+            sql="""DROP TRIGGER IF EXISTS search_vector_trigger ON bookwyrm_book;
+                   DROP FUNCTION IF EXISTS book_trigger;
+            """,
+            reverse_sql=search_vector_trigger.sql,
+        ),
+        migrations.RunSQL(
+            sql="""DROP TRIGGER IF EXISTS author_search_vector_trigger ON bookwyrm_author;
+                   DROP FUNCTION IF EXISTS author_trigger;
+            """,
+            reverse_sql=author_search_vector_trigger.sql,
+        ),
+        migrations.RunSQL(
+            # Recalculate book search vector for any missed author name changes
+            # due to bug in JOIN in the old trigger.
+            sql="UPDATE bookwyrm_book SET search_vector = NULL;",
+            reverse_sql=migrations.RunSQL.noop,
+        ),
+    ]

bookwyrm/migrations/0192_make_page_positions_text.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.23 on 2024-01-04 23:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="quotation",
+            name="endposition",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="quotation",
+            name="position",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-02 19:36
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="sitesettings",
+            old_name="version",
+            new_name="available_version",
+        ),
+    ]

bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-16 10:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="user_exports_enabled",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0193_auto_20240128_0249.py

@@ -0,0 +1,92 @@
+# Generated by Django 3.2.23 on 2024-01-28 02:49
+
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+from django.core.files.storage import storages
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0192_sitesettings_user_exports_enabled"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="bookwyrmexportjob",
+            name="export_json",
+            field=models.JSONField(
+                encoder=django.core.serializers.json.DjangoJSONEncoder, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="bookwyrmexportjob",
+            name="json_completed",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name="bookwyrmexportjob",
+            name="export_data",
+            field=models.FileField(
+                null=True,
+                storage=storages["exports"],
+                upload_to="",
+            ),
+        ),
+        migrations.CreateModel(
+            name="AddFileToTar",
+            fields=[
+                (
+                    "childjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.childjob",
+                    ),
+                ),
+                (
+                    "parent_export_job",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="child_edition_export_jobs",
+                        to="bookwyrm.bookwyrmexportjob",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.childjob",),
+        ),
+        migrations.CreateModel(
+            name="AddBookToUserExportJob",
+            fields=[
+                (
+                    "childjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.childjob",
+                    ),
+                ),
+                (
+                    "edition",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="bookwyrm.edition",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.childjob",),
+        ),
+    ]

bookwyrm/migrations/0193_merge_20240203_1539.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-02-03 15:39
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0192_make_page_positions_text"),
+        ("bookwyrm", "0192_sitesettings_user_exports_enabled"),
+    ]
+
+    operations = []

bookwyrm/migrations/0194_merge_20240203_1619.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-02-03 16:19
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0192_rename_version_sitesettings_available_version"),
+        ("bookwyrm", "0193_merge_20240203_1539"),
+    ]
+
+    operations = []

bookwyrm/migrations/0195_alter_user_preferred_language.py

@@ -0,0 +1,46 @@
+# Generated by Django 3.2.23 on 2024-02-21 00:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0194_merge_20240203_1619"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("eo-uy", "Esperanto (Esperanto)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("ko-kr", "한국어 (Korean)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("nl-nl", "Nederlands (Dutch)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("uk-ua", "Українська (Ukrainian)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0196_merge_20240318_1737.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-03-18 17:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0193_auto_20240128_0249"),
+        ("bookwyrm", "0195_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0196_merge_pr3134_into_main.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-03-18 00:48
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_migrate_search_vec_triggers_to_pgtriggers"),
+        ("bookwyrm", "0195_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0197_author_search_vector.py

@@ -0,0 +1,41 @@
+# Generated by Django 3.2.25 on 2024-03-20 15:15
+
+import django.contrib.postgres.indexes
+from django.db import migrations
+import pgtrigger.compiler
+import pgtrigger.migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0196_merge_pr3134_into_main"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="author",
+            index=django.contrib.postgres.indexes.GinIndex(
+                fields=["search_vector"], name="bookwyrm_au_search__b050a8_gin"
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="author",
+            trigger=pgtrigger.compiler.Trigger(
+                name="update_search_vector_on_author_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="new.search_vector := setweight(to_tsvector('simple', new.name), 'A') || setweight(to_tsvector('simple', coalesce(array_to_string(new.aliases, ' '), '')), 'B');RETURN NEW;",
+                    hash="b97919016236d74d0ade51a0769a173ea269da64",
+                    operation='INSERT OR UPDATE OF "name", "aliases", "search_vector"',
+                    pgid="pgtrigger_update_search_vector_on_author_edit_c61cb",
+                    table="bookwyrm_author",
+                    when="BEFORE",
+                ),
+            ),
+        ),
+        migrations.RunSQL(
+            # Calculate search vector for all Authors.
+            sql="UPDATE bookwyrm_author SET search_vector = NULL;",
+            reverse_sql="UPDATE bookwyrm_author SET search_vector = NULL;",
+        ),
+    ]

bookwyrm/migrations/0197_merge_20240324_0235.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-03-24 02:35
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0196_merge_20240318_1737"),
+        ("bookwyrm", "0196_merge_pr3134_into_main"),
+    ]
+
+    operations = []

bookwyrm/migrations/0197_mergedauthor_mergedbook.py

@@ -0,0 +1,48 @@
+# Generated by Django 3.2.24 on 2024-02-28 21:30
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0196_merge_pr3134_into_main"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="MergedBook",
+            fields=[
+                ("deleted_id", models.IntegerField(primary_key=True, serialize=False)),
+                (
+                    "merged_into",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="absorbed",
+                        to="bookwyrm.book",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="MergedAuthor",
+            fields=[
+                ("deleted_id", models.IntegerField(primary_key=True, serialize=False)),
+                (
+                    "merged_into",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="absorbed",
+                        to="bookwyrm.author",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

bookwyrm/migrations/0198_alter_bookwyrmexportjob_export_data.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.25 on 2024-03-26 11:37
+
+import bookwyrm.models.bookwyrm_export_job
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0197_merge_20240324_0235"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="bookwyrmexportjob",
+            name="export_data",
+            field=models.FileField(
+                null=True,
+                storage=bookwyrm.models.bookwyrm_export_job.select_exports_storage,
+                upload_to="",
+            ),
+        ),
+    ]

bookwyrm/migrations/0198_book_search_vector_author_aliases.py

@@ -0,0 +1,57 @@
+# Generated by Django 3.2.25 on 2024-03-20 15:52
+
+from django.db import migrations
+import pgtrigger.compiler
+import pgtrigger.migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0197_author_search_vector"),
+    ]
+
+    operations = [
+        pgtrigger.migrations.RemoveTrigger(
+            model_name="author",
+            name="reset_search_vector_on_author_edit",
+        ),
+        pgtrigger.migrations.RemoveTrigger(
+            model_name="book",
+            name="update_search_vector_on_book_edit",
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="author",
+            trigger=pgtrigger.compiler.Trigger(
+                name="reset_book_search_vector_on_author_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id ) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;",
+                    hash="68422c0f29879c5802b82159dde45297eff53e73",
+                    operation='UPDATE OF "name", "aliases"',
+                    pgid="pgtrigger_reset_book_search_vector_on_author_edit_a50c7",
+                    table="bookwyrm_author",
+                    when="AFTER",
+                ),
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="book",
+            trigger=pgtrigger.compiler.Trigger(
+                name="update_search_vector_on_book_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="WITH author_names AS (SELECT array_to_string(bookwyrm_author.name || bookwyrm_author.aliases, ' ') AS name_and_aliases FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id ) SELECT setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(name_and_aliases), ' '), '')), 'C') FROM author_names) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D') INTO new.search_vector;RETURN NEW;",
+                    hash="9324f5ca76a6f5e63931881d62d11da11f595b2c",
+                    operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"',
+                    pgid="pgtrigger_update_search_vector_on_book_edit_bec58",
+                    table="bookwyrm_book",
+                    when="BEFORE",
+                ),
+            ),
+        ),
+        migrations.RunSQL(
+            # Recalculate search vector for all Books because it now includes
+            # Author aliases.
+            sql="UPDATE bookwyrm_book SET search_vector = NULL;",
+            reverse_sql="UPDATE bookwyrm_book SET search_vector = NULL;",
+        ),
+    ]

bookwyrm/migrations/0199_alter_userblocks_user_object_and_more.py

@@ -0,0 +1,70 @@
+# Generated by Django 4.2.11 on 2024-03-29 19:25
+
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0198_book_search_vector_author_aliases"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="userblocks",
+            name="user_object",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_object",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userblocks",
+            name="user_subject",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_subject",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userfollowrequest",
+            name="user_object",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_object",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userfollowrequest",
+            name="user_subject",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_subject",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userfollows",
+            name="user_object",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_object",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userfollows",
+            name="user_subject",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_subject",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+    ]

bookwyrm/migrations/0199_merge_20240326_1217.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-03-26 12:17
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0198_alter_bookwyrmexportjob_export_data"),
+        ("bookwyrm", "0198_book_search_vector_author_aliases"),
+    ]
+
+    operations = []

bookwyrm/migrations/0199_status_bookwyrm_st_remote__06aeba_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-02 19:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0198_book_search_vector_author_aliases"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="status",
+            index=models.Index(
+                fields=["remote_id"], name="bookwyrm_st_remote__06aeba_idx"
+            ),
+        ),
+    ]