Merge branch 'main' into flowerproxy

Correctly handle serving BookWyrm on custom port

.env.example

@@ -16,6 +16,11 @@ DEFAULT_LANGUAGE="English"
 ## Leave unset to allow all hosts
 # ALLOWED_HOSTS="localhost,127.0.0.1,[::1]"
 
+# Specify when the site is served from a port that is not the default
+# for the protocol (80 for HTTP or 443 for HTTPS).
+# Probably only necessary in development.
+# PORT=1333
+
 MEDIA_ROOT=images/
 
 # Database configuration
@@ -71,14 +76,20 @@ ENABLE_THUMBNAIL_GENERATION=true
 USE_S3=false
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
+# seconds for signed S3 urls to expire
+# this is currently only used for user export files
+S3_SIGNED_URL_EXPIRY=900
 
 # Commented are example values if you use a non-AWS, S3-compatible service
 # AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
 # non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
-# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL.
+# AWS_S3_URL_PROTOCOL must end in ":" and defaults to the same protocol as
+# the BookWyrm instance ("http:" or "https:", based on USE_SSL).
 
 # AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
 # AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_URL_PROTOCOL=None        # "http:"
 # AWS_S3_REGION_NAME=None         # "fr-par"
 # AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
 
@@ -133,7 +144,14 @@ HTTP_X_FORWARDED_PROTO=false
 TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2
 TWO_FACTOR_LOGIN_MAX_SECONDS=60
 
-# Additional hosts to allow in the Content-Security-Policy, "self" (should be DOMAIN)
-# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
-# Value should be a comma-separated list of host names.
+# Additional hosts to allow in the Content-Security-Policy, "self" (should be
+# DOMAIN with optionally ":" + PORT) and AWS_S3_CUSTOM_DOMAIN (if used) are
+# added by default.  Value should be a comma-separated list of host names.
 CSP_ADDITIONAL_HOSTS=
+
+# Time before being logged out (in seconds)
+# SESSION_COOKIE_AGE=2592000   # current default: 30 days
+
+# Maximum allowed memory for file uploads (increase if users are having trouble
+# uploading BookWyrm export files).
+# DATA_UPLOAD_MAX_MEMORY_MiB=100

.github/workflows/black.yml

@@ -1,17 +0,0 @@
-name: Python Formatting (run ./bw-dev black to fix)
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-      - uses: psf/black@22.12.0
-        with:
-          version: 22.12.0

.github/workflows/codeql-analysis.yml

@@ -36,11 +36,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,7 +51,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -65,4 +65,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/curlylint.yaml

@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install curlylint
         run: pip install curlylint

.github/workflows/django-tests.yml

@@ -1,61 +0,0 @@
-name: Run Python Tests
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-20.04
-    services:
-      postgres:
-        image: postgres:13
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: hunter2
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Run Tests
-      env:
-        SECRET_KEY: beepbeep
-        DEBUG: false
-        USE_HTTPS: true
-        DOMAIN: your.domain.here
-        BOOKWYRM_DATABASE_BACKEND: postgres
-        MEDIA_ROOT: images/
-        POSTGRES_PASSWORD: hunter2
-        POSTGRES_USER: postgres
-        POSTGRES_DB: github_actions
-        POSTGRES_HOST: 127.0.0.1
-        CELERY_BROKER: ""
-        REDIS_BROKER_PORT: 6379
-        REDIS_BROKER_PASSWORD: beep
-        USE_DUMMY_CACHE: true
-        FLOWER_PORT: 8888
-        EMAIL_HOST: "smtp.mailgun.org"
-        EMAIL_PORT: 587
-        EMAIL_HOST_USER: ""
-        EMAIL_HOST_PASSWORD: ""
-        EMAIL_USE_TLS: true
-        ENABLE_PREVIEW_IMAGES: false
-        ENABLE_THUMBNAIL_GENERATION: true
-        HTTP_X_FORWARDED_PROTO: false
-      run: |
-        pytest -n 3

.github/workflows/lint-frontend.yaml

@@ -19,10 +19,11 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install modules
-        run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
+      #  run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
+        run: npm install eslint@^8.9.0
 
       # See .stylelintignore for files that are not linted.
       # - name: Run stylelint

.github/workflows/mypy.yml

@@ -1,50 +0,0 @@
-name: Mypy
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Analysing the code with mypy
-      env:
-        SECRET_KEY: beepbeep
-        DEBUG: false
-        USE_HTTPS: true
-        DOMAIN: your.domain.here
-        BOOKWYRM_DATABASE_BACKEND: postgres
-        MEDIA_ROOT: images/
-        POSTGRES_PASSWORD: hunter2
-        POSTGRES_USER: postgres
-        POSTGRES_DB: github_actions
-        POSTGRES_HOST: 127.0.0.1
-        CELERY_BROKER: ""
-        REDIS_BROKER_PORT: 6379
-        REDIS_BROKER_PASSWORD: beep
-        USE_DUMMY_CACHE: true
-        FLOWER_PORT: 8888
-        EMAIL_HOST: "smtp.mailgun.org"
-        EMAIL_PORT: 587
-        EMAIL_HOST_USER: ""
-        EMAIL_HOST_PASSWORD: ""
-        EMAIL_USE_TLS: true
-        ENABLE_PREVIEW_IMAGES: false
-        ENABLE_THUMBNAIL_GENERATION: true
-        HTTP_X_FORWARDED_PROTO: false
-      run: |
-        mypy bookwyrm celerywyrm

.github/workflows/prettier.yaml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install modules
         run: npm install prettier@2.5.1

.github/workflows/pylint.yml

@@ -1,27 +0,0 @@
-name: Pylint
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Analysing the code with pylint
-      run: |
-        pylint bookwyrm/
-

.github/workflows/python.yml

@@ -0,0 +1,99 @@
+name: Python
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+# overrides for .env.example
+env:
+  POSTGRES_HOST: 127.0.0.1
+  PGPORT: 5432
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: hunter2
+  POSTGRES_DB: github_actions
+  SECRET_KEY: beepbeep
+  EMAIL_HOST_USER: ""
+  EMAIL_HOST_PASSWORD: ""
+
+jobs:
+  pytest:
+    name: Tests (pytest)
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:13
+        env: # does not inherit from jobs.build.env
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: hunter2
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pytest-github-actions-annotate-failures
+    - name: Set up .env
+      run: cp .env.example .env
+    - name: Check migrations up-to-date
+      run: python ./manage.py makemigrations --check
+    - name: Run Tests
+      run: pytest -n 3
+
+  pylint:
+    name: Linting (pylint)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Analyse code with pylint
+      run: pylint bookwyrm/
+
+  mypy:
+    name: Typing (mypy)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Set up .env
+      run: cp .env.example .env
+    - name: Analyse code with mypy
+      run: mypy bookwyrm celerywyrm
+
+  black:
+    name: Formatting (black; run ./bw-dev black to fix)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+    - uses: psf/black@stable
+      with:
+        version: "22.*"

.gitignore

@@ -16,6 +16,8 @@
 # BookWyrm
 .env
 /images/
+/exports/
+/static/
 bookwyrm/static/css/bookwyrm.css
 bookwyrm/static/css/themes/
 !bookwyrm/static/css/themes/bookwyrm-*.scss
@@ -36,3 +38,6 @@ nginx/default.conf
 
 #macOS
 **/.DS_Store
+
+# Docker
+docker-compose.override.yml

.prettierrc

@@ -0,0 +1 @@
+'trailingComma': 'es5'

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.11
 
 ENV PYTHONUNBUFFERED 1
 

FEDERATION.md

@@ -13,14 +13,15 @@ User relationship interactions follow the standard ActivityPub spec.
 - `Block`: prevent users from seeing one another's statuses, and prevents the blocked user from viewing the actor's profile
 - `Update`: updates a user's profile and settings
 - `Delete`: deactivates a user
-- `Undo`: reverses a `Follow` or `Block`
+- `Undo`: reverses a `Block` or `Follow`
 
 ### Activities
 - `Create/Status`: saves a new status in the database.
 - `Delete/Status`: Removes a status
 - `Like/Status`: Creates a favorite on the status
 - `Announce/Status`: Boosts the status into the actor's timeline
-- `Undo/*`,: Reverses a `Like` or `Announce`
+- `Undo/*`,: Reverses an `Announce`, `Like`, or `Move`
+- `Move/User`: Moves a user from one ActivityPub id to another.
 
 ### Collections
 User's books and lists are represented by [`OrderedCollection`](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection)

README.md

@@ -10,7 +10,6 @@ BookWyrm is a social network for tracking your reading, talking about books, wri
 ## Links
 
 [![Mastodon Follow](https://img.shields.io/mastodon/follow/000146121?domain=https%3A%2F%2Ftech.lgbt&style=social)](https://tech.lgbt/@bookwyrm)
-[![Twitter Follow](https://img.shields.io/twitter/follow/BookWyrmSocial?style=social)](https://twitter.com/BookWyrmSocial)
 
  - [Project homepage](https://joinbookwyrm.com/)
  - [Support](https://patreon.com/bookwyrm)

VERSION

@@ -0,0 +1 @@
+0.7.3

bookwyrm/activitypub/__init__.py

@@ -4,7 +4,11 @@ import sys
 
 from .base_activity import ActivityEncoder, Signature, naive_parse
 from .base_activity import Link, Mention, Hashtag
-from .base_activity import ActivitySerializerError, resolve_remote_id
+from .base_activity import (
+    ActivitySerializerError,
+    resolve_remote_id,
+    get_representative,
+)
 from .image import Document, Image
 from .note import Note, GeneratedNote, Article, Comment, Quotation
 from .note import Review, Rating
@@ -19,6 +23,7 @@ from .verbs import Create, Delete, Undo, Update
 from .verbs import Follow, Accept, Reject, Block
 from .verbs import Add, Remove
 from .verbs import Announce, Like
+from .verbs import Move
 
 # this creates a list of all the Activity types that we can serialize,
 # so when an Activity comes in from outside, we can check if it's known

bookwyrm/activitypub/base_activity.py

@@ -1,4 +1,5 @@
 """ basics for an activitypub serializer """
+from __future__ import annotations
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
 import logging
@@ -19,6 +20,7 @@ from bookwyrm.tasks import app, MISC
 
 logger = logging.getLogger(__name__)
 
+# pylint: disable=invalid-name
 TBookWyrmModel = TypeVar("TBookWyrmModel", bound=base_model.BookWyrmModel)
 
 
@@ -72,8 +74,10 @@ class ActivityObject:
 
     def __init__(
         self,
-        activity_objects: Optional[list[str, base_model.BookWyrmModel]] = None,
-        **kwargs: dict[str, Any],
+        activity_objects: Optional[
+            dict[str, Union[str, list[str], ActivityObject, base_model.BookWyrmModel]]
+        ] = None,
+        **kwargs: Any,
     ):
         """this lets you pass in an object with fields that aren't in the
         dataclass, which it ignores. Any field in the dataclass is required or
@@ -233,7 +237,7 @@ class ActivityObject:
         omit = kwargs.get("omit", ())
         data = self.__dict__.copy()
         # recursively serialize
-        for (k, v) in data.items():
+        for k, v in data.items():
             try:
                 if issubclass(type(v), ActivityObject):
                     data[k] = v.serialize()
@@ -393,19 +397,15 @@ def resolve_remote_id(
 
 def get_representative():
     """Get or create an actor representing the instance
-    to sign requests to 'secure mastodon' servers"""
-    username = f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}"
-    email = "bookwyrm@localhost"
-    try:
-        user = models.User.objects.get(username=username)
-    except models.User.DoesNotExist:
-        user = models.User.objects.create_user(
-            username=username,
-            email=email,
+    to sign outgoing HTTP GET requests"""
+    return models.User.objects.get_or_create(
+        username=f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}",
+        defaults=dict(
+            email="bookwyrm@localhost",
             local=True,
             localname=INSTANCE_ACTOR_USERNAME,
-        )
-    return user
+        ),
+    )[0]
 
 
 def get_activitypub_data(url):
@@ -424,6 +424,7 @@ def get_activitypub_data(url):
                 "Date": now,
                 "Signature": make_signature("get", sender, url, now),
             },
+            timeout=15,
         )
     except requests.RequestException:
         raise ConnectorException()

bookwyrm/activitypub/book.py

@@ -22,8 +22,6 @@ class BookData(ActivityObject):
     aasin: Optional[str] = None
     isfdb: Optional[str] = None
     lastEditedBy: Optional[str] = None
-    links: list[str] = field(default_factory=list)
-    fileLinks: list[str] = field(default_factory=list)
 
 
 # pylint: disable=invalid-name
@@ -45,6 +43,8 @@ class Book(BookData):
     firstPublishedDate: str = ""
     publishedDate: str = ""
 
+    fileLinks: list[str] = field(default_factory=list)
+
     cover: Optional[Document] = None
     type: str = "Book"
 

bookwyrm/activitypub/person.py

@@ -1,5 +1,5 @@
 """ actor serializer """
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import Dict
 
 from .base_activity import ActivityObject
@@ -35,9 +35,11 @@ class Person(ActivityObject):
     endpoints: Dict = None
     name: str = None
     summary: str = None
-    icon: Image = field(default_factory=lambda: {})
+    icon: Image = None
     bookwyrmUser: bool = False
     manuallyApprovesFollowers: str = False
     discoverable: str = False
     hideFollows: str = False
+    movedTo: str = None
+    alsoKnownAs: dict[str] = None
     type: str = "Person"

bookwyrm/activitypub/verbs.py

@@ -171,9 +171,19 @@ class Reject(Verb):
     type: str = "Reject"
 
     def action(self, allow_external_connections=True):
-        """reject a follow request"""
-        obj = self.object.to_model(save=False, allow_create=False)
-        obj.reject()
+        """reject a follow or follow request"""
+
+        for model_name in ["UserFollowRequest", "UserFollows", None]:
+            model = apps.get_model(f"bookwyrm.{model_name}") if model_name else None
+            if obj := self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            ):
+                # Reject the first model that can be built.
+                obj.reject()
+                break
 
 
 @dataclass(init=False)
@@ -231,3 +241,30 @@ class Announce(Verb):
     def action(self, allow_external_connections=True):
         """boost"""
         self.to_model(allow_external_connections=allow_external_connections)
+
+
+@dataclass(init=False)
+class Move(Verb):
+    """a user moving an object"""
+
+    object: str
+    type: str = "Move"
+    origin: str = None
+    target: str = None
+
+    def action(self, allow_external_connections=True):
+        """move"""
+
+        object_is_user = resolve_remote_id(remote_id=self.object, model="User")
+
+        if object_is_user:
+            model = apps.get_model("bookwyrm.MoveUser")
+
+            self.to_model(
+                model=model,
+                save=True,
+                allow_external_connections=allow_external_connections,
+            )
+        else:
+            # we might do something with this to move other objects at some point
+            pass

bookwyrm/activitystreams.py

@@ -139,14 +139,14 @@ class ActivityStream(RedisStore):
                 | (
                     Q(following=status.user) & Q(following=status.reply_parent.user)
                 )  # if the user is following both authors
-            ).distinct()
+            )
 
         # only visible to the poster's followers and tagged users
         elif status.privacy == "followers":
             audience = audience.filter(
                 Q(following=status.user)  # if the user is following the author
             )
-        return audience.distinct()
+        return audience.distinct("id")
 
     @tracer.start_as_current_span("ActivityStream.get_audience")
     def get_audience(self, status):
@@ -156,7 +156,7 @@ class ActivityStream(RedisStore):
         status_author = models.User.objects.filter(
             is_active=True, local=True, id=status.user.id
         ).values_list("id", flat=True)
-        return list(set(list(audience) + list(status_author)))
+        return list(set(audience) | set(status_author))
 
     def get_stores_for_users(self, user_ids):
         """convert a list of user ids into redis store ids"""
@@ -183,15 +183,13 @@ class HomeStream(ActivityStream):
     def get_audience(self, status):
         trace.get_current_span().set_attribute("stream_id", self.key)
         audience = super()._get_audience(status)
-        if not audience:
-            return []
         # if the user is following the author
         audience = audience.filter(following=status.user).values_list("id", flat=True)
         # if the user is the post's author
         status_author = models.User.objects.filter(
             is_active=True, local=True, id=status.user.id
         ).values_list("id", flat=True)
-        return list(set(list(audience) + list(status_author)))
+        return list(set(audience) | set(status_author))
 
     def get_statuses_for_user(self, user):
         return models.Status.privacy_filter(
@@ -239,9 +237,7 @@ class BooksStream(ActivityStream):
         )
 
         audience = super()._get_audience(status)
-        if not audience:
-            return models.User.objects.none()
-        return audience.filter(shelfbook__book__parent_work=work).distinct()
+        return audience.filter(shelfbook__book__parent_work=work)
 
     def get_audience(self, status):
         # only show public statuses on the books feed,

bookwyrm/apps.py

@@ -1,4 +1,5 @@
 """Do further startup configuration and initialization"""
+
 import os
 import urllib
 import logging
@@ -14,16 +15,16 @@ def download_file(url, destination):
     """Downloads a file to the given path"""
     try:
         # Ensure our destination directory exists
-        os.makedirs(os.path.dirname(destination))
+        os.makedirs(os.path.dirname(destination), exist_ok=True)
         with urllib.request.urlopen(url) as stream:
             with open(destination, "b+w") as outfile:
                 outfile.write(stream.read())
-    except (urllib.error.HTTPError, urllib.error.URLError):
-        logger.info("Failed to download file %s", url)
-    except OSError:
-        logger.info("Couldn't open font file %s for writing", destination)
-    except:  # pylint: disable=bare-except
-        logger.info("Unknown error in file download")
+    except (urllib.error.HTTPError, urllib.error.URLError) as err:
+        logger.error("Failed to download file %s: %s", url, err)
+    except OSError as err:
+        logger.error("Couldn't open font file %s for writing: %s", destination, err)
+    except Exception as err:  # pylint:disable=broad-except
+        logger.error("Unknown error in file download: %s", err)
 
 
 class BookwyrmConfig(AppConfig):

bookwyrm/book_search.py

@@ -43,6 +43,7 @@ def search(
     min_confidence: float = 0,
     filters: Optional[list[Any]] = None,
     return_first: bool = False,
+    books: Optional[QuerySet[models.Edition]] = None,
 ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
     """search your local database"""
     filters = filters or []
@@ -54,13 +55,15 @@ def search(
     # first, try searching unique identifiers
     # unique identifiers never have spaces, title/author usually do
     if not " " in query:
-        results = search_identifiers(query, *filters, return_first=return_first)
+        results = search_identifiers(
+            query, *filters, return_first=return_first, books=books
+        )
 
     # if there were no identifier results...
     if not results:
         # then try searching title/author
         results = search_title_author(
-            query, min_confidence, *filters, return_first=return_first
+            query, min_confidence, *filters, return_first=return_first, books=books
         )
     return results
 
@@ -98,9 +101,17 @@ def format_search_result(search_result):
 
 
 def search_identifiers(
-    query, *filters, return_first=False
+    query,
+    *filters,
+    return_first=False,
+    books=None,
 ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
-    """tries remote_id, isbn; defined as dedupe fields on the model"""
+    """search Editions by deduplication fields
+
+    Best for cases when we can assume someone is searching for an exact match on
+    commonly unique data identifiers like isbn or specific library ids.
+    """
+    books = books or models.Edition.objects
     if connectors.maybe_isbn(query):
         # Oh did you think the 'S' in ISBN stood for 'standard'?
         normalized_isbn = query.strip().upper().rjust(10, "0")
@@ -111,7 +122,7 @@ def search_identifiers(
         for f in models.Edition._meta.get_fields()
         if hasattr(f, "deduplication_field") and f.deduplication_field
     ]
-    results = models.Edition.objects.filter(
+    results = books.filter(
         *filters, reduce(operator.or_, (Q(**f) for f in or_filters))
     ).distinct()
 
@@ -121,12 +132,17 @@ def search_identifiers(
 
 
 def search_title_author(
-    query, min_confidence, *filters, return_first=False
+    query,
+    min_confidence,
+    *filters,
+    return_first=False,
+    books=None,
 ) -> QuerySet[models.Edition]:
     """searches for title and author"""
+    books = books or models.Edition.objects
     query = SearchQuery(query, config="simple") | SearchQuery(query, config="english")
     results = (
-        models.Edition.objects.filter(*filters, search_vector=query)
+        books.filter(*filters, search_vector=query)
         .annotate(rank=SearchRank(F("search_vector"), query))
         .filter(rank__gt=min_confidence)
         .order_by("-rank")
@@ -137,7 +153,7 @@ def search_title_author(
 
     # filter out multiple editions of the same work
     list_results = []
-    for work_id in set(editions_of_work[:30]):
+    for work_id in editions_of_work[:30]:
         result = (
             results.filter(parent_work=work_id)
             .order_by("-rank", "-edition_rank")

bookwyrm/connectors/abstract_connector.py

@@ -3,7 +3,9 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from typing import Optional, TypedDict, Any, Callable, Union, Iterator
 from urllib.parse import quote_plus
-import imghdr
+
+# pylint: disable-next=deprecated-module
+import imghdr  # Deprecated in 3.11 for removal in 3.13; no good alternative yet
 import logging
 import re
 import asyncio

bookwyrm/connectors/connector_manager.py

@@ -118,9 +118,11 @@ def get_connectors() -> Iterator[abstract_connector.AbstractConnector]:
 def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnector:
     """get the connector related to the object's server"""
     url = urlparse(remote_id)
-    identifier = url.netloc
+    identifier = url.hostname
     if not identifier:
-        raise ValueError("Invalid remote id")
+        raise ValueError(f"Invalid remote id: {remote_id}")
+
+    base_url = f"{url.scheme}://{url.netloc}"
 
     try:
         connector_info = models.Connector.objects.get(identifier=identifier)
@@ -128,10 +130,10 @@ def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnec
         connector_info = models.Connector.objects.create(
             identifier=identifier,
             connector_file="bookwyrm_connector",
-            base_url=f"https://{identifier}",
-            books_url=f"https://{identifier}/book",
-            covers_url=f"https://{identifier}/images/covers",
-            search_url=f"https://{identifier}/search?q=",
+            base_url=base_url,
+            books_url=f"{base_url}/book",
+            covers_url=f"{base_url}/images/covers",
+            search_url=f"{base_url}/search?q=",
             priority=2,
         )
 
@@ -188,8 +190,11 @@ def raise_not_valid_url(url: str) -> None:
     if not parsed.scheme in ["http", "https"]:
         raise ConnectorException("Invalid scheme: ", url)
 
+    if not parsed.hostname:
+        raise ConnectorException("Hostname missing: ", url)
+
     try:
-        ipaddress.ip_address(parsed.netloc)
+        ipaddress.ip_address(parsed.hostname)
         raise ConnectorException("Provided url is an IP address: ", url)
     except ValueError:
         # it's not an IP address, which is good

bookwyrm/emailing.py

@@ -4,7 +4,7 @@ from django.template.loader import get_template
 
 from bookwyrm import models, settings
 from bookwyrm.tasks import app, EMAIL
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import DOMAIN, BASE_URL
 
 
 def email_data():
@@ -14,6 +14,7 @@ def email_data():
         "site_name": site.name,
         "logo": site.logo_small_url,
         "domain": DOMAIN,
+        "base_url": BASE_URL,
         "user": None,
     }
 

bookwyrm/forms/author.py

@@ -15,6 +15,7 @@ class AuthorForm(CustomForm):
             "aliases",
             "bio",
             "wikipedia_link",
+            "wikidata",
             "website",
             "born",
             "died",
@@ -32,6 +33,7 @@ class AuthorForm(CustomForm):
             "wikipedia_link": forms.TextInput(
                 attrs={"aria-describedby": "desc_wikipedia_link"}
             ),
+            "wikidata": forms.TextInput(attrs={"aria-describedby": "desc_wikidata"}),
             "website": forms.TextInput(attrs={"aria-describedby": "desc_website"}),
             "born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}),
             "died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}),

bookwyrm/forms/books.py

@@ -1,8 +1,9 @@
 """ using django model forms """
 from django import forms
 
+from file_resubmit.widgets import ResubmitImageWidget
+
 from bookwyrm import models
-from bookwyrm.models.fields import ClearableFileInputWithWarning
 from .custom_form import CustomForm
 from .widgets import ArrayWidget, SelectDateWidget, Select
 
@@ -70,9 +71,7 @@ class EditionForm(CustomForm):
             "published_date": SelectDateWidget(
                 attrs={"aria-describedby": "desc_published_date"}
             ),
-            "cover": ClearableFileInputWithWarning(
-                attrs={"aria-describedby": "desc_cover"}
-            ),
+            "cover": ResubmitImageWidget(attrs={"aria-describedby": "desc_cover"}),
             "physical_format": Select(
                 attrs={"aria-describedby": "desc_physical_format"}
             ),

bookwyrm/forms/edit_user.py

@@ -70,6 +70,22 @@ class DeleteUserForm(CustomForm):
         fields = ["password"]
 
 
+class MoveUserForm(CustomForm):
+    target = forms.CharField(widget=forms.TextInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+
+
+class AliasUserForm(CustomForm):
+    username = forms.CharField(widget=forms.TextInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+
+
 class ChangePasswordForm(CustomForm):
     current_password = forms.CharField(widget=forms.PasswordInput)
     confirm_password = forms.CharField(widget=forms.PasswordInput)

bookwyrm/forms/forms.py

@@ -25,6 +25,10 @@ class ImportForm(forms.Form):
     csv_file = forms.FileField()
 
 
+class ImportUserForm(forms.Form):
+    archive_file = forms.FileField()
+
+
 class ShelfForm(CustomForm):
     class Meta:
         model = models.Shelf

bookwyrm/forms/links.py

@@ -1,4 +1,5 @@
 """ using django model forms """
+
 from urllib.parse import urlparse
 
 from django.utils.translation import gettext_lazy as _
@@ -25,7 +26,7 @@ class FileLinkForm(CustomForm):
         url = cleaned_data.get("url")
         filetype = cleaned_data.get("filetype")
         book = cleaned_data.get("book")
-        domain = urlparse(url).netloc
+        domain = urlparse(url).hostname
         if models.LinkDomain.objects.filter(domain=domain).exists():
             status = models.LinkDomain.objects.get(domain=domain).status
             if status == "blocked":
@@ -37,10 +38,9 @@ class FileLinkForm(CustomForm):
                     ),
                 )
         if (
-            not self.instance
-            and models.FileLink.objects.filter(
-                url=url, book=book, filetype=filetype
-            ).exists()
+            models.FileLink.objects.filter(url=url, book=book, filetype=filetype)
+            .exclude(pk=self.instance)
+            .exists()
         ):
             # pylint: disable=line-too-long
             self.add_error(

bookwyrm/importers/__init__.py

@@ -1,6 +1,7 @@
 """ import classes """
 
 from .importer import Importer
+from .bookwyrm_import import BookwyrmImporter
 from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter

bookwyrm/importers/bookwyrm_import.py

@@ -0,0 +1,24 @@
+"""Import data from Bookwyrm export files"""
+from django.http import QueryDict
+
+from bookwyrm.models import User
+from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
+
+
+class BookwyrmImporter:
+    """Import a Bookwyrm User export file.
+    This is kind of a combination of an importer and a connector.
+    """
+
+    # pylint: disable=no-self-use
+    def process_import(
+        self, user: User, archive_file: bytes, settings: QueryDict
+    ) -> BookwyrmImportJob:
+        """import user data from a Bookwyrm export file"""
+
+        required = [k for k in settings if settings.get(k) == "on"]
+
+        job = BookwyrmImportJob.objects.create(
+            user=user, archive_file=archive_file, required=required
+        )
+        return job

bookwyrm/isbn/isbn.py

@@ -26,7 +26,7 @@ class IsbnHyphenator:
 
     def update_range_message(self) -> None:
         """Download the range message xml file and save it locally"""
-        response = requests.get(self.__range_message_url)
+        response = requests.get(self.__range_message_url, timeout=15)
         with open(self.__range_file_path, "w", encoding="utf-8") as file:
             file.write(response.text)
         self.__element_tree = None
@@ -40,7 +40,12 @@ class IsbnHyphenator:
             self.__element_tree = ElementTree.parse(self.__range_file_path)
 
         gs1_prefix = isbn_13[:3]
-        reg_group = self.__find_reg_group(isbn_13, gs1_prefix)
+        try:
+            reg_group = self.__find_reg_group(isbn_13, gs1_prefix)
+        except ValueError:
+            # if the reg groups are invalid, just return the original isbn
+            return isbn_13
+
         if reg_group is None:
             return isbn_13  # failed to hyphenate
 

bookwyrm/management/commands/deduplicate_book_data.py

@@ -1,13 +1,14 @@
 """ PROCEED WITH CAUTION: uses deduplication fields to permanently
 merge book data objects """
+
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from bookwyrm import models
-from bookwyrm.management.merge import merge_objects
 
 
-def dedupe_model(model):
+def dedupe_model(model, dry_run=False):
     """combine duplicate editions and update related models"""
+    print(f"deduplicating {model.__name__}:")
     fields = model._meta.get_fields()
     dedupe_fields = [
         f for f in fields if hasattr(f, "deduplication_field") and f.deduplication_field
@@ -16,30 +17,42 @@ def dedupe_model(model):
         dupes = (
             model.objects.values(field.name)
             .annotate(Count(field.name))
-            .filter(**{"%s__count__gt" % field.name: 1})
+            .filter(**{f"{field.name}__count__gt": 1})
+            .exclude(**{field.name: ""})
+            .exclude(**{f"{field.name}__isnull": True})
         )
 
         for dupe in dupes:
             value = dupe[field.name]
-            if not value or value == "":
-                continue
             print("----------")
-            print(dupe)
             objs = model.objects.filter(**{field.name: value}).order_by("id")
             canonical = objs.first()
-            print("keeping", canonical.remote_id)
+            action = "would merge" if dry_run else "merging"
+            print(
+                f"{action} into {model.__name__} {canonical.remote_id} based on {field.name} {value}:"
+            )
             for obj in objs[1:]:
-                print(obj.remote_id)
-                merge_objects(canonical, obj)
+                print(f"- {obj.remote_id}")
+                absorbed_fields = obj.merge_into(canonical, dry_run=dry_run)
+                print(f"  absorbed fields: {absorbed_fields}")
 
 
 class Command(BaseCommand):
     """deduplicate allllll the book data models"""
 
     help = "merges duplicate book data"
+
+    def add_arguments(self, parser):
+        """add the arguments for this command"""
+        parser.add_argument(
+            "--dry_run",
+            action="store_true",
+            help="don't actually merge, only print what would happen",
+        )
+
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
         """run deduplications"""
-        dedupe_model(models.Edition)
-        dedupe_model(models.Work)
-        dedupe_model(models.Author)
+        dedupe_model(models.Edition, dry_run=options["dry_run"])
+        dedupe_model(models.Work, dry_run=options["dry_run"])
+        dedupe_model(models.Author, dry_run=options["dry_run"])

bookwyrm/management/commands/erase_deleted_user_data.py

@@ -0,0 +1,43 @@
+""" Erase any data stored about deleted users """
+import sys
+from django.core.management.base import BaseCommand, CommandError
+from bookwyrm import models
+from bookwyrm.models.user import erase_user_data
+
+# pylint: disable=missing-function-docstring
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Remove Two Factor Authorisation from user"
+
+    def add_arguments(self, parser):  # pylint: disable=no-self-use
+        parser.add_argument(
+            "--dryrun",
+            action="store_true",
+            help="Preview users to be cleared without altering the database",
+        )
+
+    def handle(self, *args, **options):  # pylint: disable=unused-argument
+
+        # Check for anything fishy
+        bad_state = models.User.objects.filter(is_deleted=True, is_active=True)
+        if bad_state.exists():
+            raise CommandError(
+                f"{bad_state.count()} user(s) marked as both active and deleted"
+            )
+
+        deleted_users = models.User.objects.filter(is_deleted=True)
+        self.stdout.write(f"Found {deleted_users.count()} deleted users")
+        if options["dryrun"]:
+            self.stdout.write("\n".join(u.username for u in deleted_users[:5]))
+            if deleted_users.count() > 5:
+                self.stdout.write("... and more")
+            sys.exit()
+
+        self.stdout.write("Erasing user data:")
+        for user_id in deleted_users.values_list("id", flat=True):
+            erase_user_data.delay(user_id)
+            self.stdout.write(".", ending="")
+
+        self.stdout.write("")
+        self.stdout.write("Tasks created successfully")

bookwyrm/management/commands/instance_version.py

@@ -1,54 +0,0 @@
-""" Get your admin code to allow install """
-from django.core.management.base import BaseCommand
-
-from bookwyrm import models
-from bookwyrm.settings import VERSION
-
-
-# pylint: disable=no-self-use
-class Command(BaseCommand):
-    """command-line options"""
-
-    help = "What version is this?"
-
-    def add_arguments(self, parser):
-        """specify which function to run"""
-        parser.add_argument(
-            "--current",
-            action="store_true",
-            help="Version stored in database",
-        )
-        parser.add_argument(
-            "--target",
-            action="store_true",
-            help="Version stored in settings",
-        )
-        parser.add_argument(
-            "--update",
-            action="store_true",
-            help="Update database version",
-        )
-
-    # pylint: disable=unused-argument
-    def handle(self, *args, **options):
-        """execute init"""
-        site = models.SiteSettings.objects.get()
-        current = site.version or "0.0.1"
-        target = VERSION
-        if options.get("current"):
-            print(current)
-            return
-
-        if options.get("target"):
-            print(target)
-            return
-
-        if options.get("update"):
-            site.version = target
-            site.save()
-            return
-
-        if current != target:
-            print(f"{current}/{target}")
-        else:
-            print(current)

bookwyrm/management/merge.py

@@ -1,50 +0,0 @@
-from django.db.models import ManyToManyField
-
-
-def update_related(canonical, obj):
-    """update all the models with fk to the object being removed"""
-    # move related models to canonical
-    related_models = [
-        (r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
-    ]
-    for (related_field, related_model) in related_models:
-        # Skip the ManyToMany fields that aren’t auto-created. These
-        # should have a corresponding OneToMany field in the model for
-        # the linking table anyway. If we update it through that model
-        # instead then we won’t lose the extra fields in the linking
-        # table.
-        related_field_obj = related_model._meta.get_field(related_field)
-        if isinstance(related_field_obj, ManyToManyField):
-            through = related_field_obj.remote_field.through
-            if not through._meta.auto_created:
-                continue
-        related_objs = related_model.objects.filter(**{related_field: obj})
-        for related_obj in related_objs:
-            print("replacing in", related_model.__name__, related_field, related_obj.id)
-            try:
-                setattr(related_obj, related_field, canonical)
-                related_obj.save()
-            except TypeError:
-                getattr(related_obj, related_field).add(canonical)
-                getattr(related_obj, related_field).remove(obj)
-
-
-def copy_data(canonical, obj):
-    """try to get the most data possible"""
-    for data_field in obj._meta.get_fields():
-        if not hasattr(data_field, "activitypub_field"):
-            continue
-        data_value = getattr(obj, data_field.name)
-        if not data_value:
-            continue
-        if not getattr(canonical, data_field.name):
-            print("setting data field", data_field.name, data_value)
-            setattr(canonical, data_field.name, data_value)
-    canonical.save()
-
-
-def merge_objects(canonical, obj):
-    copy_data(canonical, obj)
-    update_related(canonical, obj)
-    # remove the outdated entry
-    obj.delete()

bookwyrm/management/merge_command.py

@@ -1,4 +1,3 @@
-from bookwyrm.management.merge import merge_objects
 from django.core.management.base import BaseCommand
 
 
@@ -9,6 +8,11 @@ class MergeCommand(BaseCommand):
         """add the arguments for this command"""
         parser.add_argument("--canonical", type=int, required=True)
         parser.add_argument("--other", type=int, required=True)
+        parser.add_argument(
+            "--dry_run",
+            action="store_true",
+            help="don't actually merge, only print what would happen",
+        )
 
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
@@ -26,4 +30,8 @@ class MergeCommand(BaseCommand):
             print("other book doesn’t exist!")
             return
 
-        merge_objects(canonical, other)
+        absorbed_fields = other.merge_into(canonical, dry_run=options["dry_run"])
+
+        action = "would be" if options["dry_run"] else "has been"
+        print(f"{other.remote_id} {action} merged into {canonical.remote_id}")
+        print(f"absorbed fields: {absorbed_fields}")

bookwyrm/middleware/__init__.py

@@ -1,3 +1,4 @@
 """ look at all this nice middleware! """
 from .timezone_middleware import TimezoneMiddleware
 from .ip_middleware import IPBlocklistMiddleware
+from .file_too_big import FileTooBig

bookwyrm/middleware/file_too_big.py

@@ -0,0 +1,30 @@
+"""Middleware to display a custom 413 error page"""
+
+from django.http import HttpResponse
+from django.shortcuts import render
+from django.core.exceptions import RequestDataTooBig
+
+
+class FileTooBig:
+    """Middleware to display a custom page when a
+    RequestDataTooBig exception is thrown"""
+
+    def __init__(self, get_response):
+        """boilerplate __init__ from Django docs"""
+
+        self.get_response = get_response
+
+    def __call__(self, request):
+        """If RequestDataTooBig is thrown, render the 413 error page"""
+
+        try:
+            body = request.body  # pylint: disable=unused-variable
+
+        except RequestDataTooBig:
+
+            rendered = render(request, "413.html")
+            response = HttpResponse(rendered)
+            return response
+
+        response = self.get_response(request)
+        return response

bookwyrm/migrations/0179_populate_sort_title.py

@@ -45,5 +45,7 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(populate_sort_title),
+        migrations.RunPython(
+            populate_sort_title, reverse_code=migrations.RunPython.noop
+        ),
     ]

bookwyrm/migrations/0182_auto_20231027_1122.py

@@ -0,0 +1,130 @@
+# Generated by Django 3.2.20 on 2023-10-27 11:22
+
+import bookwyrm.models.activitypub_mixin
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0181_merge_20230806_2302"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="also_known_as",
+            field=bookwyrm.models.fields.ManyToManyField(to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name="user",
+            name="moved_to",
+            field=bookwyrm.models.fields.RemoteIdField(
+                max_length=255,
+                null=True,
+                validators=[bookwyrm.models.fields.validate_remote_id],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("BOOST", "Boost"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+        migrations.CreateModel(
+            name="Move",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                ("updated_date", models.DateTimeField(auto_now=True)),
+                (
+                    "remote_id",
+                    bookwyrm.models.fields.RemoteIdField(
+                        max_length=255,
+                        null=True,
+                        validators=[bookwyrm.models.fields.validate_remote_id],
+                    ),
+                ),
+                ("object", bookwyrm.models.fields.CharField(max_length=255)),
+                (
+                    "origin",
+                    bookwyrm.models.fields.CharField(
+                        blank=True, default="", max_length=255, null=True
+                    ),
+                ),
+                (
+                    "user",
+                    bookwyrm.models.fields.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=(bookwyrm.models.activitypub_mixin.ActivityMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name="MoveUser",
+            fields=[
+                (
+                    "move_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.move",
+                    ),
+                ),
+                (
+                    "target",
+                    bookwyrm.models.fields.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="move_target",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.move",),
+        ),
+    ]

bookwyrm/migrations/0183_auto_20231105_1607.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-11-05 16:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0182_auto_20231027_1122"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="is_deleted",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0184_auto_20231106_0421.py

@@ -0,0 +1,35 @@
+# Generated by Django 3.2.20 on 2023-11-06 04:21
+
+from django.db import migrations
+from bookwyrm.models import User
+
+
+def update_deleted_users(apps, schema_editor):
+    """Find all the users who are deleted, not just inactive, and set deleted"""
+    users = apps.get_model("bookwyrm", "User")
+    db_alias = schema_editor.connection.alias
+    users.objects.using(db_alias).filter(
+        is_active=False,
+        deactivation_reason__in=[
+            "self_deletion",
+            "moderator_deletion",
+        ],
+    ).update(is_deleted=True)
+
+    # differente rules for remote users
+    users.objects.using(db_alias).filter(is_active=False, local=False,).exclude(
+        deactivation_reason="moderator_deactivation",
+    ).update(is_deleted=True)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0183_auto_20231105_1607"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            update_deleted_users, reverse_code=migrations.RunPython.noop
+        ),
+    ]

bookwyrm/migrations/0185_alter_notification_notification_type.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.20 on 2023-11-13 22:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0184_auto_20231106_0421"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0186_auto_20231116_0048.py

@@ -0,0 +1,212 @@
+# Generated by Django 3.2.20 on 2023-11-16 00:48
+
+from django.conf import settings
+import django.contrib.postgres.fields
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0185_alter_notification_notification_type"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ParentJob",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("task_id", models.UUIDField(blank=True, null=True, unique=True)),
+                (
+                    "created_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                (
+                    "updated_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("complete", models.BooleanField(default=False)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("pending", "Pending"),
+                            ("active", "Active"),
+                            ("complete", "Complete"),
+                            ("stopped", "Stopped"),
+                            ("failed", "Failed"),
+                        ],
+                        default="pending",
+                        max_length=50,
+                        null=True,
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="user_import_time_limit",
+            field=models.IntegerField(default=48),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("USER_IMPORT", "User Import"),
+                    ("USER_EXPORT", "User Export"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+        migrations.CreateModel(
+            name="BookwyrmExportJob",
+            fields=[
+                (
+                    "parentjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+                ("export_data", models.FileField(null=True, upload_to="")),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.parentjob",),
+        ),
+        migrations.CreateModel(
+            name="BookwyrmImportJob",
+            fields=[
+                (
+                    "parentjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+                ("archive_file", models.FileField(blank=True, null=True, upload_to="")),
+                ("import_data", models.JSONField(null=True)),
+                (
+                    "required",
+                    django.contrib.postgres.fields.ArrayField(
+                        base_field=models.CharField(blank=True, max_length=50),
+                        blank=True,
+                        size=None,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.parentjob",),
+        ),
+        migrations.CreateModel(
+            name="ChildJob",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("task_id", models.UUIDField(blank=True, null=True, unique=True)),
+                (
+                    "created_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                (
+                    "updated_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("complete", models.BooleanField(default=False)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("pending", "Pending"),
+                            ("active", "Active"),
+                            ("complete", "Complete"),
+                            ("stopped", "Stopped"),
+                            ("failed", "Failed"),
+                        ],
+                        default="pending",
+                        max_length=50,
+                        null=True,
+                    ),
+                ),
+                (
+                    "parent_job",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="child_jobs",
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="notification",
+            name="related_user_export",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="bookwyrm.bookwyrmexportjob",
+            ),
+        ),
+    ]

bookwyrm/migrations/0186_invite_request_notification.py

@@ -0,0 +1,48 @@
+# Generated by Django 3.2.20 on 2023-11-14 10:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0185_alter_notification_notification_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="notification",
+            name="related_invite_requests",
+            field=models.ManyToManyField(to="bookwyrm.InviteRequest"),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE_REQUEST", "Invite Request"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0187_partial_publication_dates.py

@@ -0,0 +1,54 @@
+# Generated by Django 3.2.20 on 2023-11-09 16:57
+
+import bookwyrm.models.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0186_invite_request_notification"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="first_published_date_precision",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("DAY", "Day prec."),
+                    ("MONTH", "Month prec."),
+                    ("YEAR", "Year prec."),
+                ],
+                editable=False,
+                max_length=10,
+                null=True,
+            ),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="published_date_precision",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("DAY", "Day prec."),
+                    ("MONTH", "Month prec."),
+                    ("YEAR", "Year prec."),
+                ],
+                editable=False,
+                max_length=10,
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="book",
+            name="first_published_date",
+            field=bookwyrm.models.fields.PartialDateField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="book",
+            name="published_date",
+            field=bookwyrm.models.fields.PartialDateField(blank=True, null=True),
+        ),
+    ]

bookwyrm/migrations/0188_theme_loads.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2023-11-20 18:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0187_partial_publication_dates"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="theme",
+            name="loads",
+            field=models.BooleanField(blank=True, null=True),
+        ),
+    ]

bookwyrm/migrations/0189_alter_user_preferred_language.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.23 on 2023-12-12 23:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("eo-uy", "Esperanto (Esperanto)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("nl-nl", "Nederlands (Dutch)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("uk-ua", "Українська (Ukrainian)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2023-11-22 10:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0186_auto_20231116_0048"),
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = []

bookwyrm/migrations/0190_alter_notification_notification_type.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.23 on 2023-11-23 19:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0189_merge_0186_auto_20231116_0048_0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("USER_IMPORT", "User Import"),
+                    ("USER_EXPORT", "User Export"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE_REQUEST", "Invite Request"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0190_book_search_updates.py

@@ -0,0 +1,16 @@
+# Generated by Django 3.2.20 on 2023-11-24 17:11
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name="author",
+            name="bookwyrm_au_search__b050a8_gin",
+        ),
+    ]

bookwyrm/migrations/0191_merge_20240102_0326.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-01-02 03:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0189_alter_user_preferred_language"),
+        ("bookwyrm", "0190_alter_notification_notification_type"),
+    ]
+
+    operations = []

bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py

@@ -0,0 +1,76 @@
+# Generated by Django 3.2.20 on 2023-11-25 00:47
+
+from importlib import import_module
+import re
+
+from django.db import migrations
+import pgtrigger.compiler
+import pgtrigger.migrations
+
+trigger_migration = import_module("bookwyrm.migrations.0077_auto_20210623_2155")
+
+# it's _very_ convenient for development that this migration be reversible
+search_vector_trigger = trigger_migration.Migration.operations[4]
+author_search_vector_trigger = trigger_migration.Migration.operations[5]
+
+
+assert re.search(r"\bCREATE TRIGGER search_vector_trigger\b", search_vector_trigger.sql)
+assert re.search(
+    r"\bCREATE TRIGGER author_search_vector_trigger\b",
+    author_search_vector_trigger.sql,
+)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("bookwyrm", "0190_book_search_updates"),
+    ]
+
+    operations = [
+        pgtrigger.migrations.AddTrigger(
+            model_name="book",
+            trigger=pgtrigger.compiler.Trigger(
+                name="update_search_vector_on_book_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="new.search_vector := setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(bookwyrm_author.name), ' '), '')), 'C') FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id ) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D');RETURN NEW;",
+                    hash="77d6399497c0a89b0bf09d296e33c396da63705c",
+                    operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"',
+                    pgid="pgtrigger_update_search_vector_on_book_edit_bec58",
+                    table="bookwyrm_book",
+                    when="BEFORE",
+                ),
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="author",
+            trigger=pgtrigger.compiler.Trigger(
+                name="reset_search_vector_on_author_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id ) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;",
+                    hash="e7bbf08711ff3724c58f4d92fb7a082ffb3d7826",
+                    operation='UPDATE OF "name"',
+                    pgid="pgtrigger_reset_search_vector_on_author_edit_a447c",
+                    table="bookwyrm_author",
+                    when="AFTER",
+                ),
+            ),
+        ),
+        migrations.RunSQL(
+            sql="""DROP TRIGGER IF EXISTS search_vector_trigger ON bookwyrm_book;
+                   DROP FUNCTION IF EXISTS book_trigger;
+            """,
+            reverse_sql=search_vector_trigger.sql,
+        ),
+        migrations.RunSQL(
+            sql="""DROP TRIGGER IF EXISTS author_search_vector_trigger ON bookwyrm_author;
+                   DROP FUNCTION IF EXISTS author_trigger;
+            """,
+            reverse_sql=author_search_vector_trigger.sql,
+        ),
+        migrations.RunSQL(
+            # Recalculate book search vector for any missed author name changes
+            # due to bug in JOIN in the old trigger.
+            sql="UPDATE bookwyrm_book SET search_vector = NULL;",
+            reverse_sql=migrations.RunSQL.noop,
+        ),
+    ]

bookwyrm/migrations/0192_make_page_positions_text.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.23 on 2024-01-04 23:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="quotation",
+            name="endposition",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="quotation",
+            name="position",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-02 19:36
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="sitesettings",
+            old_name="version",
+            new_name="available_version",
+        ),
+    ]

bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-16 10:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="user_exports_enabled",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0193_auto_20240128_0249.py

@@ -0,0 +1,92 @@
+# Generated by Django 3.2.23 on 2024-01-28 02:49
+
+import bookwyrm.storage_backends
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0192_sitesettings_user_exports_enabled"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="bookwyrmexportjob",
+            name="export_json",
+            field=models.JSONField(
+                encoder=django.core.serializers.json.DjangoJSONEncoder, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="bookwyrmexportjob",
+            name="json_completed",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name="bookwyrmexportjob",
+            name="export_data",
+            field=models.FileField(
+                null=True,
+                storage=bookwyrm.storage_backends.ExportsFileStorage,
+                upload_to="",
+            ),
+        ),
+        migrations.CreateModel(
+            name="AddFileToTar",
+            fields=[
+                (
+                    "childjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.childjob",
+                    ),
+                ),
+                (
+                    "parent_export_job",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="child_edition_export_jobs",
+                        to="bookwyrm.bookwyrmexportjob",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.childjob",),
+        ),
+        migrations.CreateModel(
+            name="AddBookToUserExportJob",
+            fields=[
+                (
+                    "childjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.childjob",
+                    ),
+                ),
+                (
+                    "edition",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="bookwyrm.edition",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.childjob",),
+        ),
+    ]

bookwyrm/migrations/0193_merge_20240203_1539.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-02-03 15:39
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0192_make_page_positions_text"),
+        ("bookwyrm", "0192_sitesettings_user_exports_enabled"),
+    ]
+
+    operations = []

bookwyrm/migrations/0194_merge_20240203_1619.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-02-03 16:19
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0192_rename_version_sitesettings_available_version"),
+        ("bookwyrm", "0193_merge_20240203_1539"),
+    ]
+
+    operations = []

bookwyrm/migrations/0195_alter_user_preferred_language.py

@@ -0,0 +1,46 @@
+# Generated by Django 3.2.23 on 2024-02-21 00:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0194_merge_20240203_1619"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("eo-uy", "Esperanto (Esperanto)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("ko-kr", "한국어 (Korean)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("nl-nl", "Nederlands (Dutch)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("uk-ua", "Українська (Ukrainian)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0196_merge_20240318_1737.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-03-18 17:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0193_auto_20240128_0249"),
+        ("bookwyrm", "0195_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0196_merge_pr3134_into_main.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-03-18 00:48
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_migrate_search_vec_triggers_to_pgtriggers"),
+        ("bookwyrm", "0195_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0197_author_search_vector.py

@@ -0,0 +1,41 @@
+# Generated by Django 3.2.25 on 2024-03-20 15:15
+
+import django.contrib.postgres.indexes
+from django.db import migrations
+import pgtrigger.compiler
+import pgtrigger.migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0196_merge_pr3134_into_main"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="author",
+            index=django.contrib.postgres.indexes.GinIndex(
+                fields=["search_vector"], name="bookwyrm_au_search__b050a8_gin"
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="author",
+            trigger=pgtrigger.compiler.Trigger(
+                name="update_search_vector_on_author_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="new.search_vector := setweight(to_tsvector('simple', new.name), 'A') || setweight(to_tsvector('simple', coalesce(array_to_string(new.aliases, ' '), '')), 'B');RETURN NEW;",
+                    hash="b97919016236d74d0ade51a0769a173ea269da64",
+                    operation='INSERT OR UPDATE OF "name", "aliases", "search_vector"',
+                    pgid="pgtrigger_update_search_vector_on_author_edit_c61cb",
+                    table="bookwyrm_author",
+                    when="BEFORE",
+                ),
+            ),
+        ),
+        migrations.RunSQL(
+            # Calculate search vector for all Authors.
+            sql="UPDATE bookwyrm_author SET search_vector = NULL;",
+            reverse_sql="UPDATE bookwyrm_author SET search_vector = NULL;",
+        ),
+    ]

bookwyrm/migrations/0197_merge_20240324_0235.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-03-24 02:35
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0196_merge_20240318_1737"),
+        ("bookwyrm", "0196_merge_pr3134_into_main"),
+    ]
+
+    operations = []

bookwyrm/migrations/0197_mergedauthor_mergedbook.py

@@ -0,0 +1,48 @@
+# Generated by Django 3.2.24 on 2024-02-28 21:30
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0196_merge_pr3134_into_main"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="MergedBook",
+            fields=[
+                ("deleted_id", models.IntegerField(primary_key=True, serialize=False)),
+                (
+                    "merged_into",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="absorbed",
+                        to="bookwyrm.book",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="MergedAuthor",
+            fields=[
+                ("deleted_id", models.IntegerField(primary_key=True, serialize=False)),
+                (
+                    "merged_into",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="absorbed",
+                        to="bookwyrm.author",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

bookwyrm/migrations/0198_alter_bookwyrmexportjob_export_data.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.25 on 2024-03-26 11:37
+
+import bookwyrm.models.bookwyrm_export_job
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0197_merge_20240324_0235"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="bookwyrmexportjob",
+            name="export_data",
+            field=models.FileField(
+                null=True,
+                storage=bookwyrm.models.bookwyrm_export_job.select_exports_storage,
+                upload_to="",
+            ),
+        ),
+    ]

bookwyrm/migrations/0198_book_search_vector_author_aliases.py

@@ -0,0 +1,57 @@
+# Generated by Django 3.2.25 on 2024-03-20 15:52
+
+from django.db import migrations
+import pgtrigger.compiler
+import pgtrigger.migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0197_author_search_vector"),
+    ]
+
+    operations = [
+        pgtrigger.migrations.RemoveTrigger(
+            model_name="author",
+            name="reset_search_vector_on_author_edit",
+        ),
+        pgtrigger.migrations.RemoveTrigger(
+            model_name="book",
+            name="update_search_vector_on_book_edit",
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="author",
+            trigger=pgtrigger.compiler.Trigger(
+                name="reset_book_search_vector_on_author_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id ) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;",
+                    hash="68422c0f29879c5802b82159dde45297eff53e73",
+                    operation='UPDATE OF "name", "aliases"',
+                    pgid="pgtrigger_reset_book_search_vector_on_author_edit_a50c7",
+                    table="bookwyrm_author",
+                    when="AFTER",
+                ),
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="book",
+            trigger=pgtrigger.compiler.Trigger(
+                name="update_search_vector_on_book_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="WITH author_names AS (SELECT array_to_string(bookwyrm_author.name || bookwyrm_author.aliases, ' ') AS name_and_aliases FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id ) SELECT setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(name_and_aliases), ' '), '')), 'C') FROM author_names) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D') INTO new.search_vector;RETURN NEW;",
+                    hash="9324f5ca76a6f5e63931881d62d11da11f595b2c",
+                    operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"',
+                    pgid="pgtrigger_update_search_vector_on_book_edit_bec58",
+                    table="bookwyrm_book",
+                    when="BEFORE",
+                ),
+            ),
+        ),
+        migrations.RunSQL(
+            # Recalculate search vector for all Books because it now includes
+            # Author aliases.
+            sql="UPDATE bookwyrm_book SET search_vector = NULL;",
+            reverse_sql="UPDATE bookwyrm_book SET search_vector = NULL;",
+        ),
+    ]

bookwyrm/migrations/0199_merge_20240326_1217.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-03-26 12:17
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0198_alter_bookwyrmexportjob_export_data"),
+        ("bookwyrm", "0198_book_search_vector_author_aliases"),
+    ]
+
+    operations = []

bookwyrm/migrations/0199_status_bookwyrm_st_remote__06aeba_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-02 19:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0198_book_search_vector_author_aliases"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="status",
+            index=models.Index(
+                fields=["remote_id"], name="bookwyrm_st_remote__06aeba_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0200_auto_20240327_1914.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.25 on 2024-03-27 19:14
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0199_merge_20240326_1217"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="addfiletotar",
+            name="childjob_ptr",
+        ),
+        migrations.RemoveField(
+            model_name="addfiletotar",
+            name="parent_export_job",
+        ),
+        migrations.DeleteModel(
+            name="AddBookToUserExportJob",
+        ),
+        migrations.DeleteModel(
+            name="AddFileToTar",
+        ),
+    ]

bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0199_status_bookwyrm_st_remote__06aeba_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="status",
+            index=models.Index(
+                fields=["thread_id"], name="bookwyrm_st_thread__cf064f_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0200_status_bookwyrm_st_thread__cf064f_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="keypair",
+            index=models.Index(
+                fields=["remote_id"], name="bookwyrm_ke_remote__472927_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0201_keypair_bookwyrm_ke_remote__472927_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="user",
+            index=models.Index(
+                fields=["username"], name="bookwyrm_us_usernam_b2546d_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0202_user_bookwyrm_us_usernam_b2546d_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="user",
+            index=models.Index(
+                fields=["is_active", "local"], name="bookwyrm_us_is_acti_972dc4_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0204_merge_20240409_1042.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-04-09 10:42
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0197_mergedauthor_mergedbook"),
+        ("bookwyrm", "0203_user_bookwyrm_us_is_acti_972dc4_idx"),
+    ]
+
+    operations = []

bookwyrm/migrations/0205_merge_20240413_0232.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-04-13 02:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0200_auto_20240327_1914"),
+        ("bookwyrm", "0204_merge_20240409_1042"),
+    ]
+
+    operations = []

bookwyrm/models/__init__.py

@@ -26,13 +26,17 @@ from .federated_server import FederatedServer
 from .group import Group, GroupMember, GroupMemberInvitation
 
 from .import_job import ImportJob, ImportItem
+from .bookwyrm_import_job import BookwyrmImportJob
+from .bookwyrm_export_job import BookwyrmExportJob
+
+from .move import MoveUser
 
 from .site import SiteSettings, Theme, SiteInvite
 from .site import PasswordReset, InviteRequest
 from .announcement import Announcement
 from .antispam import EmailBlocklist, IPBlocklist, AutoMod, automod_task
 
-from .notification import Notification
+from .notification import Notification, NotificationType
 
 from .hashtag import Hashtag
 

bookwyrm/models/activitypub_mixin.py

@@ -152,8 +152,9 @@ class ActivitypubMixin:
         # find anyone who's tagged in a status, for example
         mentions = self.recipients if hasattr(self, "recipients") else []
 
-        # we always send activities to explicitly mentioned users' inboxes
-        recipients = [u.inbox for u in mentions or [] if not u.local]
+        # we always send activities to explicitly mentioned users (using shared inboxes
+        # where available to avoid duplicate submissions to a given instance)
+        recipients = {u.shared_inbox or u.inbox for u in mentions if not u.local}
 
         # unless it's a dm, all the followers should receive the activity
         if privacy != "direct":
@@ -173,18 +174,18 @@ class ActivitypubMixin:
             if user:
                 queryset = queryset.filter(following=user)
 
-            # ideally, we will send to shared inboxes for efficiency
-            shared_inboxes = (
-                queryset.filter(shared_inbox__isnull=False)
-                .values_list("shared_inbox", flat=True)
-                .distinct()
+            # as above, we prefer shared inboxes if available
+            recipients.update(
+                queryset.filter(shared_inbox__isnull=False).values_list(
+                    "shared_inbox", flat=True
+                )
             )
-            # but not everyone has a shared inbox
-            inboxes = queryset.filter(shared_inbox__isnull=True).values_list(
-                "inbox", flat=True
+            recipients.update(
+                queryset.filter(shared_inbox__isnull=True).values_list(
+                    "inbox", flat=True
+                )
             )
-            recipients += list(shared_inboxes) + list(inboxes)
-        return list(set(recipients))
+        return list(recipients)
 
     def to_activity_dataclass(self):
         """convert from a model to an activity"""
@@ -602,7 +603,7 @@ def to_ordered_collection_page(
     if activity_page.has_next():
         next_page = f"{remote_id}?page={activity_page.next_page_number()}"
     if activity_page.has_previous():
-        prev_page = f"{remote_id}?page=%d{activity_page.previous_page_number()}"
+        prev_page = f"{remote_id}?page={activity_page.previous_page_number()}"
     return activitypub.OrderedCollectionPage(
         id=f"{remote_id}?page={page}",
         partOf=remote_id,

bookwyrm/models/antispam.py

@@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _
 
 from bookwyrm.tasks import app, MISC
 from .base_model import BookWyrmModel
+from .notification import NotificationType
 from .user import User
 
 
@@ -80,7 +81,7 @@ def automod_task():
     with transaction.atomic():
         for admin in admins:
             notification, _ = notification_model.objects.get_or_create(
-                user=admin, notification_type=notification_model.REPORT, read=False
+                user=admin, notification_type=NotificationType.REPORT, read=False
             )
             notification.related_reports.set(reports)
 

bookwyrm/models/author.py

@@ -1,18 +1,25 @@
 """ database schema for info about authors """
+
 import re
-from django.contrib.postgres.indexes import GinIndex
+from typing import Tuple, Any
+
 from django.db import models
+from django.contrib.postgres.indexes import GinIndex
+import pgtrigger
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
+from bookwyrm.utils.db import format_trigger
 
-from .book import BookDataModel
+from .book import BookDataModel, MergedAuthor
 from . import fields
 
 
 class Author(BookDataModel):
     """basic biographic info"""
 
+    merged_model = MergedAuthor
+
     wikipedia_link = fields.CharField(
         max_length=255, blank=True, null=True, deduplication_field=True
     )
@@ -38,7 +45,7 @@ class Author(BookDataModel):
     )
     bio = fields.HtmlField(null=True, blank=True)
 
-    def save(self, *args, **kwargs):
+    def save(self, *args: Tuple[Any, ...], **kwargs: dict[str, Any]) -> None:
         """normalize isni format"""
         if self.isni:
             self.isni = re.sub(r"\s", "", self.isni)
@@ -63,11 +70,48 @@ class Author(BookDataModel):
 
     def get_remote_id(self):
         """editions and works both use "book" instead of model_name"""
-        return f"https://{DOMAIN}/author/{self.id}"
-
-    activity_serializer = activitypub.Author
+        return f"{BASE_URL}/author/{self.id}"
 
     class Meta:
-        """sets up postgres GIN index field"""
+        """sets up indexes and triggers"""
+
+        # pylint: disable=line-too-long
 
         indexes = (GinIndex(fields=["search_vector"]),)
+        triggers = [
+            pgtrigger.Trigger(
+                name="update_search_vector_on_author_edit",
+                when=pgtrigger.Before,
+                operation=pgtrigger.Insert
+                | pgtrigger.UpdateOf("name", "aliases", "search_vector"),
+                func=format_trigger(
+                    """new.search_vector :=
+                    -- author name, with priority A
+                    setweight(to_tsvector('simple', new.name), 'A') ||
+                    -- author aliases, with priority B
+                    setweight(to_tsvector('simple', coalesce(array_to_string(new.aliases, ' '), '')), 'B');
+                    RETURN new;
+                """
+                ),
+            ),
+            pgtrigger.Trigger(
+                name="reset_book_search_vector_on_author_edit",
+                when=pgtrigger.After,
+                operation=pgtrigger.UpdateOf("name", "aliases"),
+                func=format_trigger(
+                    """WITH updated_books AS (
+                         SELECT book_id
+                         FROM bookwyrm_book_authors
+                         WHERE author_id = new.id
+                    )
+                    UPDATE bookwyrm_book
+                    SET search_vector = ''
+                    FROM updated_books
+                    WHERE id = updated_books.book_id;
+                    RETURN new;
+                """
+                ),
+            ),
+        ]
+
+    activity_serializer = activitypub.Author

bookwyrm/models/base_model.py

@@ -10,7 +10,7 @@ from django.http import Http404
 from django.utils.translation import gettext_lazy as _
 from django.utils.text import slugify
 
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from .fields import RemoteIdField
 
 
@@ -38,7 +38,7 @@ class BookWyrmModel(models.Model):
 
     def get_remote_id(self):
         """generate the url that resolves to the local object, without a slug"""
-        base_path = f"https://{DOMAIN}"
+        base_path = BASE_URL
         if hasattr(self, "user"):
             base_path = f"{base_path}{self.user.local_path}"
 
@@ -53,7 +53,7 @@ class BookWyrmModel(models.Model):
     @property
     def local_path(self):
         """how to link to this object in the local app, with a slug"""
-        local = self.get_remote_id().replace(f"https://{DOMAIN}", "")
+        local = self.get_remote_id().replace(BASE_URL, "")
 
         name = None
         if hasattr(self, "name_field"):

bookwyrm/models/book.py

@@ -1,29 +1,33 @@
 """ database schema for books and shelves """
+
 from itertools import chain
 import re
-from typing import Any
+from typing import Any, Dict
+from typing_extensions import Self
 
 from django.contrib.postgres.search import SearchVectorField
 from django.contrib.postgres.indexes import GinIndex
 from django.core.cache import cache
 from django.db import models, transaction
-from django.db.models import Prefetch
+from django.db.models import Prefetch, ManyToManyField
 from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
 from model_utils import FieldTracker
 from model_utils.managers import InheritanceManager
 from imagekit.models import ImageSpecField
+import pgtrigger
 
 from bookwyrm import activitypub
 from bookwyrm.isbn.isbn import hyphenator_singleton as hyphenator
 from bookwyrm.preview_images import generate_edition_preview_image_task
 from bookwyrm.settings import (
-    DOMAIN,
+    BASE_URL,
     DEFAULT_LANGUAGE,
     LANGUAGE_ARTICLES,
     ENABLE_PREVIEW_IMAGES,
     ENABLE_THUMBNAIL_GENERATION,
 )
+from bookwyrm.utils.db import format_trigger
 
 from .activitypub_mixin import OrderedCollectionPageMixin, ObjectMixin
 from .base_model import BookWyrmModel
@@ -106,10 +110,115 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
         """only send book data updates to other bookwyrm instances"""
         super().broadcast(activity, sender, software=software, **kwargs)
 
+    def merge_into(self, canonical: Self, dry_run=False) -> Dict[str, Any]:
+        """merge this entity into another entity"""
+        if canonical.id == self.id:
+            raise ValueError(f"Cannot merge {self} into itself")
+
+        absorbed_fields = canonical.absorb_data_from(self, dry_run=dry_run)
+
+        if dry_run:
+            return absorbed_fields
+
+        canonical.save()
+
+        self.merged_model.objects.create(deleted_id=self.id, merged_into=canonical)
+
+        # move related models to canonical
+        related_models = [
+            (r.remote_field.name, r.related_model) for r in self._meta.related_objects
+        ]
+        # pylint: disable=protected-access
+        for related_field, related_model in related_models:
+            # Skip the ManyToMany fields that aren’t auto-created. These
+            # should have a corresponding OneToMany field in the model for
+            # the linking table anyway. If we update it through that model
+            # instead then we won’t lose the extra fields in the linking
+            # table.
+            # pylint: disable=protected-access
+            related_field_obj = related_model._meta.get_field(related_field)
+            if isinstance(related_field_obj, ManyToManyField):
+                through = related_field_obj.remote_field.through
+                if not through._meta.auto_created:
+                    continue
+            related_objs = related_model.objects.filter(**{related_field: self})
+            for related_obj in related_objs:
+                try:
+                    setattr(related_obj, related_field, canonical)
+                    related_obj.save()
+                except TypeError:
+                    getattr(related_obj, related_field).add(canonical)
+                    getattr(related_obj, related_field).remove(self)
+
+        self.delete()
+        return absorbed_fields
+
+    def absorb_data_from(self, other: Self, dry_run=False) -> Dict[str, Any]:
+        """fill empty fields with values from another entity"""
+        absorbed_fields = {}
+        for data_field in self._meta.get_fields():
+            if not hasattr(data_field, "activitypub_field"):
+                continue
+            canonical_value = getattr(self, data_field.name)
+            other_value = getattr(other, data_field.name)
+            if not other_value:
+                continue
+            if isinstance(data_field, fields.ArrayField):
+                if new_values := list(set(other_value) - set(canonical_value)):
+                    # append at the end (in no particular order)
+                    if not dry_run:
+                        setattr(self, data_field.name, canonical_value + new_values)
+                    absorbed_fields[data_field.name] = new_values
+            elif isinstance(data_field, fields.PartialDateField):
+                if (
+                    (not canonical_value)
+                    or (other_value.has_day and not canonical_value.has_day)
+                    or (other_value.has_month and not canonical_value.has_month)
+                ):
+                    if not dry_run:
+                        setattr(self, data_field.name, other_value)
+                    absorbed_fields[data_field.name] = other_value
+            else:
+                if not canonical_value:
+                    if not dry_run:
+                        setattr(self, data_field.name, other_value)
+                    absorbed_fields[data_field.name] = other_value
+        return absorbed_fields
+
+
+class MergedBookDataModel(models.Model):
+    """a BookDataModel instance that has been merged into another instance. kept
+    to be able to redirect old URLs"""
+
+    deleted_id = models.IntegerField(primary_key=True)
+
+    class Meta:
+        """abstract just like BookDataModel"""
+
+        abstract = True
+
+
+class MergedBook(MergedBookDataModel):
+    """an Book that has been merged into another one"""
+
+    merged_into = models.ForeignKey(
+        "Book", on_delete=models.PROTECT, related_name="absorbed"
+    )
+
+
+class MergedAuthor(MergedBookDataModel):
+    """an Author that has been merged into another one"""
+
+    merged_into = models.ForeignKey(
+        "Author", on_delete=models.PROTECT, related_name="absorbed"
+    )
+
 
 class Book(BookDataModel):
     """a generic book, which can mean either an edition or a work"""
 
+    merged_model = MergedBook
+
     connector = models.ForeignKey("Connector", on_delete=models.PROTECT, null=True)
 
     # book/work metadata
@@ -135,8 +244,8 @@ class Book(BookDataModel):
     preview_image = models.ImageField(
         upload_to="previews/covers/", blank=True, null=True
     )
-    first_published_date = fields.DateTimeField(blank=True, null=True)
-    published_date = fields.DateTimeField(blank=True, null=True)
+    first_published_date = fields.PartialDateField(blank=True, null=True)
+    published_date = fields.PartialDateField(blank=True, null=True)
 
     objects = InheritanceManager()
     field_tracker = FieldTracker(fields=["authors", "title", "subtitle", "cover"])
@@ -190,9 +299,13 @@ class Book(BookDataModel):
         """properties of this edition, as a string"""
         items = [
             self.physical_format if hasattr(self, "physical_format") else None,
-            f"{self.languages[0]} language"
-            if self.languages and self.languages[0] and self.languages[0] != "English"
-            else None,
+            (
+                f"{self.languages[0]} language"
+                if self.languages
+                and self.languages[0]
+                and self.languages[0] != "English"
+                else None
+            ),
             str(self.published_date.year) if self.published_date else None,
             ", ".join(self.publishers) if hasattr(self, "publishers") else None,
         ]
@@ -201,21 +314,20 @@ class Book(BookDataModel):
     @property
     def alt_text(self):
         """image alt test"""
-        text = self.title
-        if self.edition_info:
-            text += f" ({self.edition_info})"
-        return text
+        author = f"{name}: " if (name := self.author_text) else ""
+        edition = f" ({info})" if (info := self.edition_info) else ""
+        return f"{author}{self.title}{edition}"
 
     def save(self, *args: Any, **kwargs: Any) -> None:
         """can't be abstract for query reasons, but you shouldn't USE it"""
-        if not isinstance(self, Edition) and not isinstance(self, Work):
+        if not isinstance(self, (Edition, Work)):
             raise ValueError("Books should be added as Editions or Works")
 
         return super().save(*args, **kwargs)
 
     def get_remote_id(self):
         """editions and works both use "book" instead of model_name"""
-        return f"https://{DOMAIN}/book/{self.id}"
+        return f"{BASE_URL}/book/{self.id}"
 
     def guess_sort_title(self):
         """Get a best-guess sort title for the current book"""
@@ -233,9 +345,49 @@ class Book(BookDataModel):
         )
 
     class Meta:
-        """sets up postgres GIN index field"""
+        """set up indexes and triggers"""
+
+        # pylint: disable=line-too-long
 
         indexes = (GinIndex(fields=["search_vector"]),)
+        triggers = [
+            pgtrigger.Trigger(
+                name="update_search_vector_on_book_edit",
+                when=pgtrigger.Before,
+                operation=pgtrigger.Insert
+                | pgtrigger.UpdateOf("title", "subtitle", "series", "search_vector"),
+                func=format_trigger(
+                    """
+                    WITH author_names AS (
+                        SELECT array_to_string(bookwyrm_author.name || bookwyrm_author.aliases, ' ') AS name_and_aliases
+                            FROM bookwyrm_author
+                        LEFT JOIN bookwyrm_book_authors
+                            ON bookwyrm_author.id = bookwyrm_book_authors.author_id
+                        WHERE bookwyrm_book_authors.book_id = new.id
+                    )
+                    SELECT
+                        -- title, with priority A (parse in English, default to simple if empty)
+                        setweight(COALESCE(nullif(
+                            to_tsvector('english', new.title), ''),
+                            to_tsvector('simple', new.title)), 'A') ||
+
+                        -- subtitle, with priority B (always in English?)
+                        setweight(to_tsvector('english', COALESCE(new.subtitle, '')), 'B') ||
+
+                        -- list of authors names and aliases (with priority C)
+                        (SELECT setweight(to_tsvector('simple', COALESCE(array_to_string(ARRAY_AGG(name_and_aliases), ' '), '')), 'C')
+                            FROM author_names
+                        ) ||
+
+                        --- last: series name, with lowest priority
+                        setweight(to_tsvector('english', COALESCE(new.series, '')), 'D')
+
+                        INTO new.search_vector;
+                    RETURN new;
+                    """
+                ),
+            )
+        ]
 
 
 class Work(OrderedCollectionPageMixin, Book):
@@ -367,9 +519,9 @@ class Edition(Book):
 
         # normalize isbn format
         if self.isbn_10:
-            self.isbn_10 = re.sub(r"[^0-9X]", "", self.isbn_10)
+            self.isbn_10 = normalize_isbn(self.isbn_10)
         if self.isbn_13:
-            self.isbn_13 = re.sub(r"[^0-9X]", "", self.isbn_13)
+            self.isbn_13 = normalize_isbn(self.isbn_13)
 
         # set rank
         self.edition_rank = self.get_rank()
@@ -464,6 +616,11 @@ def isbn_13_to_10(isbn_13):
     return converted + str(checkdigit)
 
 
+def normalize_isbn(isbn):
+    """Remove unexpected characters from ISBN 10 or 13"""
+    return re.sub(r"[^0-9X]", "", isbn)
+
+
 # pylint: disable=unused-argument
 @receiver(models.signals.post_save, sender=Edition)
 def preview_image(instance, *args, **kwargs):

bookwyrm/models/bookwyrm_export_job.py

@@ -0,0 +1,334 @@
+"""Export user account to tar.gz file for import into another Bookwyrm instance"""
+
+import logging
+import os
+
+from boto3.session import Session as BotoSession
+from s3_tar import S3Tar
+
+from django.db.models import BooleanField, FileField, JSONField
+from django.db.models import Q
+from django.core.serializers.json import DjangoJSONEncoder
+from django.core.files.base import ContentFile
+from django.utils.module_loading import import_string
+
+from bookwyrm import settings, storage_backends
+
+from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, ListItem
+from bookwyrm.models import Review, Comment, Quotation
+from bookwyrm.models import Edition
+from bookwyrm.models import UserFollows, User, UserBlocks
+from bookwyrm.models.job import ParentJob
+from bookwyrm.tasks import app, IMPORTS
+from bookwyrm.utils.tar import BookwyrmTarFile
+
+logger = logging.getLogger(__name__)
+
+
+class BookwyrmAwsSession(BotoSession):
+    """a boto session that always uses settings.AWS_S3_ENDPOINT_URL"""
+
+    def client(self, *args, **kwargs):  # pylint: disable=arguments-differ
+        kwargs["endpoint_url"] = settings.AWS_S3_ENDPOINT_URL
+        return super().client("s3", *args, **kwargs)
+
+
+def select_exports_storage():
+    """callable to allow for dependency on runtime configuration"""
+    cls = import_string(settings.EXPORTS_STORAGE)
+    return cls()
+
+
+class BookwyrmExportJob(ParentJob):
+    """entry for a specific request to export a bookwyrm user"""
+
+    export_data = FileField(null=True, storage=select_exports_storage)
+    export_json = JSONField(null=True, encoder=DjangoJSONEncoder)
+    json_completed = BooleanField(default=False)
+
+    def start_job(self):
+        """schedule the first task"""
+
+        task = create_export_json_task.delay(job_id=self.id)
+        self.task_id = task.id
+        self.save(update_fields=["task_id"])
+
+
+@app.task(queue=IMPORTS)
+def create_export_json_task(job_id):
+    """create the JSON data for the export"""
+
+    job = BookwyrmExportJob.objects.get(id=job_id)
+
+    # don't start the job if it was stopped from the UI
+    if job.complete:
+        return
+
+    try:
+        job.set_status("active")
+
+        # generate JSON structure
+        job.export_json = export_json(job.user)
+        job.save(update_fields=["export_json"])
+
+        # create archive in separate task
+        create_archive_task.delay(job_id=job.id)
+    except Exception as err:  # pylint: disable=broad-except
+        logger.exception(
+            "create_export_json_task for %s failed with error: %s", job, err
+        )
+        job.set_status("failed")
+
+
+def archive_file_location(file, directory="") -> str:
+    """get the relative location of a file inside the archive"""
+    return os.path.join(directory, file.name)
+
+
+def add_file_to_s3_tar(s3_tar: S3Tar, storage, file, directory=""):
+    """
+    add file to S3Tar inside directory, keeping any directories under its
+    storage location
+    """
+    s3_tar.add_file(
+        os.path.join(storage.location, file.name),
+        folder=os.path.dirname(archive_file_location(file, directory=directory)),
+    )
+
+
+@app.task(queue=IMPORTS)
+def create_archive_task(job_id):
+    """create the archive containing the JSON file and additional files"""
+
+    job = BookwyrmExportJob.objects.get(id=job_id)
+
+    # don't start the job if it was stopped from the UI
+    if job.complete:
+        return
+
+    try:
+        export_task_id = str(job.task_id)
+        archive_filename = f"{export_task_id}.tar.gz"
+        export_json_bytes = DjangoJSONEncoder().encode(job.export_json).encode("utf-8")
+
+        user = job.user
+        editions = get_books_for_user(user)
+
+        if settings.USE_S3:
+            # Storage for writing temporary files
+            exports_storage = storage_backends.ExportsS3Storage()
+
+            # Handle for creating the final archive
+            s3_tar = S3Tar(
+                exports_storage.bucket_name,
+                os.path.join(exports_storage.location, archive_filename),
+                session=BookwyrmAwsSession(),
+            )
+
+            # Save JSON file to a temporary location
+            export_json_tmp_file = os.path.join(export_task_id, "archive.json")
+            exports_storage.save(
+                export_json_tmp_file,
+                ContentFile(export_json_bytes),
+            )
+            s3_tar.add_file(
+                os.path.join(exports_storage.location, export_json_tmp_file)
+            )
+
+            # Add images to TAR
+            images_storage = storage_backends.ImagesStorage()
+
+            if user.avatar:
+                add_file_to_s3_tar(s3_tar, images_storage, user.avatar)
+
+            for edition in editions:
+                if edition.cover:
+                    add_file_to_s3_tar(
+                        s3_tar, images_storage, edition.cover, directory="images"
+                    )
+
+            # Create archive and store file name
+            s3_tar.tar()
+            job.export_data = archive_filename
+            job.save(update_fields=["export_data"])
+
+            # Delete temporary files
+            exports_storage.delete(export_json_tmp_file)
+
+        else:
+            job.export_data = archive_filename
+            with job.export_data.open("wb") as tar_file:
+                with BookwyrmTarFile.open(mode="w:gz", fileobj=tar_file) as tar:
+                    # save json file
+                    tar.write_bytes(export_json_bytes)
+
+                    # Add avatar image if present
+                    if user.avatar:
+                        tar.add_image(user.avatar)
+
+                    for edition in editions:
+                        if edition.cover:
+                            tar.add_image(edition.cover, directory="images")
+            job.save(update_fields=["export_data"])
+
+        job.set_status("completed")
+
+    except Exception as err:  # pylint: disable=broad-except
+        logger.exception("create_archive_task for %s failed with error: %s", job, err)
+        job.set_status("failed")
+
+
+def export_json(user: User):
+    """create export JSON"""
+    data = export_user(user)  # in the root of the JSON structure
+    data["settings"] = export_settings(user)
+    data["goals"] = export_goals(user)
+    data["books"] = export_books(user)
+    data["saved_lists"] = export_saved_lists(user)
+    data["follows"] = export_follows(user)
+    data["blocks"] = export_blocks(user)
+    return data
+
+
+def export_user(user: User):
+    """export user data"""
+    data = user.to_activity()
+    if user.avatar:
+        data["icon"]["url"] = archive_file_location(user.avatar)
+    else:
+        data["icon"] = {}
+    return data
+
+
+def export_settings(user: User):
+    """Additional settings - can't be serialized as AP"""
+    vals = [
+        "show_goal",
+        "preferred_timezone",
+        "default_post_privacy",
+        "show_suggested_users",
+    ]
+    return {k: getattr(user, k) for k in vals}
+
+
+def export_saved_lists(user: User):
+    """add user saved lists to export JSON"""
+    return [l.remote_id for l in user.saved_lists.all()]
+
+
+def export_follows(user: User):
+    """add user follows to export JSON"""
+    follows = UserFollows.objects.filter(user_subject=user).distinct()
+    following = User.objects.filter(userfollows_user_object__in=follows).distinct()
+    return [f.remote_id for f in following]
+
+
+def export_blocks(user: User):
+    """add user blocks to export JSON"""
+    blocks = UserBlocks.objects.filter(user_subject=user).distinct()
+    blocking = User.objects.filter(userblocks_user_object__in=blocks).distinct()
+    return [b.remote_id for b in blocking]
+
+
+def export_goals(user: User):
+    """add user reading goals to export JSON"""
+    reading_goals = AnnualGoal.objects.filter(user=user).distinct()
+    return [
+        {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy}
+        for goal in reading_goals
+    ]
+
+
+def export_books(user: User):
+    """add books to export JSON"""
+    editions = get_books_for_user(user)
+    return [export_book(user, edition) for edition in editions]
+
+
+def export_book(user: User, edition: Edition):
+    """add book to export JSON"""
+    data = {}
+    data["work"] = edition.parent_work.to_activity()
+    data["edition"] = edition.to_activity()
+
+    if edition.cover:
+        data["edition"]["cover"]["url"] = archive_file_location(
+            edition.cover, directory="images"
+        )
+
+    # authors
+    data["authors"] = [author.to_activity() for author in edition.authors.all()]
+
+    # Shelves this book is on
+    # Every ShelfItem is this book so we don't other serializing
+    shelf_books = (
+        ShelfBook.objects.select_related("shelf")
+        .filter(user=user, book=edition)
+        .distinct()
+    )
+    data["shelves"] = [shelfbook.shelf.to_activity() for shelfbook in shelf_books]
+
+    # Lists and ListItems
+    # ListItems include "notes" and "approved" so we need them
+    # even though we know it's this book
+    list_items = ListItem.objects.filter(book=edition, user=user).distinct()
+
+    data["lists"] = []
+    for item in list_items:
+        list_info = item.book_list.to_activity()
+        list_info[
+            "privacy"
+        ] = item.book_list.privacy  # this isn't serialized so we add it
+        list_info["list_item"] = item.to_activity()
+        data["lists"].append(list_info)
+
+    # Statuses
+    # Can't use select_subclasses here because
+    # we need to filter on the "book" value,
+    # which is not available on an ordinary Status
+    for status in ["comments", "quotations", "reviews"]:
+        data[status] = []
+
+    comments = Comment.objects.filter(user=user, book=edition).all()
+    for status in comments:
+        obj = status.to_activity()
+        obj["progress"] = status.progress
+        obj["progress_mode"] = status.progress_mode
+        data["comments"].append(obj)
+
+    quotes = Quotation.objects.filter(user=user, book=edition).all()
+    for status in quotes:
+        obj = status.to_activity()
+        obj["position"] = status.position
+        obj["endposition"] = status.endposition
+        obj["position_mode"] = status.position_mode
+        data["quotations"].append(obj)
+
+    reviews = Review.objects.filter(user=user, book=edition).all()
+    data["reviews"] = [status.to_activity() for status in reviews]
+
+    # readthroughs can't be serialized to activity
+    book_readthroughs = (
+        ReadThrough.objects.filter(user=user, book=edition).distinct().values()
+    )
+    data["readthroughs"] = list(book_readthroughs)
+    return data
+
+
+def get_books_for_user(user):
+    """Get all the books and editions related to a user"""
+
+    editions = (
+        Edition.objects.select_related("parent_work")
+        .filter(
+            Q(shelves__user=user)
+            | Q(readthrough__user=user)
+            | Q(review__user=user)
+            | Q(list__user=user)
+            | Q(comment__user=user)
+            | Q(quotation__user=user)
+        )
+        .distinct()
+    )
+
+    return editions

bookwyrm/models/bookwyrm_import_job.py

@@ -0,0 +1,462 @@
+"""Import a user from another Bookwyrm instance"""
+
+import json
+import logging
+
+from django.db.models import FileField, JSONField, CharField
+from django.utils import timezone
+from django.utils.html import strip_tags
+from django.contrib.postgres.fields import ArrayField as DjangoArrayField
+
+from bookwyrm import activitypub
+from bookwyrm import models
+from bookwyrm.tasks import app, IMPORTS
+from bookwyrm.models.job import ParentJob, ParentTask, SubTask
+from bookwyrm.utils.tar import BookwyrmTarFile
+
+logger = logging.getLogger(__name__)
+
+
+class BookwyrmImportJob(ParentJob):
+    """entry for a specific request for importing a bookwyrm user backup"""
+
+    archive_file = FileField(null=True, blank=True)
+    import_data = JSONField(null=True)
+    required = DjangoArrayField(CharField(max_length=50, blank=True), blank=True)
+
+    def start_job(self):
+        """Start the job"""
+        start_import_task.delay(job_id=self.id, no_children=True)
+
+
+@app.task(queue=IMPORTS, base=ParentTask)
+def start_import_task(**kwargs):
+    """trigger the child import tasks for each user data"""
+    job = BookwyrmImportJob.objects.get(id=kwargs["job_id"])
+    archive_file = job.archive_file
+
+    # don't start the job if it was stopped from the UI
+    if job.complete:
+        return
+
+    try:
+        archive_file.open("rb")
+        with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar:
+            json_filename = next(
+                filter(lambda n: n.startswith("archive"), tar.getnames())
+            )
+            job.import_data = json.loads(tar.read(json_filename).decode("utf-8"))
+
+            if "include_user_profile" in job.required:
+                update_user_profile(job.user, tar, job.import_data)
+            if "include_user_settings" in job.required:
+                update_user_settings(job.user, job.import_data)
+            if "include_goals" in job.required:
+                update_goals(job.user, job.import_data.get("goals", []))
+            if "include_saved_lists" in job.required:
+                upsert_saved_lists(job.user, job.import_data.get("saved_lists", []))
+            if "include_follows" in job.required:
+                upsert_follows(job.user, job.import_data.get("follows", []))
+            if "include_blocks" in job.required:
+                upsert_user_blocks(job.user, job.import_data.get("blocks", []))
+
+            process_books(job, tar)
+
+            job.set_status("complete")
+        archive_file.close()
+
+    except Exception as err:  # pylint: disable=broad-except
+        logger.exception("User Import Job %s Failed with error: %s", job.id, err)
+        job.set_status("failed")
+
+
+def process_books(job, tar):
+    """
+    Process user import data related to books
+    We always import the books even if not assigning
+    them to shelves, lists etc
+    """
+
+    books = job.import_data.get("books")
+
+    for data in books:
+        book = get_or_create_edition(data, tar)
+
+        if "include_shelves" in job.required:
+            upsert_shelves(book, job.user, data)
+
+        if "include_readthroughs" in job.required:
+            upsert_readthroughs(data.get("readthroughs"), job.user, book.id)
+
+        if "include_comments" in job.required:
+            upsert_statuses(
+                job.user, models.Comment, data.get("comments"), book.remote_id
+            )
+        if "include_quotations" in job.required:
+            upsert_statuses(
+                job.user, models.Quotation, data.get("quotations"), book.remote_id
+            )
+
+        if "include_reviews" in job.required:
+            upsert_statuses(
+                job.user, models.Review, data.get("reviews"), book.remote_id
+            )
+
+        if "include_lists" in job.required:
+            upsert_lists(job.user, data.get("lists"), book.id)
+
+
+def get_or_create_edition(book_data, tar):
+    """Take a JSON string of work and edition data,
+    find or create the edition and work in the database and
+    return an edition instance"""
+
+    edition = book_data.get("edition")
+    existing = models.Edition.find_existing(edition)
+    if existing:
+        return existing
+
+    # make sure we have the authors in the local DB
+    # replace the old author ids in the edition JSON
+    edition["authors"] = []
+    for author in book_data.get("authors"):
+        parsed_author = activitypub.parse(author)
+        instance = parsed_author.to_model(
+            model=models.Author, save=True, overwrite=True
+        )
+
+        edition["authors"].append(instance.remote_id)
+
+    # we will add the cover later from the tar
+    # don't try to load it from the old server
+    cover = edition.get("cover", {})
+    cover_path = cover.get("url", None)
+    edition["cover"] = {}
+
+    # first we need the parent work to exist
+    work = book_data.get("work")
+    work["editions"] = []
+    parsed_work = activitypub.parse(work)
+    work_instance = parsed_work.to_model(model=models.Work, save=True, overwrite=True)
+
+    # now we have a work we can add it to the edition
+    # and create the edition model instance
+    edition["work"] = work_instance.remote_id
+    parsed_edition = activitypub.parse(edition)
+    book = parsed_edition.to_model(model=models.Edition, save=True, overwrite=True)
+
+    # set the cover image from the tar
+    if cover_path:
+        tar.write_image_to_file(cover_path, book.cover)
+
+    return book
+
+
+def upsert_readthroughs(data, user, book_id):
+    """Take a JSON string of readthroughs and
+    find or create the instances in the database"""
+
+    for read_through in data:
+
+        obj = {}
+        keys = [
+            "progress_mode",
+            "start_date",
+            "finish_date",
+            "stopped_date",
+            "is_active",
+        ]
+        for key in keys:
+            obj[key] = read_through[key]
+        obj["user_id"] = user.id
+        obj["book_id"] = book_id
+
+        existing = models.ReadThrough.objects.filter(**obj).first()
+        if not existing:
+            models.ReadThrough.objects.create(**obj)
+
+
+def upsert_statuses(user, cls, data, book_remote_id):
+    """Take a JSON string of a status and
+    find or create the instances in the database"""
+
+    for status in data:
+        if is_alias(
+            user, status["attributedTo"]
+        ):  # don't let l33t hax0rs steal other people's posts
+            # update ids and remove replies
+            status["attributedTo"] = user.remote_id
+            status["to"] = update_followers_address(user, status["to"])
+            status["cc"] = update_followers_address(user, status["cc"])
+            status[
+                "replies"
+            ] = (
+                {}
+            )  # this parses incorrectly but we can't set it without knowing the new id
+            status["inReplyToBook"] = book_remote_id
+            parsed = activitypub.parse(status)
+            if not status_already_exists(
+                user, parsed
+            ):  # don't duplicate posts on multiple import
+
+                instance = parsed.to_model(model=cls, save=True, overwrite=True)
+
+                for val in [
+                    "progress",
+                    "progress_mode",
+                    "position",
+                    "endposition",
+                    "position_mode",
+                ]:
+                    if status.get(val):
+                        instance.val = status[val]
+
+                instance.remote_id = instance.get_remote_id()  # update the remote_id
+                instance.save()  # save and broadcast
+
+        else:
+            logger.warning("User does not have permission to import statuses")
+
+
+def upsert_lists(user, lists, book_id):
+    """Take a list of objects each containing
+    a list and list item as AP objects
+
+    Because we are creating new IDs we can't assume the id
+    will exist or be accurate, so we only use to_model for
+    adding new items after checking whether they exist  .
+
+    """
+
+    book = models.Edition.objects.get(id=book_id)
+
+    for blist in lists:
+        booklist = models.List.objects.filter(name=blist["name"], user=user).first()
+        if not booklist:
+
+            blist["owner"] = user.remote_id
+            parsed = activitypub.parse(blist)
+            booklist = parsed.to_model(model=models.List, save=True, overwrite=True)
+
+            booklist.privacy = blist["privacy"]
+            booklist.save()
+
+        item = models.ListItem.objects.filter(book=book, book_list=booklist).exists()
+        if not item:
+            count = booklist.books.count()
+            models.ListItem.objects.create(
+                book=book,
+                book_list=booklist,
+                user=user,
+                notes=blist["list_item"]["notes"],
+                approved=blist["list_item"]["approved"],
+                order=count + 1,
+            )
+
+
+def upsert_shelves(book, user, book_data):
+    """Take shelf JSON objects and create
+    DB entries if they don't already exist"""
+
+    shelves = book_data["shelves"]
+    for shelf in shelves:
+
+        book_shelf = models.Shelf.objects.filter(name=shelf["name"], user=user).first()
+
+        if not book_shelf:
+            book_shelf = models.Shelf.objects.create(name=shelf["name"], user=user)
+
+        # add the book as a ShelfBook if needed
+        if not models.ShelfBook.objects.filter(
+            book=book, shelf=book_shelf, user=user
+        ).exists():
+            models.ShelfBook.objects.create(
+                book=book, shelf=book_shelf, user=user, shelved_date=timezone.now()
+            )
+
+
+def update_user_profile(user, tar, data):
+    """update the user's profile from import data"""
+    name = data.get("name", None)
+    username = data.get("preferredUsername")
+    user.name = name if name else username
+    user.summary = strip_tags(data.get("summary", None))
+    user.save(update_fields=["name", "summary"])
+    if data["icon"].get("url"):
+        avatar_filename = next(filter(lambda n: n.startswith("avatar"), tar.getnames()))
+        tar.write_image_to_file(avatar_filename, user.avatar)
+
+
+def update_user_settings(user, data):
+    """update the user's settings from import data"""
+
+    update_fields = ["manually_approves_followers", "hide_follows", "discoverable"]
+
+    ap_fields = [
+        ("manuallyApprovesFollowers", "manually_approves_followers"),
+        ("hideFollows", "hide_follows"),
+        ("discoverable", "discoverable"),
+    ]
+
+    for (ap_field, bw_field) in ap_fields:
+        setattr(user, bw_field, data[ap_field])
+
+    bw_fields = [
+        "show_goal",
+        "show_suggested_users",
+        "default_post_privacy",
+        "preferred_timezone",
+    ]
+
+    for field in bw_fields:
+        update_fields.append(field)
+        setattr(user, field, data["settings"][field])
+
+    user.save(update_fields=update_fields)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def update_user_settings_task(job_id):
+    """wrapper task for user's settings import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return update_user_settings(parent_job.user, parent_job.import_data.get("user"))
+
+
+def update_goals(user, data):
+    """update the user's goals from import data"""
+
+    for goal in data:
+        # edit the existing goal if there is one
+        existing = models.AnnualGoal.objects.filter(
+            year=goal["year"], user=user
+        ).first()
+        if existing:
+            for k in goal.keys():
+                setattr(existing, k, goal[k])
+            existing.save()
+        else:
+            goal["user"] = user
+            models.AnnualGoal.objects.create(**goal)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def update_goals_task(job_id):
+    """wrapper task for user's goals import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return update_goals(parent_job.user, parent_job.import_data.get("goals"))
+
+
+def upsert_saved_lists(user, values):
+    """Take a list of remote ids and add as saved lists"""
+
+    for remote_id in values:
+        book_list = activitypub.resolve_remote_id(remote_id, models.List)
+        if book_list:
+            user.saved_lists.add(book_list)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def upsert_saved_lists_task(job_id):
+    """wrapper task for user's saved lists import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return upsert_saved_lists(
+        parent_job.user, parent_job.import_data.get("saved_lists")
+    )
+
+
+def upsert_follows(user, values):
+    """Take a list of remote ids and add as follows"""
+
+    for remote_id in values:
+        followee = activitypub.resolve_remote_id(remote_id, models.User)
+        if followee:
+            (follow_request, created,) = models.UserFollowRequest.objects.get_or_create(
+                user_subject=user,
+                user_object=followee,
+            )
+
+            if not created:
+                # this request probably failed to connect with the remote
+                # and should save to trigger a re-broadcast
+                follow_request.save()
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def upsert_follows_task(job_id):
+    """wrapper task for user's follows import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return upsert_follows(parent_job.user, parent_job.import_data.get("follows"))
+
+
+def upsert_user_blocks(user, user_ids):
+    """block users"""
+
+    for user_id in user_ids:
+        user_object = activitypub.resolve_remote_id(user_id, models.User)
+        if user_object:
+            exists = models.UserBlocks.objects.filter(
+                user_subject=user, user_object=user_object
+            ).exists()
+            if not exists:
+                models.UserBlocks.objects.create(
+                    user_subject=user, user_object=user_object
+                )
+                # remove the blocked users's lists from the groups
+                models.List.remove_from_group(user, user_object)
+                # remove the blocked user from all blocker's owned groups
+                models.GroupMember.remove(user, user_object)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def upsert_user_blocks_task(job_id):
+    """wrapper task for user's blocks import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return upsert_user_blocks(
+        parent_job.user, parent_job.import_data.get("blocked_users")
+    )
+
+
+def update_followers_address(user, field):
+    """statuses to or cc followers need to have the followers
+    address updated to the new local user"""
+
+    for i, audience in enumerate(field):
+        if audience.rsplit("/")[-1] == "followers":
+            field[i] = user.followers_url
+
+    return field
+
+
+def is_alias(user, remote_id):
+    """check that the user is listed as movedTo or also_known_as
+    in the remote user's profile"""
+
+    remote_user = activitypub.resolve_remote_id(
+        remote_id=remote_id, model=models.User, save=False
+    )
+
+    if remote_user:
+
+        if remote_user.moved_to:
+            return user.remote_id == remote_user.moved_to
+
+        if remote_user.also_known_as:
+            return user in remote_user.also_known_as.all()
+
+    return False
+
+
+def status_already_exists(user, status):
+    """check whether this status has already been published
+    by this user. We can't rely on to_model() because it
+    only matches on remote_id, which we have to change
+    *after* saving because it needs the primary key (id)"""
+
+    return models.Status.objects.filter(
+        user=user, content=status.content, published_date=status.published
+    ).exists()

bookwyrm/models/connector.py

@@ -11,7 +11,7 @@ ConnectorFiles = models.TextChoices("ConnectorFiles", CONNECTORS)
 class Connector(BookWyrmModel):
     """book data source connectors"""
 
-    identifier = models.CharField(max_length=255, unique=True)
+    identifier = models.CharField(max_length=255, unique=True)  # domain
     priority = models.IntegerField(default=2)
     name = models.CharField(max_length=255, null=True, blank=True)
     connector_file = models.CharField(max_length=255, choices=ConnectorFiles.choices)

bookwyrm/models/federated_server.py

@@ -16,7 +16,7 @@ FederationStatus = [
 class FederatedServer(BookWyrmModel):
     """store which servers we federate with"""
 
-    server_name = models.CharField(max_length=255, unique=True)
+    server_name = models.CharField(max_length=255, unique=True)  # domain
     status = models.CharField(
         max_length=255, default="federated", choices=FederationStatus
     )
@@ -64,5 +64,4 @@ class FederatedServer(BookWyrmModel):
     def is_blocked(cls, url: str) -> bool:
         """look up if a domain is blocked"""
         url = urlparse(url)
-        domain = url.netloc
-        return cls.objects.filter(server_name=domain, status="blocked").exists()
+        return cls.objects.filter(server_name=url.hostname, status="blocked").exists()

bookwyrm/models/fields.py

@@ -1,5 +1,6 @@
 """ activitypub-aware django model fields """
 from dataclasses import MISSING
+from datetime import datetime
 import re
 from uuid import uuid4
 from urllib.parse import urljoin
@@ -19,6 +20,11 @@ from markdown import markdown
 from bookwyrm import activitypub
 from bookwyrm.connectors import get_image
 from bookwyrm.utils.sanitizer import clean
+from bookwyrm.utils.partial_date import (
+    PartialDate,
+    PartialDateModel,
+    from_partial_isoformat,
+)
 from bookwyrm.settings import MEDIA_FULL_URL
 
 
@@ -254,12 +260,12 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
 
         if to == [self.public]:
             setattr(instance, self.name, "public")
+        elif self.public in cc:
+            setattr(instance, self.name, "unlisted")
         elif to == [user.followers_url]:
             setattr(instance, self.name, "followers")
         elif cc == []:
             setattr(instance, self.name, "direct")
-        elif self.public in cc:
-            setattr(instance, self.name, "unlisted")
         else:
             setattr(instance, self.name, "followers")
         return original == getattr(instance, self.name)
@@ -476,16 +482,18 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
         if not url:
             return None
 
-        return activitypub.Document(url=url, name=alt)
+        return activitypub.Image(url=url, name=alt)
 
     def field_from_activity(self, value, allow_external_connections=True):
         image_slug = value
         # when it's an inline image (User avatar/icon, Book cover), it's a json
         # blob, but when it's an attached image, it's just a url
-        if hasattr(image_slug, "url"):
-            url = image_slug.url
-        elif isinstance(image_slug, str):
+        if isinstance(image_slug, str):
             url = image_slug
+        elif isinstance(image_slug, dict):
+            url = image_slug.get("url")
+        elif hasattr(image_slug, "url"):  # Serialized to Image/Document object?
+            url = image_slug.url
         else:
             return None
 
@@ -534,8 +542,9 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
         return value.isoformat()
 
     def field_from_activity(self, value, allow_external_connections=True):
+        missing_fields = datetime(1970, 1, 1)  # "2022-10" => "2022-10-01"
         try:
-            date_value = dateutil.parser.parse(value)
+            date_value = dateutil.parser.parse(value, default=missing_fields)
             try:
                 return timezone.make_aware(date_value)
             except ValueError:
@@ -544,6 +553,37 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
             return None
 
 
+class PartialDateField(ActivitypubFieldMixin, PartialDateModel):
+    """activitypub-aware partial date field"""
+
+    def field_to_activity(self, value) -> str:
+        return value.partial_isoformat() if value else None
+
+    def field_from_activity(self, value, allow_external_connections=True):
+        # pylint: disable=no-else-return
+        try:
+            return from_partial_isoformat(value)
+        except ValueError:
+            pass
+
+        # fallback to full ISO-8601 parsing
+        try:
+            parsed = dateutil.parser.isoparse(value)
+        except (ValueError, ParserError):
+            return None
+
+        if timezone.is_aware(parsed):
+            return PartialDate.from_datetime(parsed)
+        else:
+            # Should not happen on the wire, but truncate down to date parts.
+            return PartialDate.from_date_parts(parsed.year, parsed.month, parsed.day)
+
+        # FIXME: decide whether to fix timestamps like "2023-09-30T21:00:00-03":
+        # clearly Oct 1st, not Sep 30th (an unwanted side-effect of USE_TZ). It's
+        # basically the remnants of #3028; there is a data migration pending (see …)
+        # but over the wire we might get these for an indeterminate amount of time.
+
+
 class HtmlField(ActivitypubFieldMixin, models.TextField):
     """a text field for storing html"""
 

bookwyrm/models/group.py

@@ -1,8 +1,7 @@
 """ do book related things with other users """
-from django.apps import apps
 from django.db import models, IntegrityError, transaction
 from django.db.models import Q
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from .base_model import BookWyrmModel
 from . import fields
 from .relationship import UserBlocks
@@ -18,7 +17,7 @@ class Group(BookWyrmModel):
 
     def get_remote_id(self):
         """don't want the user to be in there in this case"""
-        return f"https://{DOMAIN}/group/{self.id}"
+        return f"{BASE_URL}/group/{self.id}"
 
     @classmethod
     def followers_filter(cls, queryset, viewer):
@@ -143,26 +142,28 @@ class GroupMemberInvitation(models.Model):
     @transaction.atomic
     def accept(self):
         """turn this request into the real deal"""
+        # pylint: disable-next=import-outside-toplevel
+        from .notification import Notification, NotificationType  # circular dependency
+
         GroupMember.from_request(self)
 
-        model = apps.get_model("bookwyrm.Notification", require_ready=True)
         # tell the group owner
-        model.notify(
+        Notification.notify(
             self.group.user,
             self.user,
             related_group=self.group,
-            notification_type=model.ACCEPT,
+            notification_type=NotificationType.ACCEPT,
         )
 
         # let the other members know about it
         for membership in self.group.memberships.all():
             member = membership.user
             if member not in (self.user, self.group.user):
-                model.notify(
+                Notification.notify(
                     member,
                     self.user,
                     related_group=self.group,
-                    notification_type=model.JOIN,
+                    notification_type=NotificationType.JOIN,
                 )
 
     def reject(self):

bookwyrm/models/import_job.py

@@ -1,4 +1,5 @@
 """ track progress of goodreads imports """
+from datetime import datetime
 import math
 import re
 import dateutil.parser
@@ -259,38 +260,30 @@ class ImportItem(models.Model):
         except ValueError:
             return None
 
+    def _parse_datefield(self, field, /):
+        if not (date := self.normalized_data.get(field)):
+            return None
+
+        defaults = datetime(1970, 1, 1)  # "2022-10" => "2022-10-01"
+        parsed = dateutil.parser.parse(date, default=defaults)
+
+        # Keep timezone if import already had one, else use default.
+        return parsed if timezone.is_aware(parsed) else timezone.make_aware(parsed)
+
     @property
     def date_added(self):
         """when the book was added to this dataset"""
-        if self.normalized_data.get("date_added"):
-            parsed_date_added = dateutil.parser.parse(
-                self.normalized_data.get("date_added")
-            )
-
-            if timezone.is_aware(parsed_date_added):
-                # Keep timezone if import already had one
-                return parsed_date_added
-
-            return timezone.make_aware(parsed_date_added)
-        return None
+        return self._parse_datefield("date_added")
 
     @property
     def date_started(self):
         """when the book was started"""
-        if self.normalized_data.get("date_started"):
-            return timezone.make_aware(
-                dateutil.parser.parse(self.normalized_data.get("date_started"))
-            )
-        return None
+        return self._parse_datefield("date_started")
 
     @property
     def date_read(self):
         """the date a book was completed"""
-        if self.normalized_data.get("date_finished"):
-            return timezone.make_aware(
-                dateutil.parser.parse(self.normalized_data.get("date_finished"))
-            )
-        return None
+        return self._parse_datefield("date_finished")
 
     @property
     def reads(self):

bookwyrm/models/job.py

@@ -0,0 +1,307 @@
+"""Everything needed for Celery to multi-thread complex tasks."""
+
+from django.db import models
+from django.db import transaction
+from django.utils.translation import gettext_lazy as _
+from django.utils import timezone
+from bookwyrm.models.user import User
+
+from bookwyrm.tasks import app
+
+
+class Job(models.Model):
+    """Abstract model to store the state of a Task."""
+
+    class Status(models.TextChoices):
+        """Possible job states."""
+
+        PENDING = "pending", _("Pending")
+        ACTIVE = "active", _("Active")
+        COMPLETE = "complete", _("Complete")
+        STOPPED = "stopped", _("Stopped")
+        FAILED = "failed", _("Failed")
+
+    task_id = models.UUIDField(unique=True, null=True, blank=True)
+
+    created_date = models.DateTimeField(default=timezone.now)
+    updated_date = models.DateTimeField(default=timezone.now)
+    complete = models.BooleanField(default=False)
+    status = models.CharField(
+        max_length=50, choices=Status.choices, default=Status.PENDING, null=True
+    )
+
+    class Meta:
+        """Make it abstract"""
+
+        abstract = True
+
+    def complete_job(self):
+        """Report that the job has completed"""
+        if self.complete:
+            return
+
+        self.status = self.Status.COMPLETE
+        self.complete = True
+        self.updated_date = timezone.now()
+
+        self.save(update_fields=["status", "complete", "updated_date"])
+
+    def stop_job(self, reason=None):
+        """Stop the job"""
+        if self.complete:
+            return
+
+        self.__terminate_job()
+
+        if reason and reason == "failed":
+            self.status = self.Status.FAILED
+        else:
+            self.status = self.Status.STOPPED
+        self.complete = True
+        self.updated_date = timezone.now()
+
+        self.save(update_fields=["status", "complete", "updated_date"])
+
+    def set_status(self, status):
+        """Set job status"""
+        if self.complete:
+            return
+
+        if self.status == status:
+            return
+
+        if status == self.Status.COMPLETE:
+            self.complete_job()
+            return
+
+        if status == self.Status.STOPPED:
+            self.stop_job()
+            return
+
+        if status == self.Status.FAILED:
+            self.stop_job(reason="failed")
+            return
+
+        self.updated_date = timezone.now()
+        self.status = status
+
+        self.save(update_fields=["status", "updated_date"])
+
+    def __terminate_job(self):
+        """Tell workers to ignore and not execute this task."""
+        app.control.revoke(self.task_id, terminate=True)
+
+
+class ParentJob(Job):
+    """Store the state of a Task which can spawn many :model:`ChildJob`s to spread
+    resource load.
+
+    Intended to be sub-classed if necessary via proxy or
+    multi-table inheritance.
+    Extends :model:`Job`.
+    """
+
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+
+    def complete_job(self):
+        """Report that the job has completed and stop pending
+        children. Extend.
+        """
+        super().complete_job()
+        self.__terminate_pending_child_jobs()
+
+    def notify_child_job_complete(self):
+        """let the job know when the items get work done"""
+        if self.complete:
+            return
+
+        self.updated_date = timezone.now()
+        self.save(update_fields=["updated_date"])
+
+        if not self.complete and self.has_completed:
+            self.complete_job()
+
+    def __terminate_job(self):  # pylint: disable=unused-private-member
+        """Tell workers to ignore and not execute this task
+        & pending child tasks. Extend.
+        """
+        super().__terminate_job()
+        self.__terminate_pending_child_jobs()
+
+    def __terminate_pending_child_jobs(self):
+        """Tell workers to ignore and not execute any pending child tasks."""
+        tasks = self.pending_child_jobs.filter(task_id__isnull=False).values_list(
+            "task_id", flat=True
+        )
+        app.control.revoke(list(tasks))
+
+        self.pending_child_jobs.update(status=self.Status.STOPPED)
+
+    @property
+    def has_completed(self):
+        """has this job finished"""
+        return not self.pending_child_jobs.exists()
+
+    @property
+    def pending_child_jobs(self):
+        """items that haven't been processed yet"""
+        return self.child_jobs.filter(complete=False)
+
+
+class ChildJob(Job):
+    """Stores the state of a Task for the related :model:`ParentJob`.
+
+    Intended to be sub-classed if necessary via proxy or
+    multi-table inheritance.
+    Extends :model:`Job`.
+    """
+
+    parent_job = models.ForeignKey(
+        ParentJob, on_delete=models.CASCADE, related_name="child_jobs"
+    )
+
+    def set_status(self, status):
+        """Set job and parent_job status. Extend."""
+        super().set_status(status)
+
+        if (
+            status == self.Status.ACTIVE
+            and self.parent_job.status == self.Status.PENDING
+        ):
+            self.parent_job.set_status(self.Status.ACTIVE)
+
+    def complete_job(self):
+        """Report to parent_job that the job has completed. Extend."""
+        super().complete_job()
+        self.parent_job.notify_child_job_complete()
+
+
+class ParentTask(app.Task):
+    """Used with ParentJob, Abstract Tasks execute code at specific points in
+    a Task's lifecycle, applying to all Tasks with the same 'base'.
+
+    All status & ParentJob.task_id assignment is managed here for you.
+    Usage e.g. @app.task(base=ParentTask)
+    """
+
+    def before_start(
+        self, task_id, args, kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Handler called before the task starts. Override.
+
+        Prepare ParentJob before the task starts.
+
+        Arguments:
+            task_id (str): Unique id of the task to execute.
+            args (Tuple): Original arguments for the task to execute.
+            kwargs (Dict): Original keyword arguments for the task to execute.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            no_children (bool): If 'True' this is the only Task expected to run
+                for the given ParentJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+        job = ParentJob.objects.get(id=kwargs["job_id"])
+        job.task_id = task_id
+        job.save(update_fields=["task_id"])
+
+        if kwargs["no_children"]:
+            job.set_status(ChildJob.Status.ACTIVE)
+
+    def on_success(
+        self, retval, task_id, args, kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Run by the worker if the task executes successfully. Override.
+
+        Update ParentJob on Task complete.
+
+        Arguments:
+            retval (Any): The return value of the task.
+            task_id (str): Unique id of the executed task.
+            args (Tuple): Original arguments for the executed task.
+            kwargs (Dict): Original keyword arguments for the executed task.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            no_children (bool): If 'True' this is the only Task expected to run
+                for the given ParentJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+
+        if kwargs["no_children"]:
+            job = ParentJob.objects.get(id=kwargs["job_id"])
+            job.complete_job()
+
+
+class SubTask(app.Task):
+    """Used with ChildJob, Abstract Tasks execute code at specific points in
+    a Task's lifecycle, applying to all Tasks with the same 'base'.
+
+    All status & ChildJob.task_id assignment is managed here for you.
+    Usage e.g. @app.task(base=SubTask)
+    """
+
+    def before_start(
+        self, task_id, *args, **kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Handler called before the task starts. Override.
+
+        Prepare ChildJob before the task starts.
+
+        Arguments:
+            task_id (str): Unique id of the task to execute.
+            args (Tuple): Original arguments for the task to execute.
+            kwargs (Dict): Original keyword arguments for the task to execute.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            child_id (int): Unique 'id' of the ChildJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+        child_job = ChildJob.objects.get(id=kwargs["child_id"])
+        child_job.task_id = task_id
+        child_job.save(update_fields=["task_id"])
+        child_job.set_status(ChildJob.Status.ACTIVE)
+
+    def on_success(
+        self, retval, task_id, *args, **kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Run by the worker if the task executes successfully. Override.
+
+        Notify ChildJob of task completion.
+
+        Arguments:
+            retval (Any): The return value of the task.
+            task_id (str): Unique id of the executed task.
+            args (Tuple): Original arguments for the executed task.
+            kwargs (Dict): Original keyword arguments for the executed task.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            child_id (int): Unique 'id' of the ChildJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+        subtask = ChildJob.objects.get(id=kwargs["child_id"])
+        subtask.complete_job()
+
+
+@transaction.atomic
+def create_child_job(parent_job, task_callback):
+    """Utility method for creating a ChildJob
+    and running a task to avoid DB race conditions
+    """
+    child_job = ChildJob.objects.create(parent_job=parent_job)
+    transaction.on_commit(
+        lambda: task_callback.delay(job_id=parent_job.id, child_id=child_job.id)
+    )
+
+    return child_job

bookwyrm/models/link.py

@@ -38,7 +38,7 @@ class Link(ActivitypubMixin, BookWyrmModel):
         """create a link"""
         # get or create the associated domain
         if not self.domain:
-            domain = urlparse(self.url).netloc
+            domain = urlparse(self.url).hostname
             self.domain, _ = LinkDomain.objects.get_or_create(domain=domain)
 
         # this is never broadcast, the owning model broadcasts an update

bookwyrm/models/list.py

@@ -7,7 +7,7 @@ from django.db.models import Q
 from django.utils import timezone
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 
 from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin
 from .base_model import BookWyrmModel
@@ -50,7 +50,7 @@ class List(OrderedCollectionMixin, BookWyrmModel):
 
     def get_remote_id(self):
         """don't want the user to be in there in this case"""
-        return f"https://{DOMAIN}/list/{self.id}"
+        return f"{BASE_URL}/list/{self.id}"
 
     @property
     def collection_queryset(self):

bookwyrm/models/move.py

@@ -0,0 +1,71 @@
+""" move an object including migrating a user account """
+from django.core.exceptions import PermissionDenied
+from django.db import models
+
+from bookwyrm import activitypub
+from .activitypub_mixin import ActivityMixin
+from .base_model import BookWyrmModel
+from . import fields
+from .notification import Notification, NotificationType
+
+
+class Move(ActivityMixin, BookWyrmModel):
+    """migrating an activitypub object"""
+
+    user = fields.ForeignKey(
+        "User", on_delete=models.PROTECT, activitypub_field="actor"
+    )
+
+    object = fields.CharField(
+        max_length=255,
+        blank=False,
+        null=False,
+        activitypub_field="object",
+    )
+
+    origin = fields.CharField(
+        max_length=255,
+        blank=True,
+        null=True,
+        default="",
+        activitypub_field="origin",
+    )
+
+    activity_serializer = activitypub.Move
+
+
+class MoveUser(Move):
+    """migrating an activitypub user account"""
+
+    target = fields.ForeignKey(
+        "User",
+        on_delete=models.PROTECT,
+        related_name="move_target",
+        activitypub_field="target",
+    )
+
+    def save(self, *args, **kwargs):
+        """update user info and broadcast it"""
+
+        # only allow if the source is listed in the target's alsoKnownAs
+        if self.user in self.target.also_known_as.all():
+            self.user.also_known_as.add(self.target.id)
+            self.user.update_active_date()
+            self.user.moved_to = self.target.remote_id
+            self.user.save(update_fields=["moved_to"])
+
+            if self.user.local:
+                kwargs[
+                    "broadcast"
+                ] = True  # Only broadcast if we are initiating the Move
+
+            super().save(*args, **kwargs)
+
+            for follower in self.user.followers.all():
+                if follower.local:
+                    Notification.notify(
+                        follower, self.user, notification_type=NotificationType.MOVE
+                    )
+
+        else:
+            raise PermissionDenied()

bookwyrm/models/notification.py

@@ -1,12 +1,21 @@
 """ alert a user to activity """
 from django.db import models, transaction
 from django.dispatch import receiver
+from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob
 from .base_model import BookWyrmModel
-from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain
+from . import (
+    Boost,
+    Favorite,
+    GroupMemberInvitation,
+    ImportJob,
+    BookwyrmImportJob,
+    LinkDomain,
+)
 from . import ListItem, Report, Status, User, UserFollowRequest
+from .site import InviteRequest
 
 
-class Notification(BookWyrmModel):
+class NotificationType(models.TextChoices):
     """you've been tagged, liked, followed, etc"""
 
     # Status interactions
@@ -22,6 +31,8 @@ class Notification(BookWyrmModel):
 
     # Imports
     IMPORT = "IMPORT"
+    USER_IMPORT = "USER_IMPORT"
+    USER_EXPORT = "USER_EXPORT"
 
     # List activity
     ADD = "ADD"
@@ -29,6 +40,7 @@ class Notification(BookWyrmModel):
     # Admin
     REPORT = "REPORT"
     LINK_DOMAIN = "LINK_DOMAIN"
+    INVITE_REQUEST = "INVITE_REQUEST"
 
     # Groups
     INVITE = "INVITE"
@@ -40,12 +52,12 @@ class Notification(BookWyrmModel):
     GROUP_NAME = "GROUP_NAME"
     GROUP_DESCRIPTION = "GROUP_DESCRIPTION"
 
-    # pylint: disable=line-too-long
-    NotificationType = models.TextChoices(
-        # there has got be a better way to do this
-        "NotificationType",
-        f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}",
-    )
+    # Migrations
+    MOVE = "MOVE"
+
+
+class Notification(BookWyrmModel):
+    """a notification object"""
 
     user = models.ForeignKey("User", on_delete=models.CASCADE)
     read = models.BooleanField(default=False)
@@ -61,11 +73,15 @@ class Notification(BookWyrmModel):
     )
     related_status = models.ForeignKey("Status", on_delete=models.CASCADE, null=True)
     related_import = models.ForeignKey("ImportJob", on_delete=models.CASCADE, null=True)
+    related_user_export = models.ForeignKey(
+        "BookwyrmExportJob", on_delete=models.CASCADE, null=True
+    )
     related_list_items = models.ManyToManyField(
         "ListItem", symmetrical=False, related_name="notifications"
     )
-    related_reports = models.ManyToManyField("Report", symmetrical=False)
-    related_link_domains = models.ManyToManyField("LinkDomain", symmetrical=False)
+    related_reports = models.ManyToManyField("Report")
+    related_link_domains = models.ManyToManyField("LinkDomain")
+    related_invite_requests = models.ManyToManyField("InviteRequest")
 
     @classmethod
     @transaction.atomic
@@ -90,11 +106,11 @@ class Notification(BookWyrmModel):
             user=user,
             related_users=related_user,
             related_list_items__book_list=list_item.book_list,
-            notification_type=Notification.ADD,
+            notification_type=NotificationType.ADD,
         ).first()
         if not notification:
             notification = cls.objects.create(
-                user=user, notification_type=Notification.ADD
+                user=user, notification_type=NotificationType.ADD
             )
             notification.related_users.add(related_user)
         notification.related_list_items.add(list_item)
@@ -121,7 +137,7 @@ def notify_on_fav(sender, instance, *args, **kwargs):
         instance.status.user,
         instance.user,
         related_status=instance.status,
-        notification_type=Notification.FAVORITE,
+        notification_type=NotificationType.FAVORITE,
     )
 
 
@@ -135,7 +151,7 @@ def notify_on_unfav(sender, instance, *args, **kwargs):
         instance.status.user,
         instance.user,
         related_status=instance.status,
-        notification_type=Notification.FAVORITE,
+        notification_type=NotificationType.FAVORITE,
     )
 
 
@@ -160,7 +176,7 @@ def notify_user_on_mention(sender, instance, *args, **kwargs):
             instance.reply_parent.user,
             instance.user,
             related_status=instance,
-            notification_type=Notification.REPLY,
+            notification_type=NotificationType.REPLY,
         )
 
     for mention_user in instance.mention_users.all():
@@ -172,7 +188,7 @@ def notify_user_on_mention(sender, instance, *args, **kwargs):
         Notification.notify(
             mention_user,
             instance.user,
-            notification_type=Notification.MENTION,
+            notification_type=NotificationType.MENTION,
             related_status=instance,
         )
 
@@ -191,7 +207,7 @@ def notify_user_on_boost(sender, instance, *args, **kwargs):
         instance.boosted_status.user,
         instance.user,
         related_status=instance.boosted_status,
-        notification_type=Notification.BOOST,
+        notification_type=NotificationType.BOOST,
     )
 
 
@@ -203,7 +219,7 @@ def notify_user_on_unboost(sender, instance, *args, **kwargs):
         instance.boosted_status.user,
         instance.user,
         related_status=instance.boosted_status,
-        notification_type=Notification.BOOST,
+        notification_type=NotificationType.BOOST,
     )
 
 
@@ -218,11 +234,41 @@ def notify_user_on_import_complete(
         return
     Notification.objects.get_or_create(
         user=instance.user,
-        notification_type=Notification.IMPORT,
+        notification_type=NotificationType.IMPORT,
         related_import=instance,
     )
 
 
+@receiver(models.signals.post_save, sender=BookwyrmImportJob)
+# pylint: disable=unused-argument
+def notify_user_on_user_import_complete(
+    sender, instance, *args, update_fields=None, **kwargs
+):
+    """we imported your user details! aren't you proud of us"""
+    update_fields = update_fields or []
+    if not instance.complete or "complete" not in update_fields:
+        return
+    Notification.objects.create(
+        user=instance.user, notification_type=NotificationType.USER_IMPORT
+    )
+
+
+@receiver(models.signals.post_save, sender=BookwyrmExportJob)
+# pylint: disable=unused-argument
+def notify_user_on_user_export_complete(
+    sender, instance, *args, update_fields=None, **kwargs
+):
+    """we exported your user details! aren't you proud of us"""
+    update_fields = update_fields or []
+    if not instance.complete or "complete" not in update_fields:
+        return
+    Notification.objects.create(
+        user=instance.user,
+        notification_type=NotificationType.USER_EXPORT,
+        related_user_export=instance,
+    )
+
+
 @receiver(models.signals.post_save, sender=Report)
 @transaction.atomic
 # pylint: disable=unused-argument
@@ -233,11 +279,10 @@ def notify_admins_on_report(sender, instance, created, *args, **kwargs):
         return
 
     # moderators and superusers should be notified
-    admins = User.admins()
-    for admin in admins:
+    for admin in User.admins():
         notification, _ = Notification.objects.get_or_create(
             user=admin,
-            notification_type=Notification.REPORT,
+            notification_type=NotificationType.REPORT,
             read=False,
         )
         notification.related_reports.add(instance)
@@ -253,16 +298,33 @@ def notify_admins_on_link_domain(sender, instance, created, *args, **kwargs):
         return
 
     # moderators and superusers should be notified
-    admins = User.admins()
-    for admin in admins:
+    for admin in User.admins():
         notification, _ = Notification.objects.get_or_create(
             user=admin,
-            notification_type=Notification.LINK_DOMAIN,
+            notification_type=NotificationType.LINK_DOMAIN,
             read=False,
         )
         notification.related_link_domains.add(instance)
 
 
+@receiver(models.signals.post_save, sender=InviteRequest)
+@transaction.atomic
+# pylint: disable=unused-argument
+def notify_admins_on_invite_request(sender, instance, created, *args, **kwargs):
+    """need to handle a new invite request"""
+    if not created:
+        return
+
+    # moderators and superusers should be notified
+    for admin in User.admins():
+        notification, _ = Notification.objects.get_or_create(
+            user=admin,
+            notification_type=NotificationType.INVITE_REQUEST,
+            read=False,
+        )
+        notification.related_invite_requests.add(instance)
+
+
 @receiver(models.signals.post_save, sender=GroupMemberInvitation)
 # pylint: disable=unused-argument
 def notify_user_on_group_invite(sender, instance, *args, **kwargs):
@@ -271,7 +333,7 @@ def notify_user_on_group_invite(sender, instance, *args, **kwargs):
         instance.user,
         instance.group.user,
         related_group=instance.group,
-        notification_type=Notification.INVITE,
+        notification_type=NotificationType.INVITE,
     )
 
 
@@ -309,11 +371,12 @@ def notify_user_on_follow(sender, instance, created, *args, **kwargs):
         notification = Notification.objects.filter(
             user=instance.user_object,
             related_users=instance.user_subject,
-            notification_type=Notification.FOLLOW_REQUEST,
+            notification_type=NotificationType.FOLLOW_REQUEST,
         ).first()
         if not notification:
             notification = Notification.objects.create(
-                user=instance.user_object, notification_type=Notification.FOLLOW_REQUEST
+                user=instance.user_object,
+                notification_type=NotificationType.FOLLOW_REQUEST,
             )
         notification.related_users.set([instance.user_subject])
         notification.read = False
@@ -323,6 +386,6 @@ def notify_user_on_follow(sender, instance, created, *args, **kwargs):
         Notification.notify(
             instance.user_object,
             instance.user_subject,
-            notification_type=Notification.FOLLOW,
+            notification_type=NotificationType.FOLLOW,
             read=False,
         )

bookwyrm/models/relationship.py

@@ -65,6 +65,13 @@ class UserRelationship(BookWyrmModel):
         base_path = self.user_subject.remote_id
         return f"{base_path}#follows/{self.id}"
 
+    def get_accept_reject_id(self, status):
+        """get id for sending an accept or reject of a local user"""
+
+        base_path = self.user_object.remote_id
+        status_id = self.id or 0
+        return f"{base_path}#{status}/{status_id}"
+
 
 class UserFollows(ActivityMixin, UserRelationship):
     """Following a user"""
@@ -105,6 +112,20 @@ class UserFollows(ActivityMixin, UserRelationship):
         )
         return obj
 
+    def reject(self):
+        """generate a Reject for this follow. This would normally happen
+        when a user deletes a follow they previously accepted"""
+
+        if self.user_object.local:
+            activity = activitypub.Reject(
+                id=self.get_accept_reject_id(status="rejects"),
+                actor=self.user_object.remote_id,
+                object=self.to_activity(),
+            ).serialize()
+            self.broadcast(activity, self.user_object)
+
+        self.delete()
+
 
 class UserFollowRequest(ActivitypubMixin, UserRelationship):
     """following a user requires manual or automatic confirmation"""
@@ -148,13 +169,6 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
             if not manually_approves:
                 self.accept()
 
-    def get_accept_reject_id(self, status):
-        """get id for sending an accept or reject of a local user"""
-
-        base_path = self.user_object.remote_id
-        status_id = self.id or 0
-        return f"{base_path}#{status}/{status_id}"
-
     def accept(self, broadcast_only=False):
         """turn this request into the real deal"""
         user = self.user_object

bookwyrm/models/report.py

@@ -3,7 +3,7 @@ from django.core.exceptions import PermissionDenied
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from .base_model import BookWyrmModel
 
 
@@ -46,7 +46,7 @@ class Report(BookWyrmModel):
         raise PermissionDenied()
 
     def get_remote_id(self):
-        return f"https://{DOMAIN}/settings/reports/{self.id}"
+        return f"{BASE_URL}/settings/reports/{self.id}"
 
     def comment(self, user, note):
         """comment on a report"""