Merge from main into 'trigger_migrations'

Conflicts:
	requirements.txt

.env.example

@@ -137,3 +137,10 @@ TWO_FACTOR_LOGIN_MAX_SECONDS=60
 # and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
 # Value should be a comma-separated list of host names.
 CSP_ADDITIONAL_HOSTS=
+
+# Time before being logged out (in seconds)
+# SESSION_COOKIE_AGE=2592000   # current default: 30 days
+
+# Maximum allowed memory for file uploads (increase if users are having trouble
+# uploading BookWyrm export files).
+# DATA_UPLOAD_MAX_MEMORY_MiB=100

.github/workflows/black.yml

@@ -1,17 +0,0 @@
-name: Python Formatting (run ./bw-dev black to fix)
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-      - uses: psf/black@22.12.0
-        with:
-          version: 22.12.0

.github/workflows/codeql-analysis.yml

@@ -36,7 +36,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/curlylint.yaml

@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install curlylint
         run: pip install curlylint

.github/workflows/django-tests.yml

@@ -1,70 +0,0 @@
-name: Run Python Tests
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-20.04
-    services:
-      postgres:
-        image: postgres:13
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: hunter2
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Check migrations up-to-date
-      run: |
-        python ./manage.py makemigrations --check
-      env:
-        SECRET_KEY: beepbeep
-        DOMAIN: your.domain.here
-        EMAIL_HOST: ""
-        EMAIL_HOST_USER: ""
-        EMAIL_HOST_PASSWORD: ""
-    - name: Run Tests
-      env:
-        SECRET_KEY: beepbeep
-        DEBUG: false
-        USE_HTTPS: true
-        DOMAIN: your.domain.here
-        BOOKWYRM_DATABASE_BACKEND: postgres
-        MEDIA_ROOT: images/
-        POSTGRES_PASSWORD: hunter2
-        POSTGRES_USER: postgres
-        POSTGRES_DB: github_actions
-        POSTGRES_HOST: 127.0.0.1
-        CELERY_BROKER: ""
-        REDIS_BROKER_PORT: 6379
-        REDIS_BROKER_PASSWORD: beep
-        USE_DUMMY_CACHE: true
-        FLOWER_PORT: 8888
-        EMAIL_HOST: "smtp.mailgun.org"
-        EMAIL_PORT: 587
-        EMAIL_HOST_USER: ""
-        EMAIL_HOST_PASSWORD: ""
-        EMAIL_USE_TLS: true
-        ENABLE_PREVIEW_IMAGES: false
-        ENABLE_THUMBNAIL_GENERATION: true
-        HTTP_X_FORWARDED_PROTO: false
-      run: |
-        pytest -n 3

.github/workflows/lint-frontend.yaml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install modules
         run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint

.github/workflows/mypy.yml

@@ -1,50 +0,0 @@
-name: Mypy
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Analysing the code with mypy
-      env:
-        SECRET_KEY: beepbeep
-        DEBUG: false
-        USE_HTTPS: true
-        DOMAIN: your.domain.here
-        BOOKWYRM_DATABASE_BACKEND: postgres
-        MEDIA_ROOT: images/
-        POSTGRES_PASSWORD: hunter2
-        POSTGRES_USER: postgres
-        POSTGRES_DB: github_actions
-        POSTGRES_HOST: 127.0.0.1
-        CELERY_BROKER: ""
-        REDIS_BROKER_PORT: 6379
-        REDIS_BROKER_PASSWORD: beep
-        USE_DUMMY_CACHE: true
-        FLOWER_PORT: 8888
-        EMAIL_HOST: "smtp.mailgun.org"
-        EMAIL_PORT: 587
-        EMAIL_HOST_USER: ""
-        EMAIL_HOST_PASSWORD: ""
-        EMAIL_USE_TLS: true
-        ENABLE_PREVIEW_IMAGES: false
-        ENABLE_THUMBNAIL_GENERATION: true
-        HTTP_X_FORWARDED_PROTO: false
-      run: |
-        mypy bookwyrm celerywyrm

.github/workflows/prettier.yaml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install modules
         run: npm install prettier@2.5.1

.github/workflows/pylint.yml

@@ -1,27 +0,0 @@
-name: Pylint
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Analysing the code with pylint
-      run: |
-        pylint bookwyrm/
-

.github/workflows/python.yml

@@ -0,0 +1,99 @@
+name: Python
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+# overrides for .env.example
+env:
+  POSTGRES_HOST: 127.0.0.1
+  PGPORT: 5432
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: hunter2
+  POSTGRES_DB: github_actions
+  SECRET_KEY: beepbeep
+  EMAIL_HOST_USER: ""
+  EMAIL_HOST_PASSWORD: ""
+
+jobs:
+  pytest:
+    name: Tests (pytest)
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:13
+        env: # does not inherit from jobs.build.env
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: hunter2
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pytest-github-actions-annotate-failures
+    - name: Set up .env
+      run: cp .env.example .env
+    - name: Check migrations up-to-date
+      run: python ./manage.py makemigrations --check
+    - name: Run Tests
+      run: pytest -n 3
+
+  pylint:
+    name: Linting (pylint)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Analyse code with pylint
+      run: pylint bookwyrm/
+
+  mypy:
+    name: Typing (mypy)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Set up .env
+      run: cp .env.example .env
+    - name: Analyse code with mypy
+      run: mypy bookwyrm celerywyrm
+
+  black:
+    name: Formatting (black; run ./bw-dev black to fix)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+    - uses: psf/black@22.12.0
+      with:
+        version: 22.12.0

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.11
 
 ENV PYTHONUNBUFFERED 1
 

VERSION

@@ -1 +1 @@
-0.7.0
+0.7.2

bookwyrm/activitypub/base_activity.py

@@ -20,6 +20,7 @@ from bookwyrm.tasks import app, MISC
 
 logger = logging.getLogger(__name__)
 
+# pylint: disable=invalid-name
 TBookWyrmModel = TypeVar("TBookWyrmModel", bound=base_model.BookWyrmModel)
 
 
@@ -236,7 +237,7 @@ class ActivityObject:
         omit = kwargs.get("omit", ())
         data = self.__dict__.copy()
         # recursively serialize
-        for (k, v) in data.items():
+        for k, v in data.items():
             try:
                 if issubclass(type(v), ActivityObject):
                     data[k] = v.serialize()
@@ -397,18 +398,14 @@ def resolve_remote_id(
 def get_representative():
     """Get or create an actor representing the instance
     to sign outgoing HTTP GET requests"""
-    username = f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}"
+    return models.User.objects.get_or_create(
-    email = "bookwyrm@localhost"
+        username=f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}",
-    try:
+        defaults=dict(
-        user = models.User.objects.get(username=username)
+            email="bookwyrm@localhost",
-    except models.User.DoesNotExist:
-        user = models.User.objects.create_user(
-            username=username,
-            email=email,
             local=True,
             localname=INSTANCE_ACTOR_USERNAME,
-        )
+        ),
-    return user
+    )[0]
 
 
 def get_activitypub_data(url):
@@ -427,6 +424,7 @@ def get_activitypub_data(url):
                 "Date": now,
                 "Signature": make_signature("get", sender, url, now),
             },
+            timeout=15,
         )
     except requests.RequestException:
         raise ConnectorException()

bookwyrm/activitypub/person.py

@@ -1,5 +1,5 @@
 """ actor serializer """
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import Dict
 
 from .base_activity import ActivityObject
@@ -35,7 +35,7 @@ class Person(ActivityObject):
     endpoints: Dict = None
     name: str = None
     summary: str = None
-    icon: Image = field(default_factory=lambda: {})
+    icon: Image = None
     bookwyrmUser: bool = False
     manuallyApprovesFollowers: str = False
     discoverable: str = False

bookwyrm/activitypub/verbs.py

@@ -171,9 +171,19 @@ class Reject(Verb):
     type: str = "Reject"
 
     def action(self, allow_external_connections=True):
-        """reject a follow request"""
+        """reject a follow or follow request"""
-        obj = self.object.to_model(save=False, allow_create=False)
+
+        for model_name in ["UserFollowRequest", "UserFollows", None]:
+            model = apps.get_model(f"bookwyrm.{model_name}") if model_name else None
+            if obj := self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            ):
+                # Reject the first model that can be built.
                 obj.reject()
+                break
 
 
 @dataclass(init=False)

bookwyrm/book_search.py

@@ -43,6 +43,7 @@ def search(
     min_confidence: float = 0,
     filters: Optional[list[Any]] = None,
     return_first: bool = False,
+    books: Optional[QuerySet[models.Edition]] = None,
 ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
     """search your local database"""
     filters = filters or []
@@ -54,13 +55,15 @@ def search(
     # first, try searching unique identifiers
     # unique identifiers never have spaces, title/author usually do
     if not " " in query:
-        results = search_identifiers(query, *filters, return_first=return_first)
+        results = search_identifiers(
+            query, *filters, return_first=return_first, books=books
+        )
 
     # if there were no identifier results...
     if not results:
         # then try searching title/author
         results = search_title_author(
-            query, min_confidence, *filters, return_first=return_first
+            query, min_confidence, *filters, return_first=return_first, books=books
         )
     return results
 
@@ -98,9 +101,17 @@ def format_search_result(search_result):
 
 
 def search_identifiers(
-    query, *filters, return_first=False
+    query,
+    *filters,
+    return_first=False,
+    books=None,
 ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
-    """tries remote_id, isbn; defined as dedupe fields on the model"""
+    """search Editions by deduplication fields
+
+    Best for cases when we can assume someone is searching for an exact match on
+    commonly unique data identifiers like isbn or specific library ids.
+    """
+    books = books or models.Edition.objects
     if connectors.maybe_isbn(query):
         # Oh did you think the 'S' in ISBN stood for 'standard'?
         normalized_isbn = query.strip().upper().rjust(10, "0")
@@ -111,7 +122,7 @@ def search_identifiers(
         for f in models.Edition._meta.get_fields()
         if hasattr(f, "deduplication_field") and f.deduplication_field
     ]
-    results = models.Edition.objects.filter(
+    results = books.filter(
         *filters, reduce(operator.or_, (Q(**f) for f in or_filters))
     ).distinct()
 
@@ -121,12 +132,17 @@ def search_identifiers(
 
 
 def search_title_author(
-    query, min_confidence, *filters, return_first=False
+    query,
+    min_confidence,
+    *filters,
+    return_first=False,
+    books=None,
 ) -> QuerySet[models.Edition]:
     """searches for title and author"""
+    books = books or models.Edition.objects
     query = SearchQuery(query, config="simple") | SearchQuery(query, config="english")
     results = (
-        models.Edition.objects.filter(*filters, search_vector=query)
+        books.filter(*filters, search_vector=query)
         .annotate(rank=SearchRank(F("search_vector"), query))
         .filter(rank__gt=min_confidence)
         .order_by("-rank")
@@ -137,7 +153,7 @@ def search_title_author(
 
     # filter out multiple editions of the same work
     list_results = []
-    for work_id in set(editions_of_work[:30]):
+    for work_id in editions_of_work[:30]:
         result = (
             results.filter(parent_work=work_id)
             .order_by("-rank", "-edition_rank")

bookwyrm/connectors/abstract_connector.py

@@ -3,7 +3,9 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from typing import Optional, TypedDict, Any, Callable, Union, Iterator
 from urllib.parse import quote_plus
-import imghdr
+
+# pylint: disable-next=deprecated-module
+import imghdr  # Deprecated in 3.11 for removal in 3.13; no good alternative yet
 import logging
 import re
 import asyncio

bookwyrm/forms/books.py

@@ -1,8 +1,9 @@
 """ using django model forms """
 from django import forms
 
+from file_resubmit.widgets import ResubmitImageWidget
+
 from bookwyrm import models
-from bookwyrm.models.fields import ClearableFileInputWithWarning
 from .custom_form import CustomForm
 from .widgets import ArrayWidget, SelectDateWidget, Select
 
@@ -70,9 +71,7 @@ class EditionForm(CustomForm):
             "published_date": SelectDateWidget(
                 attrs={"aria-describedby": "desc_published_date"}
             ),
-            "cover": ClearableFileInputWithWarning(
+            "cover": ResubmitImageWidget(attrs={"aria-describedby": "desc_cover"}),
-                attrs={"aria-describedby": "desc_cover"}
-            ),
             "physical_format": Select(
                 attrs={"aria-describedby": "desc_physical_format"}
             ),

bookwyrm/forms/forms.py

@@ -25,6 +25,10 @@ class ImportForm(forms.Form):
     csv_file = forms.FileField()
 
 
+class ImportUserForm(forms.Form):
+    archive_file = forms.FileField()
+
+
 class ShelfForm(CustomForm):
     class Meta:
         model = models.Shelf

bookwyrm/forms/links.py

@@ -1,4 +1,5 @@
 """ using django model forms """
+
 from urllib.parse import urlparse
 
 from django.utils.translation import gettext_lazy as _
@@ -37,10 +38,9 @@ class FileLinkForm(CustomForm):
                     ),
                 )
         if (
-            not self.instance
+            models.FileLink.objects.filter(url=url, book=book, filetype=filetype)
-            and models.FileLink.objects.filter(
+            .exclude(pk=self.instance)
-                url=url, book=book, filetype=filetype
+            .exists()
-            ).exists()
         ):
             # pylint: disable=line-too-long
             self.add_error(

bookwyrm/importers/__init__.py

@@ -1,6 +1,7 @@
 """ import classes """
 
 from .importer import Importer
+from .bookwyrm_import import BookwyrmImporter
 from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter

bookwyrm/importers/bookwyrm_import.py

@@ -0,0 +1,24 @@
+"""Import data from Bookwyrm export files"""
+from django.http import QueryDict
+
+from bookwyrm.models import User
+from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
+
+
+class BookwyrmImporter:
+    """Import a Bookwyrm User export file.
+    This is kind of a combination of an importer and a connector.
+    """
+
+    # pylint: disable=no-self-use
+    def process_import(
+        self, user: User, archive_file: bytes, settings: QueryDict
+    ) -> BookwyrmImportJob:
+        """import user data from a Bookwyrm export file"""
+
+        required = [k for k in settings if settings.get(k) == "on"]
+
+        job = BookwyrmImportJob.objects.create(
+            user=user, archive_file=archive_file, required=required
+        )
+        return job

bookwyrm/isbn/isbn.py

@@ -26,7 +26,7 @@ class IsbnHyphenator:
 
     def update_range_message(self) -> None:
         """Download the range message xml file and save it locally"""
-        response = requests.get(self.__range_message_url)
+        response = requests.get(self.__range_message_url, timeout=15)
         with open(self.__range_file_path, "w", encoding="utf-8") as file:
             file.write(response.text)
         self.__element_tree = None

bookwyrm/management/commands/erase_deleted_user_data.py

@@ -0,0 +1,43 @@
+""" Erase any data stored about deleted users """
+import sys
+from django.core.management.base import BaseCommand, CommandError
+from bookwyrm import models
+from bookwyrm.models.user import erase_user_data
+
+# pylint: disable=missing-function-docstring
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Remove Two Factor Authorisation from user"
+
+    def add_arguments(self, parser):  # pylint: disable=no-self-use
+        parser.add_argument(
+            "--dryrun",
+            action="store_true",
+            help="Preview users to be cleared without altering the database",
+        )
+
+    def handle(self, *args, **options):  # pylint: disable=unused-argument
+
+        # Check for anything fishy
+        bad_state = models.User.objects.filter(is_deleted=True, is_active=True)
+        if bad_state.exists():
+            raise CommandError(
+                f"{bad_state.count()} user(s) marked as both active and deleted"
+            )
+
+        deleted_users = models.User.objects.filter(is_deleted=True)
+        self.stdout.write(f"Found {deleted_users.count()} deleted users")
+        if options["dryrun"]:
+            self.stdout.write("\n".join(u.username for u in deleted_users[:5]))
+            if deleted_users.count() > 5:
+                self.stdout.write("... and more")
+            sys.exit()
+
+        self.stdout.write("Erasing user data:")
+        for user_id in deleted_users.values_list("id", flat=True):
+            erase_user_data.delay(user_id)
+            self.stdout.write(".", ending="")
+
+        self.stdout.write("")
+        self.stdout.write("Tasks created successfully")

bookwyrm/management/commands/instance_version.py

@@ -1,54 +0,0 @@
-""" Get your admin code to allow install """
-from django.core.management.base import BaseCommand
-
-from bookwyrm import models
-from bookwyrm.settings import VERSION
-
-
-# pylint: disable=no-self-use
-class Command(BaseCommand):
-    """command-line options"""
-
-    help = "What version is this?"
-
-    def add_arguments(self, parser):
-        """specify which function to run"""
-        parser.add_argument(
-            "--current",
-            action="store_true",
-            help="Version stored in database",
-        )
-        parser.add_argument(
-            "--target",
-            action="store_true",
-            help="Version stored in settings",
-        )
-        parser.add_argument(
-            "--update",
-            action="store_true",
-            help="Update database version",
-        )
-
-    # pylint: disable=unused-argument
-    def handle(self, *args, **options):
-        """execute init"""
-        site = models.SiteSettings.objects.get()
-        current = site.version or "0.0.1"
-        target = VERSION
-        if options.get("current"):
-            print(current)
-            return
-
-        if options.get("target"):
-            print(target)
-            return
-
-        if options.get("update"):
-            site.version = target
-            site.save()
-            return
-
-        if current != target:
-            print(f"{current}/{target}")
-        else:
-            print(current)

bookwyrm/middleware/__init__.py

@@ -1,3 +1,4 @@
 """ look at all this nice middleware! """
 from .timezone_middleware import TimezoneMiddleware
 from .ip_middleware import IPBlocklistMiddleware
+from .file_too_big import FileTooBig

bookwyrm/middleware/file_too_big.py

@@ -0,0 +1,30 @@
+"""Middleware to display a custom 413 error page"""
+
+from django.http import HttpResponse
+from django.shortcuts import render
+from django.core.exceptions import RequestDataTooBig
+
+
+class FileTooBig:
+    """Middleware to display a custom page when a
+    RequestDataTooBig exception is thrown"""
+
+    def __init__(self, get_response):
+        """boilerplate __init__ from Django docs"""
+
+        self.get_response = get_response
+
+    def __call__(self, request):
+        """If RequestDataTooBig is thrown, render the 413 error page"""
+
+        try:
+            body = request.body  # pylint: disable=unused-variable
+
+        except RequestDataTooBig:
+
+            rendered = render(request, "413.html")
+            response = HttpResponse(rendered)
+            return response
+
+        response = self.get_response(request)
+        return response

bookwyrm/migrations/0184_auto_20231106_0421.py

@@ -22,17 +22,6 @@ def update_deleted_users(apps, schema_editor):
     ).update(is_deleted=True)
 
 
-def erase_deleted_user_data(apps, schema_editor):
-    """Retroactively clear user data"""
-    for user in User.objects.filter(is_deleted=True):
-        user.erase_user_data()
-        user.save(
-            broadcast=False,
-            update_fields=["email", "avatar", "preview_image", "summary", "name"],
-        )
-        user.erase_user_statuses(broadcast=False)
-
-
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -43,7 +32,4 @@ class Migration(migrations.Migration):
         migrations.RunPython(
             update_deleted_users, reverse_code=migrations.RunPython.noop
         ),
-        migrations.RunPython(
-            erase_deleted_user_data, reverse_code=migrations.RunPython.noop
-        ),
     ]

bookwyrm/migrations/0186_auto_20231116_0048.py

@@ -0,0 +1,212 @@
+# Generated by Django 3.2.20 on 2023-11-16 00:48
+
+from django.conf import settings
+import django.contrib.postgres.fields
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0185_alter_notification_notification_type"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ParentJob",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("task_id", models.UUIDField(blank=True, null=True, unique=True)),
+                (
+                    "created_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                (
+                    "updated_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("complete", models.BooleanField(default=False)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("pending", "Pending"),
+                            ("active", "Active"),
+                            ("complete", "Complete"),
+                            ("stopped", "Stopped"),
+                            ("failed", "Failed"),
+                        ],
+                        default="pending",
+                        max_length=50,
+                        null=True,
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="user_import_time_limit",
+            field=models.IntegerField(default=48),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("USER_IMPORT", "User Import"),
+                    ("USER_EXPORT", "User Export"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+        migrations.CreateModel(
+            name="BookwyrmExportJob",
+            fields=[
+                (
+                    "parentjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+                ("export_data", models.FileField(null=True, upload_to="")),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.parentjob",),
+        ),
+        migrations.CreateModel(
+            name="BookwyrmImportJob",
+            fields=[
+                (
+                    "parentjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+                ("archive_file", models.FileField(blank=True, null=True, upload_to="")),
+                ("import_data", models.JSONField(null=True)),
+                (
+                    "required",
+                    django.contrib.postgres.fields.ArrayField(
+                        base_field=models.CharField(blank=True, max_length=50),
+                        blank=True,
+                        size=None,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.parentjob",),
+        ),
+        migrations.CreateModel(
+            name="ChildJob",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("task_id", models.UUIDField(blank=True, null=True, unique=True)),
+                (
+                    "created_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                (
+                    "updated_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("complete", models.BooleanField(default=False)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("pending", "Pending"),
+                            ("active", "Active"),
+                            ("complete", "Complete"),
+                            ("stopped", "Stopped"),
+                            ("failed", "Failed"),
+                        ],
+                        default="pending",
+                        max_length=50,
+                        null=True,
+                    ),
+                ),
+                (
+                    "parent_job",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="child_jobs",
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="notification",
+            name="related_user_export",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="bookwyrm.bookwyrmexportjob",
+            ),
+        ),
+    ]

bookwyrm/migrations/0189_alter_user_preferred_language.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.23 on 2023-12-12 23:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("eo-uy", "Esperanto (Esperanto)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("nl-nl", "Nederlands (Dutch)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("uk-ua", "Українська (Ukrainian)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2023-11-22 10:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0186_auto_20231116_0048"),
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = []

bookwyrm/migrations/0190_alter_notification_notification_type.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.23 on 2023-11-23 19:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0189_merge_0186_auto_20231116_0048_0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("USER_IMPORT", "User Import"),
+                    ("USER_EXPORT", "User Export"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE_REQUEST", "Invite Request"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0191_merge_20240102_0326.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-01-02 03:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0189_alter_user_preferred_language"),
+        ("bookwyrm", "0190_alter_notification_notification_type"),
+    ]
+
+    operations = []

bookwyrm/migrations/0192_make_page_positions_text.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.23 on 2024-01-04 23:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="quotation",
+            name="endposition",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="quotation",
+            name="position",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-02 19:36
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="sitesettings",
+            old_name="version",
+            new_name="available_version",
+        ),
+    ]

bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-16 10:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="user_exports_enabled",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0193_merge_20240203_1539.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-02-03 15:39
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0192_make_page_positions_text"),
+        ("bookwyrm", "0192_sitesettings_user_exports_enabled"),
+    ]
+
+    operations = []

bookwyrm/migrations/0194_merge_20240203_1619.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-02-03 16:19
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0192_rename_version_sitesettings_available_version"),
+        ("bookwyrm", "0193_merge_20240203_1539"),
+    ]
+
+    operations = []

bookwyrm/migrations/0195_alter_user_preferred_language.py

@@ -0,0 +1,46 @@
+# Generated by Django 3.2.23 on 2024-02-21 00:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0194_merge_20240203_1619"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("eo-uy", "Esperanto (Esperanto)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("ko-kr", "한국어 (Korean)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("nl-nl", "Nederlands (Dutch)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("uk-ua", "Українська (Ukrainian)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0196_merge_pr3134_into_main.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-03-18 00:48
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_migrate_search_vec_triggers_to_pgtriggers"),
+        ("bookwyrm", "0195_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/models/__init__.py

@@ -26,6 +26,8 @@ from .federated_server import FederatedServer
 from .group import Group, GroupMember, GroupMemberInvitation
 
 from .import_job import ImportJob, ImportItem
+from .bookwyrm_import_job import BookwyrmImportJob
+from .bookwyrm_export_job import BookwyrmExportJob
 
 from .move import MoveUser
 

bookwyrm/models/activitypub_mixin.py

@@ -152,8 +152,9 @@ class ActivitypubMixin:
         # find anyone who's tagged in a status, for example
         mentions = self.recipients if hasattr(self, "recipients") else []
 
-        # we always send activities to explicitly mentioned users' inboxes
+        # we always send activities to explicitly mentioned users (using shared inboxes
-        recipients = [u.inbox for u in mentions or [] if not u.local]
+        # where available to avoid duplicate submissions to a given instance)
+        recipients = {u.shared_inbox or u.inbox for u in mentions if not u.local}
 
         # unless it's a dm, all the followers should receive the activity
         if privacy != "direct":
@@ -173,18 +174,18 @@ class ActivitypubMixin:
             if user:
                 queryset = queryset.filter(following=user)
 
-            # ideally, we will send to shared inboxes for efficiency
+            # as above, we prefer shared inboxes if available
-            shared_inboxes = (
+            recipients.update(
-                queryset.filter(shared_inbox__isnull=False)
+                queryset.filter(shared_inbox__isnull=False).values_list(
-                .values_list("shared_inbox", flat=True)
+                    "shared_inbox", flat=True
-                .distinct()
                 )
-            # but not everyone has a shared inbox
+            )
-            inboxes = queryset.filter(shared_inbox__isnull=True).values_list(
+            recipients.update(
+                queryset.filter(shared_inbox__isnull=True).values_list(
                     "inbox", flat=True
                 )
-            recipients += list(shared_inboxes) + list(inboxes)
+            )
-        return list(set(recipients))
+        return list(recipients)
 
     def to_activity_dataclass(self):
         """convert from a model to an activity"""
@@ -602,7 +603,7 @@ def to_ordered_collection_page(
     if activity_page.has_next():
         next_page = f"{remote_id}?page={activity_page.next_page_number()}"
     if activity_page.has_previous():
-        prev_page = f"{remote_id}?page=%d{activity_page.previous_page_number()}"
+        prev_page = f"{remote_id}?page={activity_page.previous_page_number()}"
     return activitypub.OrderedCollectionPage(
         id=f"{remote_id}?page={page}",
         partOf=remote_id,

bookwyrm/models/bookwyrm_export_job.py

@@ -0,0 +1,229 @@
+"""Export user account to tar.gz file for import into another Bookwyrm instance"""
+
+import dataclasses
+import logging
+from uuid import uuid4
+
+from django.db.models import FileField
+from django.db.models import Q
+from django.core.serializers.json import DjangoJSONEncoder
+from django.core.files.base import ContentFile
+
+from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, List, ListItem
+from bookwyrm.models import Review, Comment, Quotation
+from bookwyrm.models import Edition
+from bookwyrm.models import UserFollows, User, UserBlocks
+from bookwyrm.models.job import ParentJob, ParentTask
+from bookwyrm.tasks import app, IMPORTS
+from bookwyrm.utils.tar import BookwyrmTarFile
+
+logger = logging.getLogger(__name__)
+
+
+class BookwyrmExportJob(ParentJob):
+    """entry for a specific request to export a bookwyrm user"""
+
+    export_data = FileField(null=True)
+
+    def start_job(self):
+        """Start the job"""
+        start_export_task.delay(job_id=self.id, no_children=True)
+
+        return self
+
+
+@app.task(queue=IMPORTS, base=ParentTask)
+def start_export_task(**kwargs):
+    """trigger the child tasks for each row"""
+    job = BookwyrmExportJob.objects.get(id=kwargs["job_id"])
+
+    # don't start the job if it was stopped from the UI
+    if job.complete:
+        return
+    try:
+        # This is where ChildJobs get made
+        job.export_data = ContentFile(b"", str(uuid4()))
+        json_data = json_export(job.user)
+        tar_export(json_data, job.user, job.export_data)
+        job.save(update_fields=["export_data"])
+    except Exception as err:  # pylint: disable=broad-except
+        logger.exception("User Export Job %s Failed with error: %s", job.id, err)
+        job.set_status("failed")
+
+    job.set_status("complete")
+
+
+def tar_export(json_data: str, user, file):
+    """wrap the export information in a tar file"""
+    file.open("wb")
+    with BookwyrmTarFile.open(mode="w:gz", fileobj=file) as tar:
+        tar.write_bytes(json_data.encode("utf-8"))
+
+        # Add avatar image if present
+        if getattr(user, "avatar", False):
+            tar.add_image(user.avatar, filename="avatar")
+
+        editions = get_books_for_user(user)
+        for book in editions:
+            if getattr(book, "cover", False):
+                tar.add_image(book.cover)
+
+    file.close()
+
+
+def json_export(
+    user,
+):  # pylint: disable=too-many-locals, too-many-statements, too-many-branches
+    """Generate an export for a user"""
+
+    # User as AP object
+    exported_user = user.to_activity()
+    # I don't love this but it prevents a JSON encoding error
+    # when there is no user image
+    if exported_user.get("icon") in (None, dataclasses.MISSING):
+        exported_user["icon"] = {}
+    else:
+        # change the URL to be relative to the JSON file
+        file_type = exported_user["icon"]["url"].rsplit(".", maxsplit=1)[-1]
+        filename = f"avatar.{file_type}"
+        exported_user["icon"]["url"] = filename
+
+    # Additional settings - can't be serialized as AP
+    vals = [
+        "show_goal",
+        "preferred_timezone",
+        "default_post_privacy",
+        "show_suggested_users",
+    ]
+    exported_user["settings"] = {}
+    for k in vals:
+        exported_user["settings"][k] = getattr(user, k)
+
+    # Reading goals - can't be serialized as AP
+    reading_goals = AnnualGoal.objects.filter(user=user).distinct()
+    exported_user["goals"] = []
+    for goal in reading_goals:
+        exported_user["goals"].append(
+            {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy}
+        )
+
+    # Reading history - can't be serialized as AP
+    readthroughs = ReadThrough.objects.filter(user=user).distinct().values()
+    readthroughs = list(readthroughs)
+
+    # Books
+    editions = get_books_for_user(user)
+    exported_user["books"] = []
+
+    for edition in editions:
+        book = {}
+        book["work"] = edition.parent_work.to_activity()
+        book["edition"] = edition.to_activity()
+
+        if book["edition"].get("cover"):
+            # change the URL to be relative to the JSON file
+            filename = book["edition"]["cover"]["url"].rsplit("/", maxsplit=1)[-1]
+            book["edition"]["cover"]["url"] = f"covers/{filename}"
+
+        # authors
+        book["authors"] = []
+        for author in edition.authors.all():
+            book["authors"].append(author.to_activity())
+
+        # Shelves this book is on
+        # Every ShelfItem is this book so we don't other serializing
+        book["shelves"] = []
+        shelf_books = (
+            ShelfBook.objects.select_related("shelf")
+            .filter(user=user, book=edition)
+            .distinct()
+        )
+
+        for shelfbook in shelf_books:
+            book["shelves"].append(shelfbook.shelf.to_activity())
+
+        # Lists and ListItems
+        # ListItems include "notes" and "approved" so we need them
+        # even though we know it's this book
+        book["lists"] = []
+        list_items = ListItem.objects.filter(book=edition, user=user).distinct()
+
+        for item in list_items:
+            list_info = item.book_list.to_activity()
+            list_info[
+                "privacy"
+            ] = item.book_list.privacy  # this isn't serialized so we add it
+            list_info["list_item"] = item.to_activity()
+            book["lists"].append(list_info)
+
+        # Statuses
+        # Can't use select_subclasses here because
+        # we need to filter on the "book" value,
+        # which is not available on an ordinary Status
+        for status in ["comments", "quotations", "reviews"]:
+            book[status] = []
+
+        comments = Comment.objects.filter(user=user, book=edition).all()
+        for status in comments:
+            obj = status.to_activity()
+            obj["progress"] = status.progress
+            obj["progress_mode"] = status.progress_mode
+            book["comments"].append(obj)
+
+        quotes = Quotation.objects.filter(user=user, book=edition).all()
+        for status in quotes:
+            obj = status.to_activity()
+            obj["position"] = status.position
+            obj["endposition"] = status.endposition
+            obj["position_mode"] = status.position_mode
+            book["quotations"].append(obj)
+
+        reviews = Review.objects.filter(user=user, book=edition).all()
+        for status in reviews:
+            obj = status.to_activity()
+            book["reviews"].append(obj)
+
+        # readthroughs can't be serialized to activity
+        book_readthroughs = (
+            ReadThrough.objects.filter(user=user, book=edition).distinct().values()
+        )
+        book["readthroughs"] = list(book_readthroughs)
+
+        # append everything
+        exported_user["books"].append(book)
+
+    # saved book lists - just the remote id
+    saved_lists = List.objects.filter(id__in=user.saved_lists.all()).distinct()
+    exported_user["saved_lists"] = [l.remote_id for l in saved_lists]
+
+    # follows - just the remote id
+    follows = UserFollows.objects.filter(user_subject=user).distinct()
+    following = User.objects.filter(userfollows_user_object__in=follows).distinct()
+    exported_user["follows"] = [f.remote_id for f in following]
+
+    # blocks - just the remote id
+    blocks = UserBlocks.objects.filter(user_subject=user).distinct()
+    blocking = User.objects.filter(userblocks_user_object__in=blocks).distinct()
+
+    exported_user["blocks"] = [b.remote_id for b in blocking]
+
+    return DjangoJSONEncoder().encode(exported_user)
+
+
+def get_books_for_user(user):
+    """Get all the books and editions related to a user"""
+
+    editions = (
+        Edition.objects.select_related("parent_work")
+        .filter(
+            Q(shelves__user=user)
+            | Q(readthrough__user=user)
+            | Q(review__user=user)
+            | Q(list__user=user)
+            | Q(comment__user=user)
+            | Q(quotation__user=user)
+        )
+        .distinct()
+    )
+
+    return editions

bookwyrm/models/bookwyrm_import_job.py

@@ -0,0 +1,459 @@
+"""Import a user from another Bookwyrm instance"""
+
+import json
+import logging
+
+from django.db.models import FileField, JSONField, CharField
+from django.utils import timezone
+from django.utils.html import strip_tags
+from django.contrib.postgres.fields import ArrayField as DjangoArrayField
+
+from bookwyrm import activitypub
+from bookwyrm import models
+from bookwyrm.tasks import app, IMPORTS
+from bookwyrm.models.job import ParentJob, ParentTask, SubTask
+from bookwyrm.utils.tar import BookwyrmTarFile
+
+logger = logging.getLogger(__name__)
+
+
+class BookwyrmImportJob(ParentJob):
+    """entry for a specific request for importing a bookwyrm user backup"""
+
+    archive_file = FileField(null=True, blank=True)
+    import_data = JSONField(null=True)
+    required = DjangoArrayField(CharField(max_length=50, blank=True), blank=True)
+
+    def start_job(self):
+        """Start the job"""
+        start_import_task.delay(job_id=self.id, no_children=True)
+
+
+@app.task(queue=IMPORTS, base=ParentTask)
+def start_import_task(**kwargs):
+    """trigger the child import tasks for each user data"""
+    job = BookwyrmImportJob.objects.get(id=kwargs["job_id"])
+    archive_file = job.archive_file
+
+    # don't start the job if it was stopped from the UI
+    if job.complete:
+        return
+
+    try:
+        archive_file.open("rb")
+        with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar:
+            job.import_data = json.loads(tar.read("archive.json").decode("utf-8"))
+
+            if "include_user_profile" in job.required:
+                update_user_profile(job.user, tar, job.import_data)
+            if "include_user_settings" in job.required:
+                update_user_settings(job.user, job.import_data)
+            if "include_goals" in job.required:
+                update_goals(job.user, job.import_data.get("goals"))
+            if "include_saved_lists" in job.required:
+                upsert_saved_lists(job.user, job.import_data.get("saved_lists"))
+            if "include_follows" in job.required:
+                upsert_follows(job.user, job.import_data.get("follows"))
+            if "include_blocks" in job.required:
+                upsert_user_blocks(job.user, job.import_data.get("blocks"))
+
+            process_books(job, tar)
+
+            job.set_status("complete")
+        archive_file.close()
+
+    except Exception as err:  # pylint: disable=broad-except
+        logger.exception("User Import Job %s Failed with error: %s", job.id, err)
+        job.set_status("failed")
+
+
+def process_books(job, tar):
+    """
+    Process user import data related to books
+    We always import the books even if not assigning
+    them to shelves, lists etc
+    """
+
+    books = job.import_data.get("books")
+
+    for data in books:
+        book = get_or_create_edition(data, tar)
+
+        if "include_shelves" in job.required:
+            upsert_shelves(book, job.user, data)
+
+        if "include_readthroughs" in job.required:
+            upsert_readthroughs(data.get("readthroughs"), job.user, book.id)
+
+        if "include_comments" in job.required:
+            upsert_statuses(
+                job.user, models.Comment, data.get("comments"), book.remote_id
+            )
+        if "include_quotations" in job.required:
+            upsert_statuses(
+                job.user, models.Quotation, data.get("quotations"), book.remote_id
+            )
+
+        if "include_reviews" in job.required:
+            upsert_statuses(
+                job.user, models.Review, data.get("reviews"), book.remote_id
+            )
+
+        if "include_lists" in job.required:
+            upsert_lists(job.user, data.get("lists"), book.id)
+
+
+def get_or_create_edition(book_data, tar):
+    """Take a JSON string of work and edition data,
+    find or create the edition and work in the database and
+    return an edition instance"""
+
+    edition = book_data.get("edition")
+    existing = models.Edition.find_existing(edition)
+    if existing:
+        return existing
+
+    # make sure we have the authors in the local DB
+    # replace the old author ids in the edition JSON
+    edition["authors"] = []
+    for author in book_data.get("authors"):
+        parsed_author = activitypub.parse(author)
+        instance = parsed_author.to_model(
+            model=models.Author, save=True, overwrite=True
+        )
+
+        edition["authors"].append(instance.remote_id)
+
+    # we will add the cover later from the tar
+    # don't try to load it from the old server
+    cover = edition.get("cover", {})
+    cover_path = cover.get("url", None)
+    edition["cover"] = {}
+
+    # first we need the parent work to exist
+    work = book_data.get("work")
+    work["editions"] = []
+    parsed_work = activitypub.parse(work)
+    work_instance = parsed_work.to_model(model=models.Work, save=True, overwrite=True)
+
+    # now we have a work we can add it to the edition
+    # and create the edition model instance
+    edition["work"] = work_instance.remote_id
+    parsed_edition = activitypub.parse(edition)
+    book = parsed_edition.to_model(model=models.Edition, save=True, overwrite=True)
+
+    # set the cover image from the tar
+    if cover_path:
+        tar.write_image_to_file(cover_path, book.cover)
+
+    return book
+
+
+def upsert_readthroughs(data, user, book_id):
+    """Take a JSON string of readthroughs and
+    find or create the instances in the database"""
+
+    for read_through in data:
+
+        obj = {}
+        keys = [
+            "progress_mode",
+            "start_date",
+            "finish_date",
+            "stopped_date",
+            "is_active",
+        ]
+        for key in keys:
+            obj[key] = read_through[key]
+        obj["user_id"] = user.id
+        obj["book_id"] = book_id
+
+        existing = models.ReadThrough.objects.filter(**obj).first()
+        if not existing:
+            models.ReadThrough.objects.create(**obj)
+
+
+def upsert_statuses(user, cls, data, book_remote_id):
+    """Take a JSON string of a status and
+    find or create the instances in the database"""
+
+    for status in data:
+        if is_alias(
+            user, status["attributedTo"]
+        ):  # don't let l33t hax0rs steal other people's posts
+            # update ids and remove replies
+            status["attributedTo"] = user.remote_id
+            status["to"] = update_followers_address(user, status["to"])
+            status["cc"] = update_followers_address(user, status["cc"])
+            status[
+                "replies"
+            ] = (
+                {}
+            )  # this parses incorrectly but we can't set it without knowing the new id
+            status["inReplyToBook"] = book_remote_id
+            parsed = activitypub.parse(status)
+            if not status_already_exists(
+                user, parsed
+            ):  # don't duplicate posts on multiple import
+
+                instance = parsed.to_model(model=cls, save=True, overwrite=True)
+
+                for val in [
+                    "progress",
+                    "progress_mode",
+                    "position",
+                    "endposition",
+                    "position_mode",
+                ]:
+                    if status.get(val):
+                        instance.val = status[val]
+
+                instance.remote_id = instance.get_remote_id()  # update the remote_id
+                instance.save()  # save and broadcast
+
+        else:
+            logger.info("User does not have permission to import statuses")
+
+
+def upsert_lists(user, lists, book_id):
+    """Take a list of objects each containing
+    a list and list item as AP objects
+
+    Because we are creating new IDs we can't assume the id
+    will exist or be accurate, so we only use to_model for
+    adding new items after checking whether they exist  .
+
+    """
+
+    book = models.Edition.objects.get(id=book_id)
+
+    for blist in lists:
+        booklist = models.List.objects.filter(name=blist["name"], user=user).first()
+        if not booklist:
+
+            blist["owner"] = user.remote_id
+            parsed = activitypub.parse(blist)
+            booklist = parsed.to_model(model=models.List, save=True, overwrite=True)
+
+            booklist.privacy = blist["privacy"]
+            booklist.save()
+
+        item = models.ListItem.objects.filter(book=book, book_list=booklist).exists()
+        if not item:
+            count = booklist.books.count()
+            models.ListItem.objects.create(
+                book=book,
+                book_list=booklist,
+                user=user,
+                notes=blist["list_item"]["notes"],
+                approved=blist["list_item"]["approved"],
+                order=count + 1,
+            )
+
+
+def upsert_shelves(book, user, book_data):
+    """Take shelf JSON objects and create
+    DB entries if they don't already exist"""
+
+    shelves = book_data["shelves"]
+    for shelf in shelves:
+
+        book_shelf = models.Shelf.objects.filter(name=shelf["name"], user=user).first()
+
+        if not book_shelf:
+            book_shelf = models.Shelf.objects.create(name=shelf["name"], user=user)
+
+        # add the book as a ShelfBook if needed
+        if not models.ShelfBook.objects.filter(
+            book=book, shelf=book_shelf, user=user
+        ).exists():
+            models.ShelfBook.objects.create(
+                book=book, shelf=book_shelf, user=user, shelved_date=timezone.now()
+            )
+
+
+def update_user_profile(user, tar, data):
+    """update the user's profile from import data"""
+    name = data.get("name", None)
+    username = data.get("preferredUsername")
+    user.name = name if name else username
+    user.summary = strip_tags(data.get("summary", None))
+    user.save(update_fields=["name", "summary"])
+    if data["icon"].get("url"):
+        avatar_filename = next(filter(lambda n: n.startswith("avatar"), tar.getnames()))
+        tar.write_image_to_file(avatar_filename, user.avatar)
+
+
+def update_user_settings(user, data):
+    """update the user's settings from import data"""
+
+    update_fields = ["manually_approves_followers", "hide_follows", "discoverable"]
+
+    ap_fields = [
+        ("manuallyApprovesFollowers", "manually_approves_followers"),
+        ("hideFollows", "hide_follows"),
+        ("discoverable", "discoverable"),
+    ]
+
+    for (ap_field, bw_field) in ap_fields:
+        setattr(user, bw_field, data[ap_field])
+
+    bw_fields = [
+        "show_goal",
+        "show_suggested_users",
+        "default_post_privacy",
+        "preferred_timezone",
+    ]
+
+    for field in bw_fields:
+        update_fields.append(field)
+        setattr(user, field, data["settings"][field])
+
+    user.save(update_fields=update_fields)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def update_user_settings_task(job_id):
+    """wrapper task for user's settings import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return update_user_settings(parent_job.user, parent_job.import_data.get("user"))
+
+
+def update_goals(user, data):
+    """update the user's goals from import data"""
+
+    for goal in data:
+        # edit the existing goal if there is one
+        existing = models.AnnualGoal.objects.filter(
+            year=goal["year"], user=user
+        ).first()
+        if existing:
+            for k in goal.keys():
+                setattr(existing, k, goal[k])
+            existing.save()
+        else:
+            goal["user"] = user
+            models.AnnualGoal.objects.create(**goal)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def update_goals_task(job_id):
+    """wrapper task for user's goals import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return update_goals(parent_job.user, parent_job.import_data.get("goals"))
+
+
+def upsert_saved_lists(user, values):
+    """Take a list of remote ids and add as saved lists"""
+
+    for remote_id in values:
+        book_list = activitypub.resolve_remote_id(remote_id, models.List)
+        if book_list:
+            user.saved_lists.add(book_list)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def upsert_saved_lists_task(job_id):
+    """wrapper task for user's saved lists import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return upsert_saved_lists(
+        parent_job.user, parent_job.import_data.get("saved_lists")
+    )
+
+
+def upsert_follows(user, values):
+    """Take a list of remote ids and add as follows"""
+
+    for remote_id in values:
+        followee = activitypub.resolve_remote_id(remote_id, models.User)
+        if followee:
+            (follow_request, created,) = models.UserFollowRequest.objects.get_or_create(
+                user_subject=user,
+                user_object=followee,
+            )
+
+            if not created:
+                # this request probably failed to connect with the remote
+                # and should save to trigger a re-broadcast
+                follow_request.save()
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def upsert_follows_task(job_id):
+    """wrapper task for user's follows import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return upsert_follows(parent_job.user, parent_job.import_data.get("follows"))
+
+
+def upsert_user_blocks(user, user_ids):
+    """block users"""
+
+    for user_id in user_ids:
+        user_object = activitypub.resolve_remote_id(user_id, models.User)
+        if user_object:
+            exists = models.UserBlocks.objects.filter(
+                user_subject=user, user_object=user_object
+            ).exists()
+            if not exists:
+                models.UserBlocks.objects.create(
+                    user_subject=user, user_object=user_object
+                )
+                # remove the blocked users's lists from the groups
+                models.List.remove_from_group(user, user_object)
+                # remove the blocked user from all blocker's owned groups
+                models.GroupMember.remove(user, user_object)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def upsert_user_blocks_task(job_id):
+    """wrapper task for user's blocks import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return upsert_user_blocks(
+        parent_job.user, parent_job.import_data.get("blocked_users")
+    )
+
+
+def update_followers_address(user, field):
+    """statuses to or cc followers need to have the followers
+    address updated to the new local user"""
+
+    for i, audience in enumerate(field):
+        if audience.rsplit("/")[-1] == "followers":
+            field[i] = user.followers_url
+
+    return field
+
+
+def is_alias(user, remote_id):
+    """check that the user is listed as movedTo or also_known_as
+    in the remote user's profile"""
+
+    remote_user = activitypub.resolve_remote_id(
+        remote_id=remote_id, model=models.User, save=False
+    )
+
+    if remote_user:
+
+        if remote_user.moved_to:
+            return user.remote_id == remote_user.moved_to
+
+        if remote_user.also_known_as:
+            return user in remote_user.also_known_as.all()
+
+    return False
+
+
+def status_already_exists(user, status):
+    """check whether this status has already been published
+    by this user. We can't rely on to_model() because it
+    only matches on remote_id, which we have to change
+    *after* saving because it needs the primary key (id)"""
+
+    return models.Status.objects.filter(
+        user=user, content=status.content, published_date=status.published
+    ).exists()

bookwyrm/models/fields.py

@@ -482,7 +482,7 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
         if not url:
             return None
 
-        return activitypub.Document(url=url, name=alt)
+        return activitypub.Image(url=url, name=alt)
 
     def field_from_activity(self, value, allow_external_connections=True):
         image_slug = value

bookwyrm/models/job.py

@@ -0,0 +1,308 @@
+"""Everything needed for Celery to multi-thread complex tasks."""
+
+from django.db import models
+from django.db import transaction
+from django.utils.translation import gettext_lazy as _
+from django.utils import timezone
+from bookwyrm.models.user import User
+
+from bookwyrm.tasks import app
+
+
+class Job(models.Model):
+    """Abstract model to store the state of a Task."""
+
+    class Status(models.TextChoices):
+        """Possible job states."""
+
+        PENDING = "pending", _("Pending")
+        ACTIVE = "active", _("Active")
+        COMPLETE = "complete", _("Complete")
+        STOPPED = "stopped", _("Stopped")
+        FAILED = "failed", _("Failed")
+
+    task_id = models.UUIDField(unique=True, null=True, blank=True)
+
+    created_date = models.DateTimeField(default=timezone.now)
+    updated_date = models.DateTimeField(default=timezone.now)
+    complete = models.BooleanField(default=False)
+    status = models.CharField(
+        max_length=50, choices=Status.choices, default=Status.PENDING, null=True
+    )
+
+    class Meta:
+        """Make it abstract"""
+
+        abstract = True
+
+    def complete_job(self):
+        """Report that the job has completed"""
+        if self.complete:
+            return
+
+        self.status = self.Status.COMPLETE
+        self.complete = True
+        self.updated_date = timezone.now()
+
+        self.save(update_fields=["status", "complete", "updated_date"])
+
+    def stop_job(self, reason=None):
+        """Stop the job"""
+        if self.complete:
+            return
+
+        self.__terminate_job()
+
+        if reason and reason == "failed":
+            self.status = self.Status.FAILED
+        else:
+            self.status = self.Status.STOPPED
+        self.complete = True
+        self.updated_date = timezone.now()
+
+        self.save(update_fields=["status", "complete", "updated_date"])
+
+    def set_status(self, status):
+        """Set job status"""
+        if self.complete:
+            return
+
+        if self.status == status:
+            return
+
+        if status == self.Status.COMPLETE:
+            self.complete_job()
+            return
+
+        if status == self.Status.STOPPED:
+            self.stop_job()
+            return
+
+        if status == self.Status.FAILED:
+            self.stop_job(reason="failed")
+            return
+
+        self.updated_date = timezone.now()
+        self.status = status
+
+        self.save(update_fields=["status", "updated_date"])
+
+    def __terminate_job(self):
+        """Tell workers to ignore and not execute this task."""
+        app.control.revoke(self.task_id, terminate=True)
+
+
+class ParentJob(Job):
+    """Store the state of a Task which can spawn many :model:`ChildJob`s to spread
+    resource load.
+
+    Intended to be sub-classed if necessary via proxy or
+    multi-table inheritance.
+    Extends :model:`Job`.
+    """
+
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+
+    def complete_job(self):
+        """Report that the job has completed and stop pending
+        children. Extend.
+        """
+        super().complete_job()
+        self.__terminate_pending_child_jobs()
+
+    def notify_child_job_complete(self):
+        """let the job know when the items get work done"""
+        if self.complete:
+            return
+
+        self.updated_date = timezone.now()
+        self.save(update_fields=["updated_date"])
+
+        if not self.complete and self.has_completed:
+            self.complete_job()
+
+    def __terminate_job(self):  # pylint: disable=unused-private-member
+        """Tell workers to ignore and not execute this task
+        & pending child tasks. Extend.
+        """
+        super().__terminate_job()
+        self.__terminate_pending_child_jobs()
+
+    def __terminate_pending_child_jobs(self):
+        """Tell workers to ignore and not execute any pending child tasks."""
+        tasks = self.pending_child_jobs.filter(task_id__isnull=False).values_list(
+            "task_id", flat=True
+        )
+        app.control.revoke(list(tasks))
+
+        for task in self.pending_child_jobs:
+            task.update(status=self.Status.STOPPED)
+
+    @property
+    def has_completed(self):
+        """has this job finished"""
+        return not self.pending_child_jobs.exists()
+
+    @property
+    def pending_child_jobs(self):
+        """items that haven't been processed yet"""
+        return self.child_jobs.filter(complete=False)
+
+
+class ChildJob(Job):
+    """Stores the state of a Task for the related :model:`ParentJob`.
+
+    Intended to be sub-classed if necessary via proxy or
+    multi-table inheritance.
+    Extends :model:`Job`.
+    """
+
+    parent_job = models.ForeignKey(
+        ParentJob, on_delete=models.CASCADE, related_name="child_jobs"
+    )
+
+    def set_status(self, status):
+        """Set job and parent_job status. Extend."""
+        super().set_status(status)
+
+        if (
+            status == self.Status.ACTIVE
+            and self.parent_job.status == self.Status.PENDING
+        ):
+            self.parent_job.set_status(self.Status.ACTIVE)
+
+    def complete_job(self):
+        """Report to parent_job that the job has completed. Extend."""
+        super().complete_job()
+        self.parent_job.notify_child_job_complete()
+
+
+class ParentTask(app.Task):
+    """Used with ParentJob, Abstract Tasks execute code at specific points in
+    a Task's lifecycle, applying to all Tasks with the same 'base'.
+
+    All status & ParentJob.task_id assignment is managed here for you.
+    Usage e.g. @app.task(base=ParentTask)
+    """
+
+    def before_start(
+        self, task_id, args, kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Handler called before the task starts. Override.
+
+        Prepare ParentJob before the task starts.
+
+        Arguments:
+            task_id (str): Unique id of the task to execute.
+            args (Tuple): Original arguments for the task to execute.
+            kwargs (Dict): Original keyword arguments for the task to execute.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            no_children (bool): If 'True' this is the only Task expected to run
+                for the given ParentJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+        job = ParentJob.objects.get(id=kwargs["job_id"])
+        job.task_id = task_id
+        job.save(update_fields=["task_id"])
+
+        if kwargs["no_children"]:
+            job.set_status(ChildJob.Status.ACTIVE)
+
+    def on_success(
+        self, retval, task_id, args, kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Run by the worker if the task executes successfully. Override.
+
+        Update ParentJob on Task complete.
+
+        Arguments:
+            retval (Any): The return value of the task.
+            task_id (str): Unique id of the executed task.
+            args (Tuple): Original arguments for the executed task.
+            kwargs (Dict): Original keyword arguments for the executed task.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            no_children (bool): If 'True' this is the only Task expected to run
+                for the given ParentJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+
+        if kwargs["no_children"]:
+            job = ParentJob.objects.get(id=kwargs["job_id"])
+            job.complete_job()
+
+
+class SubTask(app.Task):
+    """Used with ChildJob, Abstract Tasks execute code at specific points in
+    a Task's lifecycle, applying to all Tasks with the same 'base'.
+
+    All status & ChildJob.task_id assignment is managed here for you.
+    Usage e.g. @app.task(base=SubTask)
+    """
+
+    def before_start(
+        self, task_id, args, kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Handler called before the task starts. Override.
+
+        Prepare ChildJob before the task starts.
+
+        Arguments:
+            task_id (str): Unique id of the task to execute.
+            args (Tuple): Original arguments for the task to execute.
+            kwargs (Dict): Original keyword arguments for the task to execute.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            child_id (int): Unique 'id' of the ChildJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+        child_job = ChildJob.objects.get(id=kwargs["child_id"])
+        child_job.task_id = task_id
+        child_job.save(update_fields=["task_id"])
+        child_job.set_status(ChildJob.Status.ACTIVE)
+
+    def on_success(
+        self, retval, task_id, args, kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Run by the worker if the task executes successfully. Override.
+
+        Notify ChildJob of task completion.
+
+        Arguments:
+            retval (Any): The return value of the task.
+            task_id (str): Unique id of the executed task.
+            args (Tuple): Original arguments for the executed task.
+            kwargs (Dict): Original keyword arguments for the executed task.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            child_id (int): Unique 'id' of the ChildJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+        subtask = ChildJob.objects.get(id=kwargs["child_id"])
+        subtask.complete_job()
+
+
+@transaction.atomic
+def create_child_job(parent_job, task_callback):
+    """Utility method for creating a ChildJob
+    and running a task to avoid DB race conditions
+    """
+    child_job = ChildJob.objects.create(parent_job=parent_job)
+    transaction.on_commit(
+        lambda: task_callback.delay(job_id=parent_job.id, child_id=child_job.id)
+    )
+
+    return child_job

bookwyrm/models/notification.py

@@ -1,8 +1,16 @@
 """ alert a user to activity """
 from django.db import models, transaction
 from django.dispatch import receiver
+from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob
 from .base_model import BookWyrmModel
-from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain
+from . import (
+    Boost,
+    Favorite,
+    GroupMemberInvitation,
+    ImportJob,
+    BookwyrmImportJob,
+    LinkDomain,
+)
 from . import ListItem, Report, Status, User, UserFollowRequest
 from .site import InviteRequest
 
@@ -23,6 +31,8 @@ class NotificationType(models.TextChoices):
 
     # Imports
     IMPORT = "IMPORT"
+    USER_IMPORT = "USER_IMPORT"
+    USER_EXPORT = "USER_EXPORT"
 
     # List activity
     ADD = "ADD"
@@ -63,6 +73,9 @@ class Notification(BookWyrmModel):
     )
     related_status = models.ForeignKey("Status", on_delete=models.CASCADE, null=True)
     related_import = models.ForeignKey("ImportJob", on_delete=models.CASCADE, null=True)
+    related_user_export = models.ForeignKey(
+        "BookwyrmExportJob", on_delete=models.CASCADE, null=True
+    )
     related_list_items = models.ManyToManyField(
         "ListItem", symmetrical=False, related_name="notifications"
     )
@@ -226,6 +239,36 @@ def notify_user_on_import_complete(
     )
 
 
+@receiver(models.signals.post_save, sender=BookwyrmImportJob)
+# pylint: disable=unused-argument
+def notify_user_on_user_import_complete(
+    sender, instance, *args, update_fields=None, **kwargs
+):
+    """we imported your user details! aren't you proud of us"""
+    update_fields = update_fields or []
+    if not instance.complete or "complete" not in update_fields:
+        return
+    Notification.objects.create(
+        user=instance.user, notification_type=NotificationType.USER_IMPORT
+    )
+
+
+@receiver(models.signals.post_save, sender=BookwyrmExportJob)
+# pylint: disable=unused-argument
+def notify_user_on_user_export_complete(
+    sender, instance, *args, update_fields=None, **kwargs
+):
+    """we exported your user details! aren't you proud of us"""
+    update_fields = update_fields or []
+    if not instance.complete or "complete" not in update_fields:
+        return
+    Notification.objects.create(
+        user=instance.user,
+        notification_type=NotificationType.USER_EXPORT,
+        related_user_export=instance,
+    )
+
+
 @receiver(models.signals.post_save, sender=Report)
 @transaction.atomic
 # pylint: disable=unused-argument

bookwyrm/models/relationship.py

@@ -65,6 +65,13 @@ class UserRelationship(BookWyrmModel):
         base_path = self.user_subject.remote_id
         return f"{base_path}#follows/{self.id}"
 
+    def get_accept_reject_id(self, status):
+        """get id for sending an accept or reject of a local user"""
+
+        base_path = self.user_object.remote_id
+        status_id = self.id or 0
+        return f"{base_path}#{status}/{status_id}"
+
 
 class UserFollows(ActivityMixin, UserRelationship):
     """Following a user"""
@@ -105,6 +112,20 @@ class UserFollows(ActivityMixin, UserRelationship):
         )
         return obj
 
+    def reject(self):
+        """generate a Reject for this follow. This would normally happen
+        when a user deletes a follow they previously accepted"""
+
+        if self.user_object.local:
+            activity = activitypub.Reject(
+                id=self.get_accept_reject_id(status="rejects"),
+                actor=self.user_object.remote_id,
+                object=self.to_activity(),
+            ).serialize()
+            self.broadcast(activity, self.user_object)
+
+        self.delete()
+
 
 class UserFollowRequest(ActivitypubMixin, UserRelationship):
     """following a user requires manual or automatic confirmation"""
@@ -148,13 +169,6 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
             if not manually_approves:
                 self.accept()
 
-    def get_accept_reject_id(self, status):
-        """get id for sending an accept or reject of a local user"""
-
-        base_path = self.user_object.remote_id
-        status_id = self.id or 0
-        return f"{base_path}#{status}/{status_id}"
-
     def accept(self, broadcast_only=False):
         """turn this request into the real deal"""
         user = self.user_object

bookwyrm/models/site.py

@@ -10,8 +10,11 @@ from django.dispatch import receiver
 from django.utils import timezone
 from model_utils import FieldTracker
 
+from bookwyrm.connectors.abstract_connector import get_data
 from bookwyrm.preview_images import generate_site_preview_image_task
 from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL
+from bookwyrm.settings import RELEASE_API
+from bookwyrm.tasks import app, MISC
 from .base_model import BookWyrmModel, new_access_code
 from .user import User
 from .fields import get_absolute_url
@@ -45,7 +48,7 @@ class SiteSettings(SiteModel):
     default_theme = models.ForeignKey(
         "Theme", null=True, blank=True, on_delete=models.SET_NULL
     )
-    version = models.CharField(null=True, blank=True, max_length=10)
+    available_version = models.CharField(null=True, blank=True, max_length=10)
 
     # admin setup options
     install_mode = models.BooleanField(default=False)
@@ -96,6 +99,8 @@ class SiteSettings(SiteModel):
     imports_enabled = models.BooleanField(default=True)
     import_size_limit = models.IntegerField(default=0)
     import_limit_reset = models.IntegerField(default=0)
+    user_exports_enabled = models.BooleanField(default=False)
+    user_import_time_limit = models.IntegerField(default=48)
 
     field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"])
 
@@ -243,3 +248,14 @@ def preview_image(instance, *args, **kwargs):
 
     if len(changed_fields) > 0:
         generate_site_preview_image_task.delay()
+
+
+@app.task(queue=MISC)
+def check_for_updates_task():
+    """See if git remote knows about a new version"""
+    site = SiteSettings.objects.get()
+    release = get_data(RELEASE_API, timeout=3)
+    available_version = release.get("tag_name", None)
+    if available_version:
+        site.available_version = available_version
+        site.save(update_fields=["available_version"])

bookwyrm/models/status.py

@@ -12,6 +12,8 @@ from django.db.models import Q
 from django.dispatch import receiver
 from django.template.loader import get_template
 from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ngettext_lazy
 from model_utils import FieldTracker
 from model_utils.managers import InheritanceManager
 
@@ -107,14 +109,14 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
     @property
     def recipients(self):
         """tagged users who definitely need to get this status in broadcast"""
-        mentions = [u for u in self.mention_users.all() if not u.local]
+        mentions = {u for u in self.mention_users.all() if not u.local}
         if (
             hasattr(self, "reply_parent")
             and self.reply_parent
             and not self.reply_parent.user.local
         ):
-            mentions.append(self.reply_parent.user)
+            mentions.add(self.reply_parent.user)
-        return list(set(mentions))
+        return list(mentions)
 
     @classmethod
     def ignore_activity(
@@ -178,6 +180,24 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         """you can't boost dms"""
         return self.privacy in ["unlisted", "public"]
 
+    @property
+    def page_title(self):
+        """title of the page when only this status is shown"""
+        return _("%(display_name)s's status") % {"display_name": self.user.display_name}
+
+    @property
+    def page_description(self):
+        """description of the page in meta tags when only this status is shown"""
+        return None
+
+    @property
+    def page_image(self):
+        """image to use as preview in meta tags when only this status is shown"""
+        if self.mention_books.exists():
+            book = self.mention_books.first()
+            return book.preview_image or book.cover
+        return self.user.preview_image
+
     def to_replies(self, **kwargs):
         """helper function for loading AP serialized replies to a status"""
         return self.to_ordered_collection(
@@ -301,6 +321,10 @@ class BookStatus(Status):
 
         abstract = True
 
+    @property
+    def page_image(self):
+        return self.book.preview_image or self.book.cover or super().page_image
+
 
 class Comment(BookStatus):
     """like a review but without a rating and transient"""
@@ -332,17 +356,26 @@ class Comment(BookStatus):
 
     activity_serializer = activitypub.Comment
 
+    @property
+    def page_title(self):
+        return _("%(display_name)s's comment on %(book_title)s") % {
+            "display_name": self.user.display_name,
+            "book_title": self.book.title,
+        }
+
 
 class Quotation(BookStatus):
     """like a review but without a rating and transient"""
 
     quote = fields.HtmlField()
     raw_quote = models.TextField(blank=True, null=True)
-    position = models.IntegerField(
+    position = models.TextField(
-        validators=[MinValueValidator(0)], null=True, blank=True
+        null=True,
+        blank=True,
     )
-    endposition = models.IntegerField(
+    endposition = models.TextField(
-        validators=[MinValueValidator(0)], null=True, blank=True
+        null=True,
+        blank=True,
     )
     position_mode = models.CharField(
         max_length=3,
@@ -374,6 +407,13 @@ class Quotation(BookStatus):
 
     activity_serializer = activitypub.Quotation
 
+    @property
+    def page_title(self):
+        return _("%(display_name)s's quote from %(book_title)s") % {
+            "display_name": self.user.display_name,
+            "book_title": self.book.title,
+        }
+
 
 class Review(BookStatus):
     """a book review"""
@@ -403,6 +443,13 @@ class Review(BookStatus):
         """indicate the book in question for mastodon (or w/e) users"""
         return self.content
 
+    @property
+    def page_title(self):
+        return _("%(display_name)s's review of %(book_title)s") % {
+            "display_name": self.user.display_name,
+            "book_title": self.book.title,
+        }
+
     activity_serializer = activitypub.Review
     pure_type = "Article"
 
@@ -426,6 +473,18 @@ class ReviewRating(Review):
         template = get_template("snippets/generated_status/rating.html")
         return template.render({"book": self.book, "rating": self.rating}).strip()
 
+    @property
+    def page_description(self):
+        return ngettext_lazy(
+            "%(display_name)s rated %(book_title)s: %(display_rating).1f star",
+            "%(display_name)s rated %(book_title)s: %(display_rating).1f stars",
+            "display_rating",
+        ) % {
+            "display_name": self.user.display_name,
+            "book_title": self.book.title,
+            "display_rating": self.rating,
+        }
+
     activity_serializer = activitypub.Rating
     pure_type = "Note"
 

bookwyrm/models/user.py

@@ -523,6 +523,20 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
         return super().save(*args, **kwargs)
 
 
+@app.task(queue=MISC)
+def erase_user_data(user_id):
+    """Erase any custom data about this user asynchronously
+    This is for deleted historical user data that pre-dates data
+    being cleared automatically"""
+    user = User.objects.get(id=user_id)
+    user.erase_user_data()
+    user.save(
+        broadcast=False,
+        update_fields=["email", "avatar", "preview_image", "summary", "name"],
+    )
+    user.erase_user_statuses(broadcast=False)
+
+
 @app.task(queue=MISC)
 def set_remote_server(user_id, allow_external_connections=False):
     """figure out the user's remote server in the background"""

bookwyrm/settings.py

@@ -30,8 +30,11 @@ RELEASE_API = env(
 
 PAGE_LENGTH = env.int("PAGE_LENGTH", 15)
 DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English")
+# TODO: extend maximum age to 1 year once termination of active sessions
+# is implemented (see bookwyrm-social#2278, bookwyrm-social#3082).
+SESSION_COOKIE_AGE = env.int("SESSION_COOKIE_AGE", 3600 * 24 * 30)  # 1 month
 
-JS_CACHE = "ac315a3b"
+JS_CACHE = "8a89cad7"
 
 # email
 EMAIL_BACKEND = env("EMAIL_BACKEND", "django.core.mail.backends.smtp.EmailBackend")
@@ -99,6 +102,7 @@ INSTALLED_APPS = [
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "django.contrib.humanize",
+    "file_resubmit",
     "sass_processor",
     "bookwyrm",
     "celery",
@@ -120,6 +124,7 @@ MIDDLEWARE = [
     "bookwyrm.middleware.IPBlocklistMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "bookwyrm.middleware.FileTooBig",
 ]
 
 ROOT_URLCONF = "bookwyrm.urls"
@@ -243,7 +248,11 @@ if env.bool("USE_DUMMY_CACHE", False):
     CACHES = {
         "default": {
             "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-        }
+        },
+        "file_resubmit": {
+            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+            "LOCATION": "/tmp/file_resubmit_tests/",
+        },
     }
 else:
     CACHES = {
@@ -253,7 +262,11 @@ else:
             "OPTIONS": {
                 "CLIENT_CLASS": "django_redis.client.DefaultClient",
             },
-        }
+        },
+        "file_resubmit": {
+            "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+            "LOCATION": "/tmp/file_resubmit/",
+        },
     }
 
     SESSION_ENGINE = "django.contrib.sessions.backends.cache"
@@ -309,6 +322,7 @@ LANGUAGES = [
     ("eu-es", _("Euskara (Basque)")),
     ("gl-es", _("Galego (Galician)")),
     ("it-it", _("Italiano (Italian)")),
+    ("ko-kr", _("한국어 (Korean)")),
     ("fi-fi", _("Suomi (Finnish)")),
     ("fr-fr", _("Français (French)")),
     ("lt-lt", _("Lietuvių (Lithuanian)")),
@@ -319,6 +333,7 @@ LANGUAGES = [
     ("pt-pt", _("Português Europeu (European Portuguese)")),
     ("ro-ro", _("Română (Romanian)")),
     ("sv-se", _("Svenska (Swedish)")),
+    ("uk-ua", _("Українська (Ukrainian)")),
     ("zh-hans", _("简体中文 (Simplified Chinese)")),
     ("zh-hant", _("繁體中文 (Traditional Chinese)")),
 ]
@@ -337,8 +352,7 @@ USE_L10N = True
 USE_TZ = True
 
 
-agent = requests.utils.default_user_agent()
+USER_AGENT = f"BookWyrm (BookWyrm/{VERSION}; +https://{DOMAIN}/)"
-USER_AGENT = f"{agent} (BookWyrm/{VERSION}; +https://{DOMAIN}/)"
 
 # Imagekit generated thumbnails
 ENABLE_THUMBNAIL_GENERATION = env.bool("ENABLE_THUMBNAIL_GENERATION", False)
@@ -432,3 +446,7 @@ if HTTP_X_FORWARDED_PROTO:
 # Do not change this setting unless you already have an existing
 # user with the same username - in which case you should change it!
 INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"
+
+# We only allow specifying DATA_UPLOAD_MAX_MEMORY_SIZE in MiB from .env
+# (note the difference in variable names).
+DATA_UPLOAD_MAX_MEMORY_SIZE = env.int("DATA_UPLOAD_MAX_MEMORY_MiB", 100) << 20

bookwyrm/static/js/autocomplete.js

@@ -111,6 +111,10 @@ const tries = {
             },
         },
         f: {
+            b: {
+                2: "FB2",
+                3: "FB3",
+            },
             l: {
                 a: {
                     c: "FLAC",

bookwyrm/templates/413.html

@@ -0,0 +1,16 @@
+{% extends 'layout.html' %}
+{% load i18n %}
+
+{% block title %}{% trans "File too large" %}{% endblock %}
+
+{% block content %}
+<div class="block">
+    <h1 class="title">{% trans "File too large" %}</h1>
+    <p class="content">{% trans "The file you are uploading is too large." %}</p>
+    <p class="content">
+        {% blocktrans %}
+        You you can try using a smaller file, or ask your BookWyrm server administrator to increase the <code>DATA_UPLOAD_MAX_MEMORY_SIZE</code> setting.
+        {% endblocktrans %}
+    </p>
+</div>
+{% endblock %}

bookwyrm/templates/about/about.html

@@ -31,10 +31,10 @@
         </p>
     </div>
 
-    <div class="columns">
+    <div class="columns is-multiline">
         {% if superlatives.top_rated %}
         {% with book=superlatives.top_rated.default_edition rating=superlatives.top_rated.rating %}
-        <div class="column is-one-third is-flex">
+        <div class="column is-half-tablet is-one-third-desktop is-flex-grow-1 is-flex">
             <div class="media notification is-clipped">
                 <div class="media-left">
                     <a href="{{ book.local_path }}">
@@ -53,7 +53,7 @@
 
         {% if superlatives.wanted %}
         {% with book=superlatives.wanted.default_edition %}
-        <div class="column is-one-third is-flex">
+        <div class="column is-half-tablet is-one-third-desktop is-flex-grow-1 is-flex">
             <div class="media notification is-clipped">
                 <div class="media-left">
                     <a href="{{ book.local_path }}">
@@ -72,7 +72,7 @@
 
         {% if superlatives.controversial %}
         {% with book=superlatives.controversial.default_edition %}
-        <div class="column is-one-third is-flex">
+        <div class="column is-half-tablet is-one-third-desktop is-flex-grow-1 is-flex">
             <div class="media notification is-clipped">
                 <div class="media-left">
                     <a href="{{ book.local_path }}">

bookwyrm/templates/book/book.html

@@ -9,7 +9,8 @@
 {% block title %}{{ book|book_title }}{% endblock %}
 
 {% block opengraph %}
-    {% include 'snippets/opengraph.html' with title=book.title description=book|book_description image=book.preview_image %}
+    {% firstof book.preview_image book.cover as book_image %}
+    {% include 'snippets/opengraph.html' with title=book.title description=book|book_description image=book_image %}
 {% endblock %}
 
 {% block content %}

bookwyrm/templates/confirm_email/confirm_email.html

@@ -6,8 +6,8 @@
 {% block content %}
 <h1 class="title">{% trans "Confirm your email address" %}</h1>
 
-<div class="columns">
+<div class="columns is-multiline">
-    <div class="column">
+    <div class="column is-full is-half-desktop">
         <div class="block content">
             <section class="block">
                 <p>{% trans "A confirmation code has been sent to the email address you used to register your account." %}</p>

bookwyrm/templates/feed/feed.html

@@ -41,7 +41,7 @@
     </section>
     {% endif %}
 
-    {% if annual_summary_year and tab.key == 'home' %}
+    {% if annual_summary_year and tab.key == 'home' and has_summary_read_throughs %}
     <section class="block is-hidden" data-hide="hide_annual_summary_{{ annual_summary_year }}">
         {% include 'feed/summary_card.html' with year=annual_summary_year %}
         <hr>

bookwyrm/templates/feed/status.html

@@ -2,13 +2,11 @@
 {% load feed_page_tags %}
 {% load i18n %}
 
+{% block title %}{{ title }}{% endblock %}
+
+
 {% block opengraph %}
-    {% firstof status.book status.mention_books.first as book %}
+    {% include 'snippets/opengraph.html' with image=page_image %}
-    {% if book %}
-        {% include 'snippets/opengraph.html' with image=preview %}
-    {% else %}
-        {% include 'snippets/opengraph.html' %}
-    {% endif %}
 {% endblock %}
 
 

bookwyrm/templates/import/import.html

@@ -1,13 +1,12 @@
-{% extends 'layout.html' %}
+{% extends 'preferences/layout.html' %}
 {% load i18n %}
 {% load humanize %}
 
-{% block title %}{% trans "Import Books" %}{% endblock %}
+{% block title %}{% trans "Import Book List" %}{% endblock %}
+{% block header %}{% trans "Import Book List" %}{% endblock %}
 
-{% block content %}
+{% block panel %}
 <div class="block">
-    <h1 class="title">{% trans "Import Books" %}</h1>
-
     {% if invalid %}
     <div class="notification is-danger">
         {% trans "Not a valid CSV file" %}

bookwyrm/templates/import/import_user.html

@@ -0,0 +1,222 @@
+{% extends 'preferences/layout.html' %}
+{% load i18n %}
+{% load humanize %}
+
+{% block title %}{% trans "Import BookWyrm Account" %}{% endblock %}
+{% block header %}{% trans "Import BookWyrm Account" %}{% endblock %}
+
+{% block panel %}
+<div class="block">
+
+    {% if invalid %}
+    <div class="notification is-danger">
+        {% trans "Not a valid import file" %}
+    </div>
+    {% endif %}
+    <p class="notification is-warning">
+        {% spaceless %}
+        {% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set this account as an <strong>alias</strong> of the one you are migrating from, or <strong>move</strong> that account to this one, before you import your user data." %}
+        {% endspaceless %}
+    </p>
+    {% if not site.imports_enabled %}
+    <div class="box notification has-text-centered is-warning m-6 content">
+        <p class="mt-5">
+            <span class="icon icon-warning is-size-2" aria-hidden="true"></span>
+        </p>
+        <p class="mb-5">
+            {% trans "Imports are temporarily disabled; thank you for your patience." %}
+        </p>
+    </div>
+    {% elif next_available %}
+        <div class="notification is-warning">
+            <p>{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}</p>
+            <p>{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}</p>
+        </div>
+    {% else %}
+    <form class="box content" name="import-user" action="/user-import" method="post" enctype="multipart/form-data">
+        {% csrf_token %}
+
+        <div class="block">
+            <div class="notification">
+                <h2 class="is-size-5">{% trans "Step 1:" %}</h2>
+                <p>
+                    {% blocktrans trimmed %}
+                    Select an export file generated from another BookWyrm account. The file format should be <code>.tar.gz</code>.
+                    {% endblocktrans %}
+                </p>
+            </div>
+            <div class="block m-5">
+                <label class="label" for="id_archive_file">{% trans "Data file:" %}</label>
+                {{ import_form.archive_file }}
+            </div>
+        </div>
+
+        <hr aria-hidden="true">
+
+        <div class="block">
+            <div class="notification">
+                <h2 class="is-size-5">{% trans "Step 2:" %}</h2>
+                <p>
+                    {% blocktrans trimmed %}
+                    Deselect any checkboxes for data you do not wish to include in your import.
+                    {% endblocktrans %}
+                </p>
+                <p class="block">Unless specified below, importing will not delete any data. Imported data will be <strong>added if it does not already exist</strong>. For example, if you have an existing list with the same name as an imported list, the existing list settings will not change, any new list items will be added, and no existing list items will be deleted.</p>
+            </div>
+            <div class="block m-5 columns">
+                <div class="column is-half">
+                    <div class="field">
+                        <label class="label mb-0">
+                            <input type="checkbox" name="include_user_profile" checked aria-describedby="desc_include_user_profile">
+                            {% trans "User profile" %}
+                        </label>
+                        <p id="desc_include_user_profile">
+                        {% trans "Overwrites display name, summary, and avatar" %}
+                        </p>
+                    </div>
+                    <div class="field">
+                        <label class="label mb-0">
+                            <input type="checkbox" name="include_user_settings" checked aria-describedby="desc_include_user_settings">
+                            {% trans "User settings" %}
+                        </label>
+                        <div id="desc_include_user_settings">
+                            {% trans "Overwrites:" %}
+                            <ul class="mt-0">
+                                <li>
+                                    {% trans "Whether manual approval is required for other users to follow your account" %}
+                                </li>
+                                <li>
+                                    {% trans "Whether following/followers are shown on your profile" %}
+                                </li>
+                                <li>
+                                    {% trans "Whether your reading goal is shown on your profile" %}
+                                </li>
+                                <li>
+                                    {% trans "Whether you see user follow suggestions" %}
+                                </li>
+                                <li>
+                                    {% trans "Whether your account is suggested to others" %}
+                                </li>
+                                <li>
+                                    {% trans "Your timezone" %}
+                                </li>
+                                <li>
+                                    {% trans "Your default post privacy setting" %}
+                                </li>
+                            </ul>
+                        </div>
+                    </div>
+                    <div class="field">
+                        <label class="label">
+                            <input type="checkbox" name="include_follows" checked>
+                            {% trans "Followers and following" %}
+                        </label>
+                    </div>
+                    <label class="label">
+                        <input type="checkbox" name="include_blocks" checked> {% trans "User blocks" %}
+                    </label>
+                </div>
+                <div class="column is-half">
+                    <div class="field">
+                        <label class="label mb-0">
+                            <input type="checkbox" name="include_goals" checked aria-describedby="desc_include_goals">
+                            {% trans "Reading goals" %}
+                        </label>
+                        <p id="desc_include_goals">
+                            {% trans "Overwrites reading goals for all years listed in the import file" %}
+                        </p>
+                    </div>
+                    <label class="label">
+                        <input type="checkbox" name="include_shelves" checked> {% trans "Shelves" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_readthroughs" checked> {% trans "Reading history" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_reviews" checked> {% trans "Book reviews" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_quotations" checked> {% trans "Quotations" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_comments" checked> {% trans "Comments about books" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_lists" checked> {% trans "Book lists" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_saved_lists" checked> {% trans "Saved lists" %}
+                    </label>
+                </div>
+            </div>
+        </div>
+
+        {% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %}
+            <button class="button is-primary" type="submit">{% trans "Import" %}</button>
+        {% else %}
+            <button class="button is-primary is-disabled" type="submit">{% trans "Import" %}</button>
+            <p>{% trans "You've reached the import limit." %}</p>
+        {% endif%}
+    </form>
+    {% endif %}
+
+</div>
+
+<div class="content block">
+    <h2 class="title">{% trans "Recent Imports" %}</h2>
+    <div class="table-container">
+        <table class="table is-striped is-fullwidth">
+            <tr>
+                <th>
+                    {% trans "Date Created" %}
+                </th>
+                <th>
+                    {% trans "Last Updated" %}
+                </th>
+                <th>
+                    {% trans "Status" %}
+                </th>
+            </tr>
+            {% if not jobs %}
+            <tr>
+                <td colspan="4">
+                    <em>{% trans "No recent imports" %}</em>
+                </td>
+            </tr>
+            {% endif %}
+            {% for job in jobs %}
+            <tr>
+                <td>
+                    <p>{{ job.created_date }}</p>
+                </td>
+                <td>{{ job.updated_date }}</td>
+                <td>
+                    <span
+                        {% if job.status == "stopped" or job.status == "failed" %}
+                        class="tag is-danger"
+                        {% elif job.status == "pending" %}
+                        class="tag is-warning"
+                        {% elif job.complete %}
+                        class="tag"
+                        {% else %}
+                        class="tag is-success"
+                        {% endif %}
+                    >
+                        {% if job.status %}
+                        {{ job.status }}
+                            {{ job.status_display }}
+                        {% elif job.complete %}
+                            {% trans "Complete" %}
+                        {% else %}
+                            {% trans "Active" %}
+                        {% endif %}
+                    </span>
+                </td>
+            </tr>
+            {% endfor %}
+        </table>
+    </div>
+
+    {% include 'snippets/pagination.html' with page=jobs path=request.path %}
+</div>
+{% endblock %}

bookwyrm/templates/landing/invite.html

@@ -6,8 +6,8 @@
 {% block content %}
 
 <h1 class="title">{% trans "Create an Account" %}</h1>
-<div class="columns">
+<div class="columns is-multiline">
-    <div class="column">
+    <div class="column is-full is-half-desktop">
         <div class="block">
             {% if valid %}
             <div>

bookwyrm/templates/landing/login.html

@@ -6,7 +6,7 @@
 {% block content %}
 <h1 class="title">{% trans "Log in" %}</h1>
 <div class="columns is-multiline">
-    <div class="column is-half">
+    <div class="column {% if site.allow_registration %}is-half{% else %}is-full is-half-desktop{% endif %}">
         {% if login_form.non_field_errors %}
         <p class="notification is-danger">{{ login_form.non_field_errors }}</p>
         {% endif %}
@@ -20,13 +20,15 @@
             <div class="field">
                 <label class="label" for="id_localname_confirm">{% trans "Username:" %}</label>
                 <div class="control">
-                    <input type="text" name="localname" maxlength="255" class="input" required="" id="id_localname_confirm" value="{{ login_form.localname.value|default:'' }}">
+                    <input type="text" name="localname" maxlength="255" class="input" required=""
+                        id="id_localname_confirm" value="{{ login_form.localname.value|default:'' }}">
                 </div>
             </div>
             <div class="field">
                 <label class="label" for="id_password_confirm">{% trans "Password:" %}</label>
                 <div class="control">
-                    <input type="password" name="password" maxlength="128" class="input" required="" id="id_password_confirm" aria-describedby="desc_password">
+                    <input type="password" name="password" maxlength="128" class="input" required=""
+                        id="id_password_confirm" aria-describedby="desc_password">
                 </div>
 
                 {% include 'snippets/form_errors.html' with errors_list=login_form.password.errors id="desc_password" %}

bookwyrm/templates/landing/password_reset.html

@@ -4,8 +4,8 @@
 {% block title %}{% trans "Reset Password" %}{% endblock %}
 
 {% block content %}
-<div class="columns">
+<div class="columns is-multiline">
-    <div class="column">
+    <div class="column is-full is-half-desktop">
         <div class="block">
             <h1 class="title">{% trans "Reset Password" %}</h1>
 

bookwyrm/templates/landing/reactivate.html

@@ -6,7 +6,7 @@
 {% block content %}
 <h1 class="title">{% trans "Reactivate Account" %}</h1>
 <div class="columns is-multiline">
-    <div class="column is-half">
+    <div class="column {% if site.allow_registration %}is-half{% else %}is-full is-half-desktop{% endif %}">
         {% if login_form.non_field_errors %}
         <p class="notification is-danger">{{ login_form.non_field_errors }}</p>
         {% endif %}
@@ -16,13 +16,15 @@
             <div class="field">
                 <label class="label" for="id_localname_confirm">{% trans "Username:" %}</label>
                 <div class="control">
-                    <input type="text" name="localname" maxlength="255" class="input" required="" id="id_localname_confirm" value="{{ login_form.localname.value|default:'' }}">
+                    <input type="text" name="localname" maxlength="255" class="input" required=""
+                        id="id_localname_confirm" value="{{ login_form.localname.value|default:'' }}">
                 </div>
             </div>
             <div class="field">
                 <label class="label" for="id_password_confirm">{% trans "Password:" %}</label>
                 <div class="control">
-                    <input type="password" name="password" maxlength="128" class="input" required="" id="id_password_confirm" aria-describedby="desc_password">
+                    <input type="password" name="password" maxlength="128" class="input" required=""
+                        id="id_password_confirm" aria-describedby="desc_password">
                 </div>
 
                 {% include 'snippets/form_errors.html' with errors_list=login_form.password.errors id="desc_password" %}

bookwyrm/templates/layout.html

@@ -36,7 +36,7 @@
                 <div class="field has-addons">
                     <div class="control">
                         {% if request.user.is_authenticated %}
-                            {% trans "Search for a book, user, or list" as search_placeholder %}
+                            {% trans "Search for a book, author, user, or list" as search_placeholder %}
                         {% else %}
                             {% trans "Search for a book" as search_placeholder %}
                         {% endif %}

bookwyrm/templates/notifications/item.html

@@ -10,11 +10,13 @@
 {% elif notification.notification_type == 'FOLLOW' %}
     {% include 'notifications/items/follow.html' %}
 {% elif notification.notification_type == 'FOLLOW_REQUEST' %}
-    {% if notification.related_users.0.is_active %}
     {% include 'notifications/items/follow_request.html' %}
-    {% endif %}
 {% elif notification.notification_type == 'IMPORT' %}
     {% include 'notifications/items/import.html' %}
+{% elif notification.notification_type == 'USER_IMPORT' %}
+    {% include 'notifications/items/user_import.html' %}
+{% elif notification.notification_type == 'USER_EXPORT' %}
+    {% include 'notifications/items/user_export.html' %}
 {% elif notification.notification_type == 'ADD' %}
     {% include 'notifications/items/add.html' %}
 {% elif notification.notification_type == 'REPORT' %}

bookwyrm/templates/notifications/items/move_user.html

@@ -14,7 +14,7 @@
 
 {% block description %}
     {% if related_user_moved_to %}
-        {% id_to_username request.user.moved_to  as username %}
+        {% id_to_username related_user_moved_to as username %}
         {% blocktrans trimmed %}
         {{ related_user }} has moved to <a href="{{ related_user_moved_to }}">{{ username }}</a>
         {% endblocktrans %}

bookwyrm/templates/notifications/items/user_export.html

@@ -0,0 +1,15 @@
+{% extends 'notifications/items/layout.html' %}
+{% load i18n %}
+
+{% block primary_link %}{% spaceless %}
+{% url 'prefs-user-export' %}
+{% endspaceless %}{% endblock %}
+
+{% block icon %}
+<span class="icon icon-list"></span>
+{% endblock %}
+
+{% block description %}
+    {% url 'prefs-user-export'  as url %}
+    {% blocktrans %}Your <a href="{{ url }}">user export</a> is ready.{% endblocktrans %}
+{% endblock %}

bookwyrm/templates/notifications/items/user_import.html

@@ -0,0 +1,16 @@
+{% extends 'notifications/items/layout.html' %}
+{% load i18n %}
+
+{% block primary_link %}{% spaceless %}
+{% url 'user-import' %}
+{% endspaceless %}{% endblock %}
+
+{% block icon %}
+<span class="icon icon-list"></span>
+{% endblock %}
+
+{% block description %}
+{% url 'user-import' as import_url %}
+{% blocktrans %}Your <a href="{{ import_url }}">user import</a> is complete.{% endblocktrans %}
+
+{% endblock %}

bookwyrm/templates/preferences/export-user.html

@@ -0,0 +1,147 @@
+{% extends 'preferences/layout.html' %}
+{% load i18n %}
+{% load utilities %}
+
+{% block title %}{% trans "Export BookWyrm Account" %}{% endblock %}
+
+{% block header %}
+{% trans "Export BookWyrm Account" %}
+{% endblock %}
+
+{% block panel %}
+<div class="block content">
+    <div class="block content">
+        <p> {% trans "You can create an export file here. This will allow you to migrate your data to another BookWyrm account." %}</p>
+    </div>
+        <div class="block mx-5 columns">
+            <div class="column is-half">
+                <h2 class="is-size-5">{% trans "Your file will include:" %}</h2>
+                <ul>
+                    <li>{% trans "User profile" %}</li>
+                    <li>{% trans "Most user settings" %}</li>
+                    <li>{% trans "Reading goals" %}</li>
+                    <li>{% trans "Shelves" %}</li>
+                    <li>{% trans "Reading history" %}</li>
+                    <li>{% trans "Book reviews" %}</li>
+                    <li>{% trans "Statuses" %}</li>
+                    <li>{% trans "Your own lists and saved lists" %}</li>
+                    <li>{% trans "Which users you follow and block" %}</li>
+                </ul>
+            </div>
+            <div class="column is-half">
+                <h2 class="is-size-5">{% trans "Your file will not include:" %}</h2>
+                <ul>
+                    <li>{% trans "Direct messages" %}</li>
+                    <li>{% trans "Replies to your statuses" %}</li>
+                    <li>{% trans "Groups" %}</li>
+                    <li>{% trans "Favorites" %}</li>
+                </ul>
+            </div>
+        </div>
+        <p class="block">{% trans "In your new BookWyrm account can choose what to import: you will not have to import everything that is exported." %}</p>
+        <p class="notification is-warning">
+            {% spaceless %}
+            {% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set the account you are moving to as an <strong>alias</strong> of this one, or <strong>move</strong> this account to the new account, before you import your user data." %}
+            {% endspaceless %}
+        </p>
+    {% if not site.user_exports_enabled %}
+    <p class="notification is-danger">
+        {% trans "New user exports are currently disabled." %}
+        {% if perms.bookwyrm.edit_instance_settings %}
+        <br/>
+        {% url 'settings-imports' as url %}
+        {% blocktrans trimmed %}
+            User exports settings can be changed from <a href="{{ url }}">the Imports page</a> in the Admin dashboard.
+        {% endblocktrans %}
+    {% endif%}
+    </p>
+    {% elif next_available %}
+    <p class="notification is-warning">
+        {% blocktrans trimmed %}
+        You will be able to create a new export file at {{ next_available }}
+        {% endblocktrans %}
+    </p>
+    {% else %}
+    <form name="export" method="POST" href="{% url 'prefs-user-export' %}">
+        {% csrf_token %}
+        <button type="submit" class="button">
+            <span class="icon icon-download" aria-hidden="true"></span>
+            <span>{% trans "Create user export file" %}</span>
+        </button>
+    </form>
+    {% endif %}
+
+</div>
+<div class="content block">
+    <h2 class="title">{% trans "Recent Exports" %}</h2>
+    <p class="content">
+    {% trans "User export files will show 'complete' once ready. This may take a little while. Click the link to download your file." %}
+    </p>
+    <div class="table-container">
+        <table class="table is-striped is-fullwidth">
+            <tr>
+                <th>
+                    {% trans "Date" %}
+                </th>
+                <th>
+                    {% trans "Status" %}
+                </th>
+                <th colspan="2">
+                    {% trans "Size" %}
+                </th>
+            </tr>
+            {% if not jobs %}
+            <tr>
+                <td colspan="4">
+                    <em>{% trans "No recent imports" %}</em>
+                </td>
+            </tr>
+            {% endif %}
+            {% for job in jobs %}
+            <tr>
+                <td>{{ job.updated_date }}</td>
+                <td>
+                    <span
+                        {% if job.status == "stopped" or job.status == "failed" %}
+                        class="tag is-danger"
+                        {% elif job.status == "pending" %}
+                        class="tag is-warning"
+                        {% elif job.complete %}
+                        class="tag"
+                        {% else %}
+                        class="tag is-success"
+                        {% endif %}
+                    >
+                        {% if job.status %}
+                        {{ job.status }}
+                            {{ job.status_display }}
+                        {% elif job.complete %}
+                            {% trans "Complete" %}
+                        {% else %}
+                            {% trans "Active" %}
+                        {% endif %}
+                    </span>
+                </td>
+                <td>
+                    <span>{{ job.export_data|get_file_size }}</span>
+                </td>
+                <td>
+                    {% if job.complete and not job.status == "stopped" and not job.status == "failed" %}
+                    <p>
+                        <a download="" href="/preferences/user-export/{{ job.task_id }}">
+                            <span class="icon icon-download" aria-hidden="true"></span>
+                            <span class="is-hidden-mobile">
+                                {% trans "Download your export" %}
+                            </span>
+                        </a>
+                    </p>
+                    {% endif %}
+                </td>
+            </tr>
+            {% endfor %}
+        </table>
+    </div>
+
+    {% include 'snippets/pagination.html' with page=jobs path=request.path %}
+</div>
+{% endblock %}

bookwyrm/templates/preferences/export.html

@@ -1,16 +1,16 @@
 {% extends 'preferences/layout.html' %}
 {% load i18n %}
 
-{% block title %}{% trans "CSV Export" %}{% endblock %}
+{% block title %}{% trans "Export Book List" %}{% endblock %}
 
 {% block header %}
-{% trans "CSV Export" %}
+{% trans "Export Book List" %}
 {% endblock %}
 
 {% block panel %}
 <div class="block content">
     <p class="notification">
-        {% trans "Your export will include all the books on your shelves, books you have reviewed, and books with reading activity." %}
+        {% trans "Your CSV export file will include all the books on your shelves, books you have reviewed, and books with reading activity. <br/>Use this to import into a service like Goodreads." %}
     </p>
     <p>
         <form name="export" method="POST" href="{% url 'prefs-export' %}">

bookwyrm/templates/preferences/layout.html

@@ -40,11 +40,19 @@
         <ul class="menu-list">
             <li>
                 {% url 'import' as url %}
-                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Import" %}</a>
+                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Import Book List" %}</a>
             </li>
             <li>
                 {% url 'prefs-export' as url %}
-                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "CSV export" %}</a>
+                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Export Book List" %}</a>
+            </li>
+            <li>
+                {% url 'user-import' as url %}
+                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Import BookWyrm Account" %}</a>
+            </li>
+            <li>
+                {% url 'prefs-user-export' as url %}
+                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Export BookWyrm Account" %}</a>
             </li>
         </ul>
         <h2 class="menu-label">{% trans "Relationships" %}</h2>

bookwyrm/templates/search/author.html

@@ -0,0 +1,17 @@
+{% extends 'search/layout.html' %}
+
+{% block panel %}
+
+{% if results %}
+<ul class="block">
+{% for author in results %}
+    <li class="">
+        <a href="{{ author.local_path }}" class="author" itemprop="author" itemscope itemtype="https://schema.org/Thing">
+            <span itemprop="name">{{ author.name }}</span>
+        </a>
+    </li>
+{% endfor %}
+</ul>
+{% endif %}
+
+{% endblock %}

bookwyrm/templates/search/layout.html

@@ -20,6 +20,7 @@
             <div class="select" aria-label="{% trans 'Search type' %}">
                 <select name="type">
                     <option value="book" {% if type == "book" %}selected{% endif %}>{% trans "Books" %}</option>
+                    <option value="author" {% if type == "author" %}selected{% endif %}>{% trans "Authors" %}</option>
                     {% if request.user.is_authenticated %}
                     <option value="user" {% if type == "user" %}selected{% endif %}>{% trans "Users" %}</option>
                     {% endif %}
@@ -42,6 +43,9 @@
         <li{% if type == "book" %} class="is-active"{% endif %}>
             <a href="{% url 'search' %}?q={{ query }}&type=book">{% trans "Books" %}</a>
         </li>
+        <li{% if type == "author" %} class="is-active"{% endif %}>
+            <a href="{% url 'search' %}?q={{ query }}&type=author">{% trans "Authors" %}</a>
+        </li>
         {% if request.user.is_authenticated %}
         <li{% if type == "user" %} class="is-active"{% endif %}>
             <a href="{% url 'search' %}?q={{ query }}&type=user">{% trans "Users" %}</a>

bookwyrm/templates/settings/dashboard/dashboard.html

@@ -45,6 +45,10 @@
     {% include 'settings/dashboard/warnings/update_version.html' with warning_level="warning" fullwidth=True %}
     {% endif %}
 
+    {% if schedule_form %}
+    {% include 'settings/dashboard/warnings/check_for_updates.html' with warning_level="success" fullwidth=True %}
+    {% endif %}
+
     {% if missing_privacy or missing_conduct %}
     <div class="column is-12 columns m-0 p-0">
         {% if missing_privacy %}

bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html

@@ -0,0 +1,24 @@
+{% extends 'settings/dashboard/warnings/layout.html' %}
+{% load i18n %}
+
+{% block warning_link %}#{% endblock %}
+
+{% block warning_text %}
+
+<form name="check-version" method="POST" action="{% url 'settings-dashboard' %}" class="is-flex is-align-items-center">
+    {% csrf_token %}
+
+    <p class="pr-2">
+        {% blocktrans trimmed with current=current_version available=available_version %}
+            Would you like to automatically check for new BookWyrm releases? (recommended)
+        {% endblocktrans %}
+    </p>
+
+    {{ schedule_form.every.as_hidden }}
+    {{ schedule_form.period.as_hidden }}
+
+    <button class="button is-small" type="submit">{% trans "Schedule checks" %}</button>
+</form>
+
+{% endblock %}
+

bookwyrm/templates/settings/imports/complete_user_import_modal.html

@@ -0,0 +1,23 @@
+{% extends 'components/modal.html' %}
+{% load i18n %}
+
+{% block modal-title %}{% trans "Stop import?" %}{% endblock %}
+
+{% block modal-body %}
+{% trans "This action will stop the user import before it is complete and cannot be un-done" %}
+{% endblock %}
+
+{% block modal-footer %}
+<form name="complete-import-{{ import.id }}" action="{% url 'settings-user-import-complete' import.id %}" method="POST" class="is-flex-grow-1">
+    {% csrf_token %}
+    <input type="hidden" name="id" value="{{ import.id }}">
+    <div class="buttons is-right is-flex-grow-1">
+        <button type="button" class="button" data-modal-close>
+            {% trans "Cancel" %}
+        </button>
+        <button class="button is-danger" type="submit">
+            {% trans "Confirm" %}
+        </button>
+    </div>
+</form>
+{% endblock %}

bookwyrm/templates/settings/imports/imports.html

@@ -29,6 +29,7 @@
             <div class="notification">
                 {% trans "This is only intended to be used when things have gone very wrong with imports and you need to pause the feature while addressing issues." %}
                 {% trans "While imports are disabled, users will not be allowed to start new imports, but existing imports will not be affected." %}
+                {% trans "This setting prevents both book imports and user imports." %}
             </div>
             {% csrf_token %}
             <div class="control">
@@ -89,8 +90,89 @@
             </div>
         </form>
     </details>
+
+    {% if site.user_exports_enabled %}
+    <details class="details-panel box">
+        <summary>
+            <span role="heading" aria-level="2" class="title is-6">
+                {% trans "Disable starting new user exports" %}
+            </span>
+            <span class="details-close icon icon-x" aria-hidden="true"></span>
+        </summary>
+        <form
+            name="disable-user-exports"
+            id="disable-user-exports"
+            method="POST"
+            action="{% url 'settings-user-exports-disable' %}"
+        >
+            <div class="notification">
+                {% trans "This is only intended to be used when things have gone very wrong with exports and you need to pause the feature while addressing issues." %}
+                {% trans "While exports are disabled, users will not be allowed to start new user exports, but existing exports will not be affected." %}
+            </div>
+            {% csrf_token %}
+            <div class="control">
+                <button type="submit" class="button is-danger">
+                    {% trans "Disable user exports" %}
+                </button>
+            </div>
+        </form>
+    </details>
+    <details class="details-panel box">
+        <summary>
+            <span role="heading" aria-level="2" class="title is-6">
+                {% trans "Limit how often users can import and export" %}
+            </span>
+            <span class="details-close icon icon-x" aria-hidden="true"></span>
+        </summary>
+        <form
+            name="user-imports-set-limit"
+            id="user-imports-set-limit"
+            method="POST"
+            action="{% url 'settings-user-imports-set-limit' %}"
+        >
+            <div class="notification">
+                {% trans "Some users might try to run user imports or exports very frequently, which you want to limit." %}
+                {% trans "Set the value to 0 to not enforce any limit." %}
+            </div>
+            <div class="align.to-t">
+                <label for="limit">{% trans "Limit how often users can import and export user data" %}</label>
+                <input name="limit" class="input is-w-xs is-h-em" type="text" placeholder="0" value="{{ user_import_time_limit }}">
+                <label>{% trans "hours" %}</label>
+                {% csrf_token %}
+                <div class="control">
+                    <button type="submit" class="button is-warning">
+                        {% trans "Change limit" %}
+                    </button>
+                </div>
+            </div>
+        </form>
+    </details>
+    {% else %}
+    <form
+        name="enable-user-imports"
+        id="enable-user-imports"
+        method="POST"
+        action="{% url 'settings-user-exports-enable' %}"
+        class="box"
+    >
+        <div class="notification is-danger is-light">
+            <p class="my-2">{% trans "Users are currently unable to start new user exports. This is the default setting." %}</p>
+            {% if use_s3 %}
+            <p>{% trans "It is not currently possible to provide user exports when using s3 storage. The BookWyrm development team are working on a fix for this." %}</p>
+            {% endif %}
+        </div>
+        {% csrf_token %}
+        <div class="control">
+            <button type="submit" class="button is-success" {% if use_s3 %}disabled{% endif %}>
+                {% trans "Enable user exports" %}
+            </button>
+        </div>
+    </form>
+    {% endif %}
 </div>
 <div class="block">
+    <h4 class="title is-4">{% trans "Book Imports" %}</h4>
+    <div class="block">
         <div class="tabs">
             <ul>
                 {% url 'settings-imports' as url %}
@@ -103,9 +185,9 @@
                 </li>
             </ul>
         </div>
-</div>
+    </div>
 
-<div class="table-container block content">
+    <div class="table-container block content">
         <table class="table is-striped is-fullwidth">
             <tr>
                 {% url 'settings-imports' status as url %}
@@ -172,8 +254,100 @@
             </tr>
             {% endif %}
         </table>
+    </div>
+
+    {% include 'snippets/pagination.html' with page=imports path=request.path %}
+
 </div>
 
-{% include 'snippets/pagination.html' with page=imports path=request.path %}
+<div class="block">
-{% endblock %}
+    <h4 class="title is-4">{% trans "User Imports" %}</h4>
+    <div class="block">
+        <div class="tabs">
+            <ul>
+                {% url 'settings-imports' as url %}
+                <li {% if request.path in url %}class="is-active" aria-current="page"{% endif %}>
+                    <a href="{{ url }}">{% trans "Active" %}</a>
+                </li>
+                {% url 'settings-imports' status="complete" as url %}
+                <li {% if url in request.path %}class="is-active" aria-current="page"{% endif %}>
+                    <a href="{{ url }}">{% trans "Completed" %}</a>
+                </li>
+            </ul>
+        </div>
+    </div>
 
+    <div class="table-container block content">
+        <table class="table is-striped is-fullwidth">
+            <tr>
+                {% url 'settings-imports' status as url %}
+                <th>
+                    {% trans "ID" %}
+                </th>
+                <th>
+                    {% trans "User" as text %}
+                    {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %}
+                </th>
+                <th>
+                    {% trans "Date Created" as text %}
+                    {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %}
+                </th>
+                {% if status != "active" %}
+                <th>
+                    {% trans "Date Updated" %}
+                </th>
+                {% endif %}
+
+                {% if status == "active" %}
+                <th>{% trans "Actions" %}</th>
+                {% else %}
+                <th>{% trans "Status" %}</th>
+                {% endif %}
+            </tr>
+            {% for import in user_imports %}
+            <tr>
+                <td>{{ import.id }}</td>
+                <td class="overflow-wrap-anywhere">
+                    <a href="{% url 'settings-user' import.user.id %}">{{ import.user|username }}</a>
+                </td>
+                <td>{{ import.created_date }}</td>
+                {% if status != "active" %}
+                <td>{{ import.updated_date }}</td>
+                {% endif %}
+                {% if status == "active" %}
+                <td>
+                    {% join "complete" import.id as modal_id %}
+                    <button type="button" data-modal-open="{{ modal_id }}" class="button is-danger">{% trans "Stop import" %}</button>
+                    {% include "settings/imports/complete_user_import_modal.html" with id=modal_id %}
+                </td>
+                {% else %}
+                <td>
+                    <span
+                        {% if import.status == "stopped" or import.status == "failed" %}
+                        class="tag is-danger"
+                        {% elif import.status == "pending" %}
+                        class="tag is-warning"
+                        {% elif import.complete %}
+                        class="tag"
+                        {% else %}
+                        class="tag is-success"
+                        {% endif %}
+                        >{{ import.status }}
+                    </span>
+                </td>
+                {% endif %}
+            </tr>
+            {% endfor %}
+            {% if not user_imports %}
+            <tr>
+                <td colspan="6">
+                    <em>{% trans "No matching imports found." %} </em>
+                </td>
+            </tr>
+            {% endif %}
+        </table>
+    </div>
+
+    {% include 'snippets/pagination.html' with page=user_imports path=request.path %}
+</div>
+{% endblock %}

bookwyrm/templates/settings/layout.html

@@ -85,6 +85,10 @@
                 {% url 'settings-celery' as url %}
                 <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Celery status" %}</a>
             </li>
+            <li>
+                {% url 'settings-schedules' as url %}
+                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Scheduled tasks" %}</a>
+            </li>
             <li>
                 {% url 'settings-email-config' as url %}
                 <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Email Configuration" %}</a>

bookwyrm/templates/settings/schedules.html

@@ -0,0 +1,127 @@
+{% extends 'settings/layout.html' %}
+{% load i18n %}
+{% load humanize %}
+{% load utilities %}
+
+{% block title %}
+{% trans "Scheduled tasks" %}
+{% endblock %}
+
+{% block header %}
+{% trans "Scheduled tasks" %}
+{% endblock %}
+
+{% block panel %}
+
+<div class="block content">
+    <h3>{% trans "Tasks" %}</h3>
+    <div class="table-container">
+        <table class="table is-striped is-fullwidth">
+            <tr>
+                <th>
+                    {% trans "Name" %}
+                </th>
+                <th>
+                    {% trans "Celery task" %}
+                </th>
+                <th>
+                    {% trans "Date changed" %}
+                </th>
+                <th>
+                    {% trans "Last run at" %}
+                </th>
+                <th>
+                    {% trans "Schedule" %}
+                </th>
+                <th>
+                    {% trans "Schedule ID" %}
+                </th>
+                <th>
+                    {% trans "Enabled" %}
+                </th>
+            </tr>
+            {% for task in tasks %}
+            <tr>
+                <td>
+                    {{ task.name }}
+                </td>
+                <td class="overflow-wrap-anywhere">
+                    {{ task.task }}
+                </td>
+                <td>
+                    {{ task.date_changed }}
+                </td>
+                <td>
+                    {{ task.last_run_at }}
+                </td>
+                <td>
+                    {% firstof task.interval task.crontab "None" %}
+                </td>
+                <td>
+                    {{ task.interval.id }}
+                </td>
+                <td>
+                    <span class="tag">
+                        {% if task.enabled %}
+                        <span class="icon icon-check" aria-hidden="true"></span>
+                        {% endif %}
+                    {{ task.enabled|yesno }}
+                    </span>
+                    {% if task.name != "celery.backend_cleanup" %}
+                    <form name="unschedule-{{ task.id }}" method="POST" action="{% url 'settings-schedules' task.id %}">
+                        {% csrf_token %}
+                        <button type="submit" class="button is-danger is-small">{% trans "Un-schedule" %}</button>
+                    </form>
+                    {% endif %}
+                </td>
+            </tr>
+            {% empty %}
+            <tr>
+                <td colspan="2">
+                    {% trans "No scheduled tasks" %}
+                </td>
+            </tr>
+            {% endfor %}
+        </table>
+    </div>
+</div>
+
+<div class="block content">
+    <h3>{% trans "Schedules" %}</h3>
+    <div class="table-container">
+        <table class="table is-striped is-fullwidth">
+            <tr>
+                <th>
+                    {% trans "ID" %}
+                </th>
+                <th>
+                    {% trans "Schedule" %}
+                </th>
+                <th>
+                    {% trans "Tasks" %}
+                </th>
+            </tr>
+            {% for schedule in schedules %}
+            <tr>
+                <td>
+                    {{ schedule.id }}
+                </td>
+                <td class="overflow-wrap-anywhere">
+                    {{ schedule }}
+                </td>
+                <td>
+                    {{ schedule.periodictask_set.count }}
+                </td>
+            </tr>
+            {% empty %}
+            <tr>
+                <td colspan="2">
+                    {% trans "No schedules found" %}
+                </td>
+            </tr>
+            {% endfor %}
+        </table>
+    </div>
+</div>
+
+{% endblock %}

bookwyrm/templates/shelf/shelf.html

@@ -19,20 +19,7 @@
     </h1>
 </header>
 {% if user.moved_to %}
-<div class="container my-6">
+    {% include "snippets/moved_user_notice.html" with user=user %}
-    <div class="notification is-info has-text-centered">
-        <p>
-            {% trans "You have have moved to" %}
-            <a href="{{user.moved_to}}">{% id_to_username user.moved_to %}</a>
-        </p>
-        <p> {% trans "You can undo this move to restore full functionality, but some followers may have already unfollowed this account." %}</p>
-        <form name="remove-alias" action="{% url 'prefs-unmove' %}" method="post">
-            {% csrf_token %}
-            <input type="hidden" name="remote_id" id="remote_id" value="{{user.moved_to}}">
-            <button type="submit" class="button is-small">{% trans "Undo move" %}</button>
-        </form>
-    </div>
-</div>
 {% else %}
 <nav class="breadcrumb subtitle" aria-label="breadcrumbs">
     <ul>
@@ -114,7 +101,6 @@
                     {% plural %}
                         {{ formatted_count }} books
                     {% endblocktrans %}
-
                     {% if books.has_other_pages %}
                         {% blocktrans trimmed with start=books.start_index end=books.end_index %}
                         (showing {{ start }}-{{ end }})
@@ -124,6 +110,8 @@
                 {% endif %}
                 {% endwith %}
             </h2>
+            {% include 'shelf/shelves_filters.html' with user=user query=query %}
+
         </div>
         {% if is_self and shelf.id %}
         <div class="column is-narrow">
@@ -222,7 +210,17 @@
             </tbody>
         </table>
         {% else %}
-        <p><em>{% trans "This shelf is empty." %}</em></p>
+        <p>
+            <em>
+                {% if shelves_filter_query %}
+                    {% blocktrans trimmed %}
+                        We couldn't find any books that matched {{ shelves_filter_query }}
+                    {% endblocktrans %}
+                {% else %}
+                    {% trans "This shelf is empty." %}
+                {% endif %}
+            </em>
+        </p>
         {% endif %}
 
     </div>

bookwyrm/templates/shelf/shelves_filter_field.html

@@ -0,0 +1,9 @@
+{% extends 'snippets/filters_panel/filter_field.html' %}
+{% load i18n %}
+
+{% block filter %}
+  <div class="control">
+    <label class="label" for="my-books-filter">{% trans 'Filter by keyword' %}</label>
+    <input id="my-books-filter" class="input" type="text" name="filter" placeholder="{% trans 'Enter text here' %}" value="{{ shelves_filter_query|default:'' }}" spellcheck="false" />
+  </div>
+{% endblock %}

bookwyrm/templates/shelf/shelves_filters.html

@@ -0,0 +1,5 @@
+{% extends 'snippets/filters_panel/filters_panel.html' %}
+
+{% block filter_fields %}
+  {% include 'shelf/shelves_filter_field.html' %}
+{% endblock %}

bookwyrm/templates/snippets/create_status/quotation.html

@@ -56,8 +56,7 @@ uuid: a unique identifier used to make html "id" attributes unique and clarify j
                 <input
                     aria-label="{% if draft.position_mode == 'PG' %}Page{% else %}Percent{% endif %}"
                     class="input"
-                    type="number"
+                    type="text"
-                    min="0"
                     name="position"
                     size="3"
                     value="{% firstof draft.position '' %}"
@@ -72,8 +71,7 @@ uuid: a unique identifier used to make html "id" attributes unique and clarify j
                 <input
                     aria-label="{% if draft.position_mode == 'PG' %}Page{% else %}Percent{% endif %}"
                     class="input"
-                    type="number"
+                    type="text"
-                    min="0"
                     name="endposition"
                     size="3"
                     value="{% firstof draft.endposition '' %}"

bookwyrm/templates/snippets/follow_button.html

@@ -43,7 +43,7 @@
     </div>
     {% if not minimal %}
     <div class="control">
-        {% include 'snippets/user_options.html' with user=user class="is-small" %}
+        {% include 'snippets/user_options.html' with user=user followers_page=followers_page class="is-small" %}
     </div>
     {% endif %}
 </div>

bookwyrm/templates/snippets/moved_user_notice.html

@@ -0,0 +1,12 @@
+{% load i18n %}
+{% load utilities %}
+<div class="container my-6">
+    <div class="notification is-info has-text-centered">
+        <p>
+            {% id_to_username user.moved_to as moved_to_name %}
+            {% blocktrans trimmed with user=user|username moved_to_link=user.moved_to %}
+            <em>{{ user }}</em> has moved to <a href="{{ moved_to_link }}">{{ moved_to_name }}</a>
+            {% endblocktrans %}
+        </p>
+    </div>
+</div>

bookwyrm/templates/snippets/opengraph.html

@@ -1,24 +1,25 @@
 {% load static %}
 
-{% if preview_images_enabled is True %}
+{% firstof image site.preview_image as page_image %}
+{% if page_image %}
     <meta name="twitter:card" content="summary_large_image">
-    {% if image %}
+    <meta name="twitter:image" content="{{ media_full_url }}{{ page_image }}">
-        <meta name="twitter:image" content="{{ media_full_url }}{{ image }}">
+    <meta name="og:image" content="{{ media_full_url }}{{ page_image }}">
-        <meta name="og:image" content="{{ media_full_url }}{{ image }}">
+{% elif site.logo %}
-    {% else %}
+    <meta name="twitter:card" content="summary">
-        <meta name="twitter:image" content="{{ media_full_url }}{{ site.preview_image }}">
+    <meta name="twitter:image" content="{{ media_full_url }}{{ site.logo }}">
-        <meta name="og:image" content="{{ media_full_url }}{{ site.preview_image }}">
+    <meta name="twitter:image:alt" content="{{ site.name }} Logo">
-    {% endif %}
+    <meta name="og:image" content="{{ media_full_url }}{{ site.logo }}">
 {% else %}
     <meta name="twitter:card" content="summary">
-    <meta name="twitter:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
+    <meta name="twitter:image" content="{% static "images/logo.png" %}">
-    <meta name="og:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
+    <meta name="twitter:image:alt" content="BookWyrm Logo">
+    <meta name="og:image" content="{% static "images/logo.png" %}">
 {% endif %}
 
-<meta name="twitter:image:alt" content="BookWyrm Logo">
-
 <meta name="twitter:title" content="{% if title %}{{ title }} - {% endif %}{{ site.name }}">
 <meta name="og:title" content="{% if title %}{{ title }} - {% endif %}{{ site.name }}">
 
-<meta name="twitter:description" content="{% if description %}{{ description }}{% else %}{{ site.instance_tagline }}{% endif %}">
+{% firstof description site.instance_tagline as description %}
-<meta name="og:description" content="{% if description %}{{ description }}{% else %}{{ site.instance_tagline }}{% endif %}">
+<meta name="twitter:description" content="{{ description }}">
+<meta name="og:description" content="{{ description }}">

bookwyrm/templates/snippets/remove_follower_button.html

@@ -0,0 +1,5 @@
+{% load i18n %}
+<form name="remove" method="post" action="/remove-follow/{{ user.id }}">
+    {% csrf_token %}
+    <button class="button is-danger is-light is-small {{ class }}" type="submit">{% trans "Remove" %}</button>
+</form>

bookwyrm/templates/snippets/user_options.html

@@ -20,4 +20,9 @@
 <li role="menuitem">
     {% include 'snippets/block_button.html' with user=user class="is-fullwidth" blocks=False %}
 </li>
+{% if followers_page %}
+<li role="menuitem">
+    {% include 'snippets/remove_follower_button.html' with user=user class="is-fullwidth" %}
+</li>
+{% endif %}
 {% endblock %}

bookwyrm/templates/user/layout.html

@@ -45,12 +45,7 @@
     </div>
     <div>
     {% if user.moved_to %}
-        <div class="container my-6">
+        {% include "snippets/moved_user_notice.html" with user=user %}
-            <div class="notification is-info has-text-centered">
-                <p><em>{{ user.localname }}</em> {% trans "has moved to" %} <a href="{{user.moved_to}}">{% id_to_username user.moved_to %}</a></p>
-            </div>
-        </div>
-
     {% else %}
         {% if not is_self and request.user.is_authenticated %}
         {% include 'snippets/follow_button.html' with user=user %}

bookwyrm/templates/user/relationships/followers.html

@@ -25,6 +25,11 @@
 </nav>
 {% endblock %}
 
+{% block panel %}
+    {% with followers_page=True %}
+    {{ block.super }}
+    {% endwith %}
+{% endblock %}
 
 {% block nullstate %}
 <div>

bookwyrm/templates/user/relationships/layout.html

@@ -31,7 +31,7 @@
             ({{ follow.username }})
         </div>
         <div class="column is-narrow">
-            {% include 'snippets/follow_button.html' with user=follow %}
+            {% include 'snippets/follow_button.html' with user=follow followers_page=followers_page %}
         </div>
     </div>
     {% endfor %}

bookwyrm/templatetags/date_ext.py

@@ -15,16 +15,10 @@ def naturalday_partial(date, arg=None):
     If arg is a Django-defined format such as "DATE_FORMAT", it will be adjusted
     so that the precision of the PartialDate object is honored.
     """
-    django_formats = ("DATE_FORMAT", "SHORT_DATE_FORMAT", "YEAR_MONTH_FORMAT")
+    if not isinstance(date, PartialDate) or date.has_day:
-    if not isinstance(date, PartialDate):
+        return naturalday(date, arg)
+    if not arg or arg == "DATE_FORMAT":
+        arg = "YEAR_MONTH_FORMAT" if date.has_month else "Y"
+    elif not date.has_month and arg in ("SHORT_DATE_FORMAT", "YEAR_MONTH_FORMAT"):
+        arg = "Y"
     return defaultfilters.date(date, arg)
-    if arg is None:
-        arg = "DATE_FORMAT"
-    if date.has_day:
-        fmt = arg
-    elif date.has_month:
-        # there is no SHORT_YEAR_MONTH_FORMAT, so we ignore SHORT_DATE_FORMAT :(
-        fmt = "YEAR_MONTH_FORMAT" if arg == "DATE_FORMAT" else arg
-    else:
-        fmt = "Y" if arg in django_formats else arg
-    return naturalday(date, fmt)

bookwyrm/templatetags/utilities.py

@@ -126,6 +126,24 @@ def id_to_username(user_id):
         value = f"{name}@{domain}"
 
         return value
+    return _("a new user account")
+
+
+@register.filter(name="get_file_size")
+def get_file_size(file):
+    """display the size of a file in human readable terms"""
+
+    try:
+        raw_size = os.stat(file.path).st_size
+        if raw_size < 1024:
+            return f"{raw_size} bytes"
+        if raw_size < 1024**2:
+            return f"{raw_size/1024:.2f} KB"
+        if raw_size < 1024**3:
+            return f"{raw_size/1024**2:.2f} MB"
+        return f"{raw_size/1024**3:.2f} GB"
+    except Exception:  # pylint: disable=broad-except
+        return ""
 
 
 @register.filter(name="get_user_permission")

bookwyrm/tests/activitypub/test_author.py

@@ -6,7 +6,8 @@ from bookwyrm import models
 class Author(TestCase):
     """serialize author tests"""
 
-    def setUp(self):
+    @classmethod
+    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
         """initial data"""
         self.book = models.Edition.objects.create(
             title="Example Edition",

bookwyrm/tests/activitypub/test_base_activity.py

@@ -28,8 +28,8 @@ from bookwyrm import models
 class BaseActivity(TestCase):
     """the super class for model-linked activitypub dataclasses"""
 
-    # pylint: disable=invalid-name
+    @classmethod
-    def setUp(self):
+    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
         """we're probably going to re-use this so why copy/paste"""
         with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
             "bookwyrm.activitystreams.populate_stream_task.delay"
@@ -40,6 +40,7 @@ class BaseActivity(TestCase):
         self.user.remote_id = "http://example.com/a/b"
         self.user.save(broadcast=False, update_fields=["remote_id"])
 
+    def setUp(self):
         datafile = pathlib.Path(__file__).parent.joinpath("../data/ap_user.json")
         self.userdata = json.loads(datafile.read_bytes())
         # don't try to load the user icon

bookwyrm/tests/activitypub/test_note.py

@@ -10,8 +10,8 @@ from bookwyrm import models
 class Note(TestCase):
     """the model-linked ActivityPub dataclass for Note-based types"""
 
-    # pylint: disable=invalid-name
+    @classmethod
-    def setUp(self):
+    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
         """create a shared user"""
         with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
             "bookwyrm.activitystreams.populate_stream_task.delay"

bookwyrm/tests/activitypub/test_quotation.py

@@ -10,7 +10,8 @@ from bookwyrm import activitypub, models
 class Quotation(TestCase):
     """we have hecka ways to create statuses"""
 
-    def setUp(self):
+    @classmethod
+    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
         """model objects we'll need"""
         with patch("bookwyrm.models.user.set_remote_server.delay"):
             self.user = models.User.objects.create_user(
@@ -26,6 +27,9 @@ class Quotation(TestCase):
             title="Example Edition",
             remote_id="https://example.com/book/1",
         )
+
+    def setUp(self):
+        """other test data"""
         datafile = pathlib.Path(__file__).parent.joinpath("../data/ap_quotation.json")
         self.status_data = json.loads(datafile.read_bytes())
 

bookwyrm/tests/activitystreams/test_abstractstream.py

@@ -15,7 +15,8 @@ from bookwyrm import activitystreams, models
 class Activitystreams(TestCase):
     """using redis to build activity streams"""
 
-    def setUp(self):
+    @classmethod
+    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
         """use a test csv"""
         with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
             "bookwyrm.activitystreams.populate_stream_task.delay"
@@ -43,6 +44,9 @@ class Activitystreams(TestCase):
         work = models.Work.objects.create(title="test work")
         self.book = models.Edition.objects.create(title="test book", parent_work=work)
 
+    def setUp(self):
+        """per-test setUp"""
+
         class TestStream(activitystreams.ActivityStream):
             """test stream, don't have to do anything here"""