Merge pull request #3114 from SMillerDev/feat/api/oauth

feat: add OAuth authentication

.env.example

@@ -16,6 +16,11 @@ DEFAULT_LANGUAGE="English"
 ## Leave unset to allow all hosts
 # ALLOWED_HOSTS="localhost,127.0.0.1,[::1]"
 
+# Specify when the site is served from a port that is not the default
+# for the protocol (80 for HTTP or 443 for HTTPS).
+# Probably only necessary in development.
+# PORT=1333
+
 MEDIA_ROOT=images/
 
 # Database configuration
@@ -71,14 +76,20 @@ ENABLE_THUMBNAIL_GENERATION=true
 USE_S3=false
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
+# seconds for signed S3 urls to expire
+# this is currently only used for user export files
+S3_SIGNED_URL_EXPIRY=900
 
 # Commented are example values if you use a non-AWS, S3-compatible service
 # AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
 # non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
-# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL.
+# AWS_S3_URL_PROTOCOL must end in ":" and defaults to the same protocol as
+# the BookWyrm instance ("http:" or "https:", based on USE_SSL).
 
 # AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
 # AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_URL_PROTOCOL=None        # "http:"
 # AWS_S3_REGION_NAME=None         # "fr-par"
 # AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
 
@@ -133,9 +144,9 @@ HTTP_X_FORWARDED_PROTO=false
 TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2
 TWO_FACTOR_LOGIN_MAX_SECONDS=60
 
-# Additional hosts to allow in the Content-Security-Policy, "self" (should be DOMAIN)
-# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
-# Value should be a comma-separated list of host names.
+# Additional hosts to allow in the Content-Security-Policy, "self" (should be
+# DOMAIN with optionally ":" + PORT) and AWS_S3_CUSTOM_DOMAIN (if used) are
+# added by default.  Value should be a comma-separated list of host names.
 CSP_ADDITIONAL_HOSTS=
 
 # Time before being logged out (in seconds)

.github/workflows/codeql-analysis.yml

@@ -40,7 +40,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,7 +51,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -65,4 +65,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/lint-frontend.yaml

@@ -22,7 +22,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install modules
-        run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
+      #  run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
+        run: npm install eslint@^8.9.0
 
       # See .stylelintignore for files that are not linted.
       # - name: Run stylelint

.gitignore

@@ -16,6 +16,7 @@
 # BookWyrm
 .env
 /images/
+/exports/
 /static/
 bookwyrm/static/css/bookwyrm.css
 bookwyrm/static/css/themes/
@@ -37,3 +38,6 @@ nginx/default.conf
 
 #macOS
 **/.DS_Store
+
+# Docker
+docker-compose.override.yml

README.md

@@ -10,7 +10,6 @@ BookWyrm is a social network for tracking your reading, talking about books, wri
 ## Links
 
 [![Mastodon Follow](https://img.shields.io/mastodon/follow/000146121?domain=https%3A%2F%2Ftech.lgbt&style=social)](https://tech.lgbt/@bookwyrm)
-[![Twitter Follow](https://img.shields.io/twitter/follow/BookWyrmSocial?style=social)](https://twitter.com/BookWyrmSocial)
 
  - [Project homepage](https://joinbookwyrm.com/)
  - [Support](https://patreon.com/bookwyrm)

bookwyrm/activitystreams.py

@@ -139,14 +139,14 @@ class ActivityStream(RedisStore):
                 | (
                     Q(following=status.user) & Q(following=status.reply_parent.user)
                 )  # if the user is following both authors
-            ).distinct()
+            )
 
         # only visible to the poster's followers and tagged users
         elif status.privacy == "followers":
             audience = audience.filter(
                 Q(following=status.user)  # if the user is following the author
             )
-        return audience.distinct()
+        return audience.distinct("id")
 
     @tracer.start_as_current_span("ActivityStream.get_audience")
     def get_audience(self, status):
@@ -156,7 +156,7 @@ class ActivityStream(RedisStore):
         status_author = models.User.objects.filter(
             is_active=True, local=True, id=status.user.id
         ).values_list("id", flat=True)
-        return list(set(list(audience) + list(status_author)))
+        return list(set(audience) | set(status_author))
 
     def get_stores_for_users(self, user_ids):
         """convert a list of user ids into redis store ids"""
@@ -183,15 +183,13 @@ class HomeStream(ActivityStream):
     def get_audience(self, status):
         trace.get_current_span().set_attribute("stream_id", self.key)
         audience = super()._get_audience(status)
-        if not audience:
-            return []
         # if the user is following the author
         audience = audience.filter(following=status.user).values_list("id", flat=True)
         # if the user is the post's author
         status_author = models.User.objects.filter(
             is_active=True, local=True, id=status.user.id
         ).values_list("id", flat=True)
-        return list(set(list(audience) + list(status_author)))
+        return list(set(audience) | set(status_author))
 
     def get_statuses_for_user(self, user):
         return models.Status.privacy_filter(
@@ -239,9 +237,7 @@ class BooksStream(ActivityStream):
         )
 
         audience = super()._get_audience(status)
-        if not audience:
-            return models.User.objects.none()
-        return audience.filter(shelfbook__book__parent_work=work).distinct()
+        return audience.filter(shelfbook__book__parent_work=work)
 
     def get_audience(self, status):
         # only show public statuses on the books feed,

bookwyrm/connectors/connector_manager.py

@@ -118,9 +118,11 @@ def get_connectors() -> Iterator[abstract_connector.AbstractConnector]:
 def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnector:
     """get the connector related to the object's server"""
     url = urlparse(remote_id)
-    identifier = url.netloc
+    identifier = url.hostname
     if not identifier:
-        raise ValueError("Invalid remote id")
+        raise ValueError(f"Invalid remote id: {remote_id}")
+
+    base_url = f"{url.scheme}://{url.netloc}"
 
     try:
         connector_info = models.Connector.objects.get(identifier=identifier)
@@ -128,10 +130,10 @@ def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnec
         connector_info = models.Connector.objects.create(
             identifier=identifier,
             connector_file="bookwyrm_connector",
-            base_url=f"https://{identifier}",
-            books_url=f"https://{identifier}/book",
-            covers_url=f"https://{identifier}/images/covers",
-            search_url=f"https://{identifier}/search?q=",
+            base_url=base_url,
+            books_url=f"{base_url}/book",
+            covers_url=f"{base_url}/images/covers",
+            search_url=f"{base_url}/search?q=",
             priority=2,
         )
 
@@ -188,8 +190,11 @@ def raise_not_valid_url(url: str) -> None:
     if not parsed.scheme in ["http", "https"]:
         raise ConnectorException("Invalid scheme: ", url)
 
+    if not parsed.hostname:
+        raise ConnectorException("Hostname missing: ", url)
+
     try:
-        ipaddress.ip_address(parsed.netloc)
+        ipaddress.ip_address(parsed.hostname)
         raise ConnectorException("Provided url is an IP address: ", url)
     except ValueError:
         # it's not an IP address, which is good

bookwyrm/emailing.py

@@ -4,7 +4,7 @@ from django.template.loader import get_template
 
 from bookwyrm import models, settings
 from bookwyrm.tasks import app, EMAIL
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import DOMAIN, BASE_URL
 
 
 def email_data():
@@ -14,6 +14,7 @@ def email_data():
         "site_name": site.name,
         "logo": site.logo_small_url,
         "domain": DOMAIN,
+        "base_url": BASE_URL,
         "user": None,
     }
 

bookwyrm/forms/links.py

@@ -26,7 +26,7 @@ class FileLinkForm(CustomForm):
         url = cleaned_data.get("url")
         filetype = cleaned_data.get("filetype")
         book = cleaned_data.get("book")
-        domain = urlparse(url).netloc
+        domain = urlparse(url).hostname
         if models.LinkDomain.objects.filter(domain=domain).exists():
             status = models.LinkDomain.objects.get(domain=domain).status
             if status == "blocked":

bookwyrm/management/commands/deduplicate_book_data.py

@@ -1,13 +1,14 @@
 """ PROCEED WITH CAUTION: uses deduplication fields to permanently
 merge book data objects """
+
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from bookwyrm import models
-from bookwyrm.management.merge import merge_objects
 
 
-def dedupe_model(model):
+def dedupe_model(model, dry_run=False):
     """combine duplicate editions and update related models"""
+    print(f"deduplicating {model.__name__}:")
     fields = model._meta.get_fields()
     dedupe_fields = [
         f for f in fields if hasattr(f, "deduplication_field") and f.deduplication_field
@@ -16,30 +17,42 @@ def dedupe_model(model):
         dupes = (
             model.objects.values(field.name)
             .annotate(Count(field.name))
-            .filter(**{"%s__count__gt" % field.name: 1})
+            .filter(**{f"{field.name}__count__gt": 1})
+            .exclude(**{field.name: ""})
+            .exclude(**{f"{field.name}__isnull": True})
         )
 
         for dupe in dupes:
             value = dupe[field.name]
-            if not value or value == "":
-                continue
             print("----------")
-            print(dupe)
             objs = model.objects.filter(**{field.name: value}).order_by("id")
             canonical = objs.first()
-            print("keeping", canonical.remote_id)
+            action = "would merge" if dry_run else "merging"
+            print(
+                f"{action} into {model.__name__} {canonical.remote_id} based on {field.name} {value}:"
+            )
             for obj in objs[1:]:
-                print(obj.remote_id)
-                merge_objects(canonical, obj)
+                print(f"- {obj.remote_id}")
+                absorbed_fields = obj.merge_into(canonical, dry_run=dry_run)
+                print(f"  absorbed fields: {absorbed_fields}")
 
 
 class Command(BaseCommand):
     """deduplicate allllll the book data models"""
 
     help = "merges duplicate book data"
+
+    def add_arguments(self, parser):
+        """add the arguments for this command"""
+        parser.add_argument(
+            "--dry_run",
+            action="store_true",
+            help="don't actually merge, only print what would happen",
+        )
+
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
         """run deduplications"""
-        dedupe_model(models.Edition)
-        dedupe_model(models.Work)
-        dedupe_model(models.Author)
+        dedupe_model(models.Edition, dry_run=options["dry_run"])
+        dedupe_model(models.Work, dry_run=options["dry_run"])
+        dedupe_model(models.Author, dry_run=options["dry_run"])

bookwyrm/management/merge.py

@@ -1,50 +0,0 @@
-from django.db.models import ManyToManyField
-
-
-def update_related(canonical, obj):
-    """update all the models with fk to the object being removed"""
-    # move related models to canonical
-    related_models = [
-        (r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
-    ]
-    for (related_field, related_model) in related_models:
-        # Skip the ManyToMany fields that aren’t auto-created. These
-        # should have a corresponding OneToMany field in the model for
-        # the linking table anyway. If we update it through that model
-        # instead then we won’t lose the extra fields in the linking
-        # table.
-        related_field_obj = related_model._meta.get_field(related_field)
-        if isinstance(related_field_obj, ManyToManyField):
-            through = related_field_obj.remote_field.through
-            if not through._meta.auto_created:
-                continue
-        related_objs = related_model.objects.filter(**{related_field: obj})
-        for related_obj in related_objs:
-            print("replacing in", related_model.__name__, related_field, related_obj.id)
-            try:
-                setattr(related_obj, related_field, canonical)
-                related_obj.save()
-            except TypeError:
-                getattr(related_obj, related_field).add(canonical)
-                getattr(related_obj, related_field).remove(obj)
-
-
-def copy_data(canonical, obj):
-    """try to get the most data possible"""
-    for data_field in obj._meta.get_fields():
-        if not hasattr(data_field, "activitypub_field"):
-            continue
-        data_value = getattr(obj, data_field.name)
-        if not data_value:
-            continue
-        if not getattr(canonical, data_field.name):
-            print("setting data field", data_field.name, data_value)
-            setattr(canonical, data_field.name, data_value)
-    canonical.save()
-
-
-def merge_objects(canonical, obj):
-    copy_data(canonical, obj)
-    update_related(canonical, obj)
-    # remove the outdated entry
-    obj.delete()

bookwyrm/management/merge_command.py

@@ -1,4 +1,3 @@
-from bookwyrm.management.merge import merge_objects
 from django.core.management.base import BaseCommand
 
 
@@ -9,6 +8,11 @@ class MergeCommand(BaseCommand):
         """add the arguments for this command"""
         parser.add_argument("--canonical", type=int, required=True)
         parser.add_argument("--other", type=int, required=True)
+        parser.add_argument(
+            "--dry_run",
+            action="store_true",
+            help="don't actually merge, only print what would happen",
+        )
 
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
@@ -26,4 +30,8 @@ class MergeCommand(BaseCommand):
             print("other book doesn’t exist!")
             return
 
-        merge_objects(canonical, other)
+        absorbed_fields = other.merge_into(canonical, dry_run=options["dry_run"])
+
+        action = "would be" if options["dry_run"] else "has been"
+        print(f"{other.remote_id} {action} merged into {canonical.remote_id}")
+        print(f"absorbed fields: {absorbed_fields}")

bookwyrm/migrations/0193_auto_20240128_0249.py

@@ -0,0 +1,92 @@
+# Generated by Django 3.2.23 on 2024-01-28 02:49
+
+import bookwyrm.storage_backends
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0192_sitesettings_user_exports_enabled"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="bookwyrmexportjob",
+            name="export_json",
+            field=models.JSONField(
+                encoder=django.core.serializers.json.DjangoJSONEncoder, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="bookwyrmexportjob",
+            name="json_completed",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name="bookwyrmexportjob",
+            name="export_data",
+            field=models.FileField(
+                null=True,
+                storage=bookwyrm.storage_backends.ExportsFileStorage,
+                upload_to="",
+            ),
+        ),
+        migrations.CreateModel(
+            name="AddFileToTar",
+            fields=[
+                (
+                    "childjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.childjob",
+                    ),
+                ),
+                (
+                    "parent_export_job",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="child_edition_export_jobs",
+                        to="bookwyrm.bookwyrmexportjob",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.childjob",),
+        ),
+        migrations.CreateModel(
+            name="AddBookToUserExportJob",
+            fields=[
+                (
+                    "childjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.childjob",
+                    ),
+                ),
+                (
+                    "edition",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="bookwyrm.edition",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.childjob",),
+        ),
+    ]

bookwyrm/migrations/0196_merge_20240318_1737.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-03-18 17:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0193_auto_20240128_0249"),
+        ("bookwyrm", "0195_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0197_merge_20240324_0235.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-03-24 02:35
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0196_merge_20240318_1737"),
+        ("bookwyrm", "0196_merge_pr3134_into_main"),
+    ]
+
+    operations = []

bookwyrm/migrations/0197_mergedauthor_mergedbook.py

@@ -0,0 +1,48 @@
+# Generated by Django 3.2.24 on 2024-02-28 21:30
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0196_merge_pr3134_into_main"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="MergedBook",
+            fields=[
+                ("deleted_id", models.IntegerField(primary_key=True, serialize=False)),
+                (
+                    "merged_into",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="absorbed",
+                        to="bookwyrm.book",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="MergedAuthor",
+            fields=[
+                ("deleted_id", models.IntegerField(primary_key=True, serialize=False)),
+                (
+                    "merged_into",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="absorbed",
+                        to="bookwyrm.author",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

bookwyrm/migrations/0198_alter_bookwyrmexportjob_export_data.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.25 on 2024-03-26 11:37
+
+import bookwyrm.models.bookwyrm_export_job
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0197_merge_20240324_0235"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="bookwyrmexportjob",
+            name="export_data",
+            field=models.FileField(
+                null=True,
+                storage=bookwyrm.models.bookwyrm_export_job.select_exports_storage,
+                upload_to="",
+            ),
+        ),
+    ]

bookwyrm/migrations/0199_merge_20240326_1217.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-03-26 12:17
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0198_alter_bookwyrmexportjob_export_data"),
+        ("bookwyrm", "0198_book_search_vector_author_aliases"),
+    ]
+
+    operations = []

bookwyrm/migrations/0199_status_bookwyrm_st_remote__06aeba_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-02 19:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0198_book_search_vector_author_aliases"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="status",
+            index=models.Index(
+                fields=["remote_id"], name="bookwyrm_st_remote__06aeba_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0200_auto_20240327_1914.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.25 on 2024-03-27 19:14
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0199_merge_20240326_1217"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="addfiletotar",
+            name="childjob_ptr",
+        ),
+        migrations.RemoveField(
+            model_name="addfiletotar",
+            name="parent_export_job",
+        ),
+        migrations.DeleteModel(
+            name="AddBookToUserExportJob",
+        ),
+        migrations.DeleteModel(
+            name="AddFileToTar",
+        ),
+    ]

bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0199_status_bookwyrm_st_remote__06aeba_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="status",
+            index=models.Index(
+                fields=["thread_id"], name="bookwyrm_st_thread__cf064f_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0200_status_bookwyrm_st_thread__cf064f_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="keypair",
+            index=models.Index(
+                fields=["remote_id"], name="bookwyrm_ke_remote__472927_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0201_keypair_bookwyrm_ke_remote__472927_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="user",
+            index=models.Index(
+                fields=["username"], name="bookwyrm_us_usernam_b2546d_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0202_user_bookwyrm_us_usernam_b2546d_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="user",
+            index=models.Index(
+                fields=["is_active", "local"], name="bookwyrm_us_is_acti_972dc4_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0204_merge_20240409_1042.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-04-09 10:42
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0197_mergedauthor_mergedbook"),
+        ("bookwyrm", "0203_user_bookwyrm_us_is_acti_972dc4_idx"),
+    ]
+
+    operations = []

bookwyrm/migrations/0205_merge_20240413_0232.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-04-13 02:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0200_auto_20240327_1914"),
+        ("bookwyrm", "0204_merge_20240409_1042"),
+    ]
+
+    operations = []

bookwyrm/models/author.py

@@ -1,4 +1,5 @@
 """ database schema for info about authors """
+
 import re
 from typing import Tuple, Any
 
@@ -7,16 +8,18 @@ from django.contrib.postgres.indexes import GinIndex
 import pgtrigger
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from bookwyrm.utils.db import format_trigger
 
-from .book import BookDataModel
+from .book import BookDataModel, MergedAuthor
 from . import fields
 
 
 class Author(BookDataModel):
     """basic biographic info"""
 
+    merged_model = MergedAuthor
+
     wikipedia_link = fields.CharField(
         max_length=255, blank=True, null=True, deduplication_field=True
     )
@@ -67,7 +70,7 @@ class Author(BookDataModel):
 
     def get_remote_id(self):
         """editions and works both use "book" instead of model_name"""
-        return f"https://{DOMAIN}/author/{self.id}"
+        return f"{BASE_URL}/author/{self.id}"
 
     class Meta:
         """sets up indexes and triggers"""

bookwyrm/models/base_model.py

@@ -10,7 +10,7 @@ from django.http import Http404
 from django.utils.translation import gettext_lazy as _
 from django.utils.text import slugify
 
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from .fields import RemoteIdField
 
 
@@ -38,7 +38,7 @@ class BookWyrmModel(models.Model):
 
     def get_remote_id(self):
         """generate the url that resolves to the local object, without a slug"""
-        base_path = f"https://{DOMAIN}"
+        base_path = BASE_URL
         if hasattr(self, "user"):
             base_path = f"{base_path}{self.user.local_path}"
 
@@ -53,7 +53,7 @@ class BookWyrmModel(models.Model):
     @property
     def local_path(self):
         """how to link to this object in the local app, with a slug"""
-        local = self.get_remote_id().replace(f"https://{DOMAIN}", "")
+        local = self.get_remote_id().replace(BASE_URL, "")
 
         name = None
         if hasattr(self, "name_field"):

bookwyrm/models/book.py

@@ -1,13 +1,15 @@
 """ database schema for books and shelves """
+
 from itertools import chain
 import re
-from typing import Any
+from typing import Any, Dict
+from typing_extensions import Self
 
 from django.contrib.postgres.search import SearchVectorField
 from django.contrib.postgres.indexes import GinIndex
 from django.core.cache import cache
 from django.db import models, transaction
-from django.db.models import Prefetch
+from django.db.models import Prefetch, ManyToManyField
 from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
 from model_utils import FieldTracker
@@ -19,7 +21,7 @@ from bookwyrm import activitypub
 from bookwyrm.isbn.isbn import hyphenator_singleton as hyphenator
 from bookwyrm.preview_images import generate_edition_preview_image_task
 from bookwyrm.settings import (
-    DOMAIN,
+    BASE_URL,
     DEFAULT_LANGUAGE,
     LANGUAGE_ARTICLES,
     ENABLE_PREVIEW_IMAGES,
@@ -108,10 +110,115 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
         """only send book data updates to other bookwyrm instances"""
         super().broadcast(activity, sender, software=software, **kwargs)
 
+    def merge_into(self, canonical: Self, dry_run=False) -> Dict[str, Any]:
+        """merge this entity into another entity"""
+        if canonical.id == self.id:
+            raise ValueError(f"Cannot merge {self} into itself")
+
+        absorbed_fields = canonical.absorb_data_from(self, dry_run=dry_run)
+
+        if dry_run:
+            return absorbed_fields
+
+        canonical.save()
+
+        self.merged_model.objects.create(deleted_id=self.id, merged_into=canonical)
+
+        # move related models to canonical
+        related_models = [
+            (r.remote_field.name, r.related_model) for r in self._meta.related_objects
+        ]
+        # pylint: disable=protected-access
+        for related_field, related_model in related_models:
+            # Skip the ManyToMany fields that aren’t auto-created. These
+            # should have a corresponding OneToMany field in the model for
+            # the linking table anyway. If we update it through that model
+            # instead then we won’t lose the extra fields in the linking
+            # table.
+            # pylint: disable=protected-access
+            related_field_obj = related_model._meta.get_field(related_field)
+            if isinstance(related_field_obj, ManyToManyField):
+                through = related_field_obj.remote_field.through
+                if not through._meta.auto_created:
+                    continue
+            related_objs = related_model.objects.filter(**{related_field: self})
+            for related_obj in related_objs:
+                try:
+                    setattr(related_obj, related_field, canonical)
+                    related_obj.save()
+                except TypeError:
+                    getattr(related_obj, related_field).add(canonical)
+                    getattr(related_obj, related_field).remove(self)
+
+        self.delete()
+        return absorbed_fields
+
+    def absorb_data_from(self, other: Self, dry_run=False) -> Dict[str, Any]:
+        """fill empty fields with values from another entity"""
+        absorbed_fields = {}
+        for data_field in self._meta.get_fields():
+            if not hasattr(data_field, "activitypub_field"):
+                continue
+            canonical_value = getattr(self, data_field.name)
+            other_value = getattr(other, data_field.name)
+            if not other_value:
+                continue
+            if isinstance(data_field, fields.ArrayField):
+                if new_values := list(set(other_value) - set(canonical_value)):
+                    # append at the end (in no particular order)
+                    if not dry_run:
+                        setattr(self, data_field.name, canonical_value + new_values)
+                    absorbed_fields[data_field.name] = new_values
+            elif isinstance(data_field, fields.PartialDateField):
+                if (
+                    (not canonical_value)
+                    or (other_value.has_day and not canonical_value.has_day)
+                    or (other_value.has_month and not canonical_value.has_month)
+                ):
+                    if not dry_run:
+                        setattr(self, data_field.name, other_value)
+                    absorbed_fields[data_field.name] = other_value
+            else:
+                if not canonical_value:
+                    if not dry_run:
+                        setattr(self, data_field.name, other_value)
+                    absorbed_fields[data_field.name] = other_value
+        return absorbed_fields
+
+
+class MergedBookDataModel(models.Model):
+    """a BookDataModel instance that has been merged into another instance. kept
+    to be able to redirect old URLs"""
+
+    deleted_id = models.IntegerField(primary_key=True)
+
+    class Meta:
+        """abstract just like BookDataModel"""
+
+        abstract = True
+
+
+class MergedBook(MergedBookDataModel):
+    """an Book that has been merged into another one"""
+
+    merged_into = models.ForeignKey(
+        "Book", on_delete=models.PROTECT, related_name="absorbed"
+    )
+
+
+class MergedAuthor(MergedBookDataModel):
+    """an Author that has been merged into another one"""
+
+    merged_into = models.ForeignKey(
+        "Author", on_delete=models.PROTECT, related_name="absorbed"
+    )
+
 
 class Book(BookDataModel):
     """a generic book, which can mean either an edition or a work"""
 
+    merged_model = MergedBook
+
     connector = models.ForeignKey("Connector", on_delete=models.PROTECT, null=True)
 
     # book/work metadata
@@ -192,9 +299,13 @@ class Book(BookDataModel):
         """properties of this edition, as a string"""
         items = [
             self.physical_format if hasattr(self, "physical_format") else None,
-            f"{self.languages[0]} language"
-            if self.languages and self.languages[0] and self.languages[0] != "English"
-            else None,
+            (
+                f"{self.languages[0]} language"
+                if self.languages
+                and self.languages[0]
+                and self.languages[0] != "English"
+                else None
+            ),
             str(self.published_date.year) if self.published_date else None,
             ", ".join(self.publishers) if hasattr(self, "publishers") else None,
         ]
@@ -216,7 +327,7 @@ class Book(BookDataModel):
 
     def get_remote_id(self):
         """editions and works both use "book" instead of model_name"""
-        return f"https://{DOMAIN}/book/{self.id}"
+        return f"{BASE_URL}/book/{self.id}"
 
     def guess_sort_title(self):
         """Get a best-guess sort title for the current book"""

bookwyrm/models/bookwyrm_export_job.py

@@ -1,213 +1,318 @@
 """Export user account to tar.gz file for import into another Bookwyrm instance"""
 
-import dataclasses
 import logging
-from uuid import uuid4
+import os
 
-from django.db.models import FileField
+from boto3.session import Session as BotoSession
+from s3_tar import S3Tar
+
+from django.db.models import BooleanField, FileField, JSONField
 from django.db.models import Q
 from django.core.serializers.json import DjangoJSONEncoder
 from django.core.files.base import ContentFile
+from django.utils.module_loading import import_string
 
-from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, List, ListItem
+from bookwyrm import settings, storage_backends
+
+from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, ListItem
 from bookwyrm.models import Review, Comment, Quotation
 from bookwyrm.models import Edition
 from bookwyrm.models import UserFollows, User, UserBlocks
-from bookwyrm.models.job import ParentJob, ParentTask
+from bookwyrm.models.job import ParentJob
 from bookwyrm.tasks import app, IMPORTS
 from bookwyrm.utils.tar import BookwyrmTarFile
 
 logger = logging.getLogger(__name__)
 
 
+class BookwyrmAwsSession(BotoSession):
+    """a boto session that always uses settings.AWS_S3_ENDPOINT_URL"""
+
+    def client(self, *args, **kwargs):  # pylint: disable=arguments-differ
+        kwargs["endpoint_url"] = settings.AWS_S3_ENDPOINT_URL
+        return super().client("s3", *args, **kwargs)
+
+
+def select_exports_storage():
+    """callable to allow for dependency on runtime configuration"""
+    cls = import_string(settings.EXPORTS_STORAGE)
+    return cls()
+
+
 class BookwyrmExportJob(ParentJob):
     """entry for a specific request to export a bookwyrm user"""
 
-    export_data = FileField(null=True)
+    export_data = FileField(null=True, storage=select_exports_storage)
+    export_json = JSONField(null=True, encoder=DjangoJSONEncoder)
+    json_completed = BooleanField(default=False)
 
     def start_job(self):
-        """Start the job"""
-        start_export_task.delay(job_id=self.id, no_children=True)
+        """schedule the first task"""
 
-        return self
+        task = create_export_json_task.delay(job_id=self.id)
+        self.task_id = task.id
+        self.save(update_fields=["task_id"])
 
 
-@app.task(queue=IMPORTS, base=ParentTask)
-def start_export_task(**kwargs):
-    """trigger the child tasks for each row"""
-    job = BookwyrmExportJob.objects.get(id=kwargs["job_id"])
+@app.task(queue=IMPORTS)
+def create_export_json_task(job_id):
+    """create the JSON data for the export"""
+
+    job = BookwyrmExportJob.objects.get(id=job_id)
 
     # don't start the job if it was stopped from the UI
     if job.complete:
         return
+
     try:
-        # This is where ChildJobs get made
-        job.export_data = ContentFile(b"", str(uuid4()))
-        json_data = json_export(job.user)
-        tar_export(json_data, job.user, job.export_data)
-        job.save(update_fields=["export_data"])
+        job.set_status("active")
+
+        # generate JSON structure
+        job.export_json = export_json(job.user)
+        job.save(update_fields=["export_json"])
+
+        # create archive in separate task
+        create_archive_task.delay(job_id=job.id)
     except Exception as err:  # pylint: disable=broad-except
-        logger.exception("User Export Job %s Failed with error: %s", job.id, err)
+        logger.exception(
+            "create_export_json_task for %s failed with error: %s", job, err
+        )
         job.set_status("failed")
 
-    job.set_status("complete")
+
+def archive_file_location(file, directory="") -> str:
+    """get the relative location of a file inside the archive"""
+    return os.path.join(directory, file.name)
 
 
-def tar_export(json_data: str, user, file):
-    """wrap the export information in a tar file"""
-    file.open("wb")
-    with BookwyrmTarFile.open(mode="w:gz", fileobj=file) as tar:
-        tar.write_bytes(json_data.encode("utf-8"))
+def add_file_to_s3_tar(s3_tar: S3Tar, storage, file, directory=""):
+    """
+    add file to S3Tar inside directory, keeping any directories under its
+    storage location
+    """
+    s3_tar.add_file(
+        os.path.join(storage.location, file.name),
+        folder=os.path.dirname(archive_file_location(file, directory=directory)),
+    )
 
-        # Add avatar image if present
-        if getattr(user, "avatar", False):
-            tar.add_image(user.avatar, filename="avatar")
 
+@app.task(queue=IMPORTS)
+def create_archive_task(job_id):
+    """create the archive containing the JSON file and additional files"""
+
+    job = BookwyrmExportJob.objects.get(id=job_id)
+
+    # don't start the job if it was stopped from the UI
+    if job.complete:
+        return
+
+    try:
+        export_task_id = str(job.task_id)
+        archive_filename = f"{export_task_id}.tar.gz"
+        export_json_bytes = DjangoJSONEncoder().encode(job.export_json).encode("utf-8")
+
+        user = job.user
         editions = get_books_for_user(user)
-        for book in editions:
-            if getattr(book, "cover", False):
-                tar.add_image(book.cover)
 
-    file.close()
+        if settings.USE_S3:
+            # Storage for writing temporary files
+            exports_storage = storage_backends.ExportsS3Storage()
+
+            # Handle for creating the final archive
+            s3_tar = S3Tar(
+                exports_storage.bucket_name,
+                os.path.join(exports_storage.location, archive_filename),
+                session=BookwyrmAwsSession(),
+            )
+
+            # Save JSON file to a temporary location
+            export_json_tmp_file = os.path.join(export_task_id, "archive.json")
+            exports_storage.save(
+                export_json_tmp_file,
+                ContentFile(export_json_bytes),
+            )
+            s3_tar.add_file(
+                os.path.join(exports_storage.location, export_json_tmp_file)
+            )
+
+            # Add images to TAR
+            images_storage = storage_backends.ImagesStorage()
+
+            if user.avatar:
+                add_file_to_s3_tar(s3_tar, images_storage, user.avatar)
+
+            for edition in editions:
+                if edition.cover:
+                    add_file_to_s3_tar(
+                        s3_tar, images_storage, edition.cover, directory="images"
+                    )
+
+            # Create archive and store file name
+            s3_tar.tar()
+            job.export_data = archive_filename
+            job.save(update_fields=["export_data"])
+
+            # Delete temporary files
+            exports_storage.delete(export_json_tmp_file)
+
+        else:
+            job.export_data = archive_filename
+            with job.export_data.open("wb") as tar_file:
+                with BookwyrmTarFile.open(mode="w:gz", fileobj=tar_file) as tar:
+                    # save json file
+                    tar.write_bytes(export_json_bytes)
+
+                    # Add avatar image if present
+                    if user.avatar:
+                        tar.add_image(user.avatar)
+
+                    for edition in editions:
+                        if edition.cover:
+                            tar.add_image(edition.cover, directory="images")
+            job.save(update_fields=["export_data"])
+
+        job.set_status("completed")
+
+    except Exception as err:  # pylint: disable=broad-except
+        logger.exception("create_archive_task for %s failed with error: %s", job, err)
+        job.set_status("failed")
 
 
-def json_export(
-    user,
-):  # pylint: disable=too-many-locals, too-many-statements, too-many-branches
-    """Generate an export for a user"""
+def export_json(user: User):
+    """create export JSON"""
+    data = export_user(user)  # in the root of the JSON structure
+    data["settings"] = export_settings(user)
+    data["goals"] = export_goals(user)
+    data["books"] = export_books(user)
+    data["saved_lists"] = export_saved_lists(user)
+    data["follows"] = export_follows(user)
+    data["blocks"] = export_blocks(user)
+    return data
 
-    # User as AP object
-    exported_user = user.to_activity()
-    # I don't love this but it prevents a JSON encoding error
-    # when there is no user image
-    if exported_user.get("icon") in (None, dataclasses.MISSING):
-        exported_user["icon"] = {}
+
+def export_user(user: User):
+    """export user data"""
+    data = user.to_activity()
+    if user.avatar:
+        data["icon"]["url"] = archive_file_location(user.avatar)
     else:
-        # change the URL to be relative to the JSON file
-        file_type = exported_user["icon"]["url"].rsplit(".", maxsplit=1)[-1]
-        filename = f"avatar.{file_type}"
-        exported_user["icon"]["url"] = filename
+        data["icon"] = {}
+    return data
 
-    # Additional settings - can't be serialized as AP
+
+def export_settings(user: User):
+    """Additional settings - can't be serialized as AP"""
     vals = [
         "show_goal",
         "preferred_timezone",
         "default_post_privacy",
         "show_suggested_users",
     ]
-    exported_user["settings"] = {}
-    for k in vals:
-        exported_user["settings"][k] = getattr(user, k)
+    return {k: getattr(user, k) for k in vals}
 
-    # Reading goals - can't be serialized as AP
-    reading_goals = AnnualGoal.objects.filter(user=user).distinct()
-    exported_user["goals"] = []
-    for goal in reading_goals:
-        exported_user["goals"].append(
-            {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy}
-        )
 
-    # Reading history - can't be serialized as AP
-    readthroughs = ReadThrough.objects.filter(user=user).distinct().values()
-    readthroughs = list(readthroughs)
+def export_saved_lists(user: User):
+    """add user saved lists to export JSON"""
+    return [l.remote_id for l in user.saved_lists.all()]
 
-    # Books
-    editions = get_books_for_user(user)
-    exported_user["books"] = []
 
-    for edition in editions:
-        book = {}
-        book["work"] = edition.parent_work.to_activity()
-        book["edition"] = edition.to_activity()
-
-        if book["edition"].get("cover"):
-            # change the URL to be relative to the JSON file
-            filename = book["edition"]["cover"]["url"].rsplit("/", maxsplit=1)[-1]
-            book["edition"]["cover"]["url"] = f"covers/{filename}"
-
-        # authors
-        book["authors"] = []
-        for author in edition.authors.all():
-            book["authors"].append(author.to_activity())
-
-        # Shelves this book is on
-        # Every ShelfItem is this book so we don't other serializing
-        book["shelves"] = []
-        shelf_books = (
-            ShelfBook.objects.select_related("shelf")
-            .filter(user=user, book=edition)
-            .distinct()
-        )
-
-        for shelfbook in shelf_books:
-            book["shelves"].append(shelfbook.shelf.to_activity())
-
-        # Lists and ListItems
-        # ListItems include "notes" and "approved" so we need them
-        # even though we know it's this book
-        book["lists"] = []
-        list_items = ListItem.objects.filter(book=edition, user=user).distinct()
-
-        for item in list_items:
-            list_info = item.book_list.to_activity()
-            list_info[
-                "privacy"
-            ] = item.book_list.privacy  # this isn't serialized so we add it
-            list_info["list_item"] = item.to_activity()
-            book["lists"].append(list_info)
-
-        # Statuses
-        # Can't use select_subclasses here because
-        # we need to filter on the "book" value,
-        # which is not available on an ordinary Status
-        for status in ["comments", "quotations", "reviews"]:
-            book[status] = []
-
-        comments = Comment.objects.filter(user=user, book=edition).all()
-        for status in comments:
-            obj = status.to_activity()
-            obj["progress"] = status.progress
-            obj["progress_mode"] = status.progress_mode
-            book["comments"].append(obj)
-
-        quotes = Quotation.objects.filter(user=user, book=edition).all()
-        for status in quotes:
-            obj = status.to_activity()
-            obj["position"] = status.position
-            obj["endposition"] = status.endposition
-            obj["position_mode"] = status.position_mode
-            book["quotations"].append(obj)
-
-        reviews = Review.objects.filter(user=user, book=edition).all()
-        for status in reviews:
-            obj = status.to_activity()
-            book["reviews"].append(obj)
-
-        # readthroughs can't be serialized to activity
-        book_readthroughs = (
-            ReadThrough.objects.filter(user=user, book=edition).distinct().values()
-        )
-        book["readthroughs"] = list(book_readthroughs)
-
-        # append everything
-        exported_user["books"].append(book)
-
-    # saved book lists - just the remote id
-    saved_lists = List.objects.filter(id__in=user.saved_lists.all()).distinct()
-    exported_user["saved_lists"] = [l.remote_id for l in saved_lists]
-
-    # follows - just the remote id
+def export_follows(user: User):
+    """add user follows to export JSON"""
     follows = UserFollows.objects.filter(user_subject=user).distinct()
     following = User.objects.filter(userfollows_user_object__in=follows).distinct()
-    exported_user["follows"] = [f.remote_id for f in following]
+    return [f.remote_id for f in following]
 
-    # blocks - just the remote id
+
+def export_blocks(user: User):
+    """add user blocks to export JSON"""
     blocks = UserBlocks.objects.filter(user_subject=user).distinct()
     blocking = User.objects.filter(userblocks_user_object__in=blocks).distinct()
+    return [b.remote_id for b in blocking]
 
-    exported_user["blocks"] = [b.remote_id for b in blocking]
 
-    return DjangoJSONEncoder().encode(exported_user)
+def export_goals(user: User):
+    """add user reading goals to export JSON"""
+    reading_goals = AnnualGoal.objects.filter(user=user).distinct()
+    return [
+        {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy}
+        for goal in reading_goals
+    ]
+
+
+def export_books(user: User):
+    """add books to export JSON"""
+    editions = get_books_for_user(user)
+    return [export_book(user, edition) for edition in editions]
+
+
+def export_book(user: User, edition: Edition):
+    """add book to export JSON"""
+    data = {}
+    data["work"] = edition.parent_work.to_activity()
+    data["edition"] = edition.to_activity()
+
+    if edition.cover:
+        data["edition"]["cover"]["url"] = archive_file_location(
+            edition.cover, directory="images"
+        )
+
+    # authors
+    data["authors"] = [author.to_activity() for author in edition.authors.all()]
+
+    # Shelves this book is on
+    # Every ShelfItem is this book so we don't other serializing
+    shelf_books = (
+        ShelfBook.objects.select_related("shelf")
+        .filter(user=user, book=edition)
+        .distinct()
+    )
+    data["shelves"] = [shelfbook.shelf.to_activity() for shelfbook in shelf_books]
+
+    # Lists and ListItems
+    # ListItems include "notes" and "approved" so we need them
+    # even though we know it's this book
+    list_items = ListItem.objects.filter(book=edition, user=user).distinct()
+
+    data["lists"] = []
+    for item in list_items:
+        list_info = item.book_list.to_activity()
+        list_info[
+            "privacy"
+        ] = item.book_list.privacy  # this isn't serialized so we add it
+        list_info["list_item"] = item.to_activity()
+        data["lists"].append(list_info)
+
+    # Statuses
+    # Can't use select_subclasses here because
+    # we need to filter on the "book" value,
+    # which is not available on an ordinary Status
+    for status in ["comments", "quotations", "reviews"]:
+        data[status] = []
+
+    comments = Comment.objects.filter(user=user, book=edition).all()
+    for status in comments:
+        obj = status.to_activity()
+        obj["progress"] = status.progress
+        obj["progress_mode"] = status.progress_mode
+        data["comments"].append(obj)
+
+    quotes = Quotation.objects.filter(user=user, book=edition).all()
+    for status in quotes:
+        obj = status.to_activity()
+        obj["position"] = status.position
+        obj["endposition"] = status.endposition
+        obj["position_mode"] = status.position_mode
+        data["quotations"].append(obj)
+
+    reviews = Review.objects.filter(user=user, book=edition).all()
+    data["reviews"] = [status.to_activity() for status in reviews]
+
+    # readthroughs can't be serialized to activity
+    book_readthroughs = (
+        ReadThrough.objects.filter(user=user, book=edition).distinct().values()
+    )
+    data["readthroughs"] = list(book_readthroughs)
+    return data
 
 
 def get_books_for_user(user):

bookwyrm/models/bookwyrm_import_job.py

@@ -42,20 +42,23 @@ def start_import_task(**kwargs):
     try:
         archive_file.open("rb")
         with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar:
-            job.import_data = json.loads(tar.read("archive.json").decode("utf-8"))
+            json_filename = next(
+                filter(lambda n: n.startswith("archive"), tar.getnames())
+            )
+            job.import_data = json.loads(tar.read(json_filename).decode("utf-8"))
 
             if "include_user_profile" in job.required:
                 update_user_profile(job.user, tar, job.import_data)
             if "include_user_settings" in job.required:
                 update_user_settings(job.user, job.import_data)
             if "include_goals" in job.required:
-                update_goals(job.user, job.import_data.get("goals"))
+                update_goals(job.user, job.import_data.get("goals", []))
             if "include_saved_lists" in job.required:
-                upsert_saved_lists(job.user, job.import_data.get("saved_lists"))
+                upsert_saved_lists(job.user, job.import_data.get("saved_lists", []))
             if "include_follows" in job.required:
-                upsert_follows(job.user, job.import_data.get("follows"))
+                upsert_follows(job.user, job.import_data.get("follows", []))
             if "include_blocks" in job.required:
-                upsert_user_blocks(job.user, job.import_data.get("blocks"))
+                upsert_user_blocks(job.user, job.import_data.get("blocks", []))
 
             process_books(job, tar)
 
@@ -212,7 +215,7 @@ def upsert_statuses(user, cls, data, book_remote_id):
                 instance.save()  # save and broadcast
 
         else:
-            logger.info("User does not have permission to import statuses")
+            logger.warning("User does not have permission to import statuses")
 
 
 def upsert_lists(user, lists, book_id):

bookwyrm/models/connector.py

@@ -11,7 +11,7 @@ ConnectorFiles = models.TextChoices("ConnectorFiles", CONNECTORS)
 class Connector(BookWyrmModel):
     """book data source connectors"""
 
-    identifier = models.CharField(max_length=255, unique=True)
+    identifier = models.CharField(max_length=255, unique=True)  # domain
     priority = models.IntegerField(default=2)
     name = models.CharField(max_length=255, null=True, blank=True)
     connector_file = models.CharField(max_length=255, choices=ConnectorFiles.choices)

bookwyrm/models/federated_server.py

@@ -16,7 +16,7 @@ FederationStatus = [
 class FederatedServer(BookWyrmModel):
     """store which servers we federate with"""
 
-    server_name = models.CharField(max_length=255, unique=True)
+    server_name = models.CharField(max_length=255, unique=True)  # domain
     status = models.CharField(
         max_length=255, default="federated", choices=FederationStatus
     )
@@ -64,5 +64,4 @@ class FederatedServer(BookWyrmModel):
     def is_blocked(cls, url: str) -> bool:
         """look up if a domain is blocked"""
         url = urlparse(url)
-        domain = url.netloc
-        return cls.objects.filter(server_name=domain, status="blocked").exists()
+        return cls.objects.filter(server_name=url.hostname, status="blocked").exists()

bookwyrm/models/fields.py

@@ -260,12 +260,12 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
 
         if to == [self.public]:
             setattr(instance, self.name, "public")
+        elif self.public in cc:
+            setattr(instance, self.name, "unlisted")
         elif to == [user.followers_url]:
             setattr(instance, self.name, "followers")
         elif cc == []:
             setattr(instance, self.name, "direct")
-        elif self.public in cc:
-            setattr(instance, self.name, "unlisted")
         else:
             setattr(instance, self.name, "followers")
         return original == getattr(instance, self.name)

bookwyrm/models/group.py

@@ -1,7 +1,7 @@
 """ do book related things with other users """
 from django.db import models, IntegrityError, transaction
 from django.db.models import Q
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from .base_model import BookWyrmModel
 from . import fields
 from .relationship import UserBlocks
@@ -17,7 +17,7 @@ class Group(BookWyrmModel):
 
     def get_remote_id(self):
         """don't want the user to be in there in this case"""
-        return f"https://{DOMAIN}/group/{self.id}"
+        return f"{BASE_URL}/group/{self.id}"
 
     @classmethod
     def followers_filter(cls, queryset, viewer):

bookwyrm/models/job.py

@@ -135,8 +135,7 @@ class ParentJob(Job):
         )
         app.control.revoke(list(tasks))
 
-        for task in self.pending_child_jobs:
-            task.update(status=self.Status.STOPPED)
+        self.pending_child_jobs.update(status=self.Status.STOPPED)
 
     @property
     def has_completed(self):
@@ -248,7 +247,7 @@ class SubTask(app.Task):
     """
 
     def before_start(
-        self, task_id, args, kwargs
+        self, task_id, *args, **kwargs
     ):  # pylint: disable=no-self-use, unused-argument
         """Handler called before the task starts. Override.
 
@@ -272,7 +271,7 @@ class SubTask(app.Task):
         child_job.set_status(ChildJob.Status.ACTIVE)
 
     def on_success(
-        self, retval, task_id, args, kwargs
+        self, retval, task_id, *args, **kwargs
     ):  # pylint: disable=no-self-use, unused-argument
         """Run by the worker if the task executes successfully. Override.
 

bookwyrm/models/link.py

@@ -38,7 +38,7 @@ class Link(ActivitypubMixin, BookWyrmModel):
         """create a link"""
         # get or create the associated domain
         if not self.domain:
-            domain = urlparse(self.url).netloc
+            domain = urlparse(self.url).hostname
             self.domain, _ = LinkDomain.objects.get_or_create(domain=domain)
 
         # this is never broadcast, the owning model broadcasts an update

bookwyrm/models/list.py

@@ -7,7 +7,7 @@ from django.db.models import Q
 from django.utils import timezone
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 
 from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin
 from .base_model import BookWyrmModel
@@ -50,7 +50,7 @@ class List(OrderedCollectionMixin, BookWyrmModel):
 
     def get_remote_id(self):
         """don't want the user to be in there in this case"""
-        return f"https://{DOMAIN}/list/{self.id}"
+        return f"{BASE_URL}/list/{self.id}"
 
     @property
     def collection_queryset(self):

bookwyrm/models/move.py

@@ -10,7 +10,7 @@ from .notification import Notification, NotificationType
 
 
 class Move(ActivityMixin, BookWyrmModel):
-    """migrating an activitypub user account"""
+    """migrating an activitypub object"""
 
     user = fields.ForeignKey(
         "User", on_delete=models.PROTECT, activitypub_field="actor"

bookwyrm/models/report.py

@@ -3,7 +3,7 @@ from django.core.exceptions import PermissionDenied
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from .base_model import BookWyrmModel
 
 
@@ -46,7 +46,7 @@ class Report(BookWyrmModel):
         raise PermissionDenied()
 
     def get_remote_id(self):
-        return f"https://{DOMAIN}/settings/reports/{self.id}"
+        return f"{BASE_URL}/settings/reports/{self.id}"
 
     def comment(self, user, note):
         """comment on a report"""

bookwyrm/models/shelf.py

@@ -6,7 +6,7 @@ from django.db import models
 from django.utils import timezone
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from bookwyrm.tasks import BROADCAST
 from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin
 from .base_model import BookWyrmModel
@@ -71,7 +71,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
     @property
     def local_path(self):
         """No slugs"""
-        return self.get_remote_id().replace(f"https://{DOMAIN}", "")
+        return self.get_remote_id().replace(BASE_URL, "")
 
     def raise_not_deletable(self, viewer):
         """don't let anyone delete a default shelf"""

bookwyrm/models/site.py

@@ -12,7 +12,7 @@ from model_utils import FieldTracker
 
 from bookwyrm.connectors.abstract_connector import get_data
 from bookwyrm.preview_images import generate_site_preview_image_task
-from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL
+from bookwyrm.settings import BASE_URL, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL
 from bookwyrm.settings import RELEASE_API
 from bookwyrm.tasks import app, MISC
 from .base_model import BookWyrmModel, new_access_code
@@ -188,7 +188,7 @@ class SiteInvite(models.Model):
     @property
     def link(self):
         """formats the invite link"""
-        return f"https://{DOMAIN}/invite/{self.code}"
+        return f"{BASE_URL}/invite/{self.code}"
 
 
 class InviteRequest(BookWyrmModel):
@@ -235,7 +235,7 @@ class PasswordReset(models.Model):
     @property
     def link(self):
         """formats the invite link"""
-        return f"https://{DOMAIN}/password-reset/{self.code}"
+        return f"{BASE_URL}/password-reset/{self.code}"
 
 
 # pylint: disable=unused-argument

bookwyrm/models/status.py

@@ -80,6 +80,10 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         """default sorting"""
 
         ordering = ("-published_date",)
+        indexes = [
+            models.Index(fields=["remote_id"]),
+            models.Index(fields=["thread_id"]),
+        ]
 
     def save(self, *args, **kwargs):
         """save and notify"""
@@ -388,10 +392,10 @@ class Quotation(BookStatus):
     def _format_position(self) -> Optional[str]:
         """serialize page position"""
         beg = self.position
-        end = self.endposition or 0
+        end = self.endposition
         if self.position_mode != "PG" or not beg:
             return None
-        return f"pp. {beg}-{end}" if end > beg else f"p. {beg}"
+        return f"pp. {beg}-{end}" if end else f"p. {beg}"
 
     @property
     def pure_content(self):

bookwyrm/models/user.py

@@ -19,7 +19,7 @@ from bookwyrm.connectors import get_data, ConnectorException
 from bookwyrm.models.shelf import Shelf
 from bookwyrm.models.status import Status
 from bookwyrm.preview_images import generate_user_preview_image_task
-from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, USE_HTTPS, LANGUAGES
+from bookwyrm.settings import BASE_URL, ENABLE_PREVIEW_IMAGES, LANGUAGES
 from bookwyrm.signatures import create_key_pair
 from bookwyrm.tasks import app, MISC
 from bookwyrm.utils import regex
@@ -42,12 +42,6 @@ def get_feed_filter_choices():
     return [f[0] for f in FeedFilterChoices]
 
 
-def site_link():
-    """helper for generating links to the site"""
-    protocol = "https" if USE_HTTPS else "http"
-    return f"{protocol}://{DOMAIN}"
-
-
 # pylint: disable=too-many-public-methods
 class User(OrderedCollectionPageMixin, AbstractUser):
     """a user who wants to read books"""
@@ -198,6 +192,14 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     hotp_secret = models.CharField(max_length=32, default=None, blank=True, null=True)
     hotp_count = models.IntegerField(default=0, blank=True, null=True)
 
+    class Meta(AbstractUser.Meta):
+        """indexes"""
+
+        indexes = [
+            models.Index(fields=["username"]),
+            models.Index(fields=["is_active", "local"]),
+        ]
+
     @property
     def active_follower_requests(self):
         """Follow requests from active users"""
@@ -206,8 +208,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     @property
     def confirmation_link(self):
         """helper for generating confirmation links"""
-        link = site_link()
-        return f"{link}/confirm-email/{self.confirmation_code}"
+        return f"{BASE_URL}/confirm-email/{self.confirmation_code}"
 
     @property
     def following_link(self):
@@ -341,7 +342,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         if not self.local and not re.match(regex.FULL_USERNAME, self.username):
             # generate a username that uses the domain (webfinger format)
             actor_parts = urlparse(self.remote_id)
-            self.username = f"{self.username}@{actor_parts.netloc}"
+            self.username = f"{self.username}@{actor_parts.hostname}"
 
         # this user already exists, no need to populate fields
         if not created:
@@ -361,11 +362,10 @@ class User(OrderedCollectionPageMixin, AbstractUser):
 
         with transaction.atomic():
             # populate fields for local users
-            link = site_link()
-            self.remote_id = f"{link}/user/{self.localname}"
+            self.remote_id = f"{BASE_URL}/user/{self.localname}"
             self.followers_url = f"{self.remote_id}/followers"
             self.inbox = f"{self.remote_id}/inbox"
-            self.shared_inbox = f"{link}/inbox"
+            self.shared_inbox = f"{BASE_URL}/inbox"
             self.outbox = f"{self.remote_id}/outbox"
 
             # an id needs to be set before we can proceed with related models
@@ -509,6 +509,13 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
     activity_serializer = activitypub.PublicKey
     serialize_reverse_fields = [("owner", "owner", "id")]
 
+    class Meta:
+        """indexes"""
+
+        indexes = [
+            models.Index(fields=["remote_id"]),
+        ]
+
     def get_remote_id(self):
         # self.owner is set by the OneToOneField on User
         return f"{self.owner.remote_id}/#main-key"
@@ -543,7 +550,7 @@ def set_remote_server(user_id, allow_external_connections=False):
     user = User.objects.get(id=user_id)
     actor_parts = urlparse(user.remote_id)
     federated_server = get_or_create_remote_server(
-        actor_parts.netloc, allow_external_connections=allow_external_connections
+        actor_parts.hostname, allow_external_connections=allow_external_connections
     )
     # if we were unable to find the server, we need to create a new entry for it
     if not federated_server:

bookwyrm/preview_images.py

@@ -175,11 +175,13 @@ def generate_instance_layer(content_width):
     site = models.SiteSettings.objects.get()
 
     if site.logo_small:
-        logo_img = Image.open(site.logo_small)
+        with Image.open(site.logo_small) as logo_img:
+            logo_img.load()
     else:
         try:
             static_path = os.path.join(settings.STATIC_ROOT, "images/logo-small.png")
-            logo_img = Image.open(static_path)
+            with Image.open(static_path) as logo_img:
+                logo_img.load()
         except FileNotFoundError:
             logo_img = None
 
@@ -211,18 +213,9 @@ def generate_instance_layer(content_width):
 
 def generate_rating_layer(rating, content_width):
     """Places components for rating preview"""
-    try:
-        icon_star_full = Image.open(
-            os.path.join(settings.STATIC_ROOT, "images/icons/star-full.png")
-        )
-        icon_star_empty = Image.open(
-            os.path.join(settings.STATIC_ROOT, "images/icons/star-empty.png")
-        )
-        icon_star_half = Image.open(
-            os.path.join(settings.STATIC_ROOT, "images/icons/star-half.png")
-        )
-    except FileNotFoundError:
-        return None
+    path_star_full = os.path.join(settings.STATIC_ROOT, "images/icons/star-full.png")
+    path_star_empty = os.path.join(settings.STATIC_ROOT, "images/icons/star-empty.png")
+    path_star_half = os.path.join(settings.STATIC_ROOT, "images/icons/star-half.png")
 
     icon_size = 64
     icon_margin = 10
@@ -237,17 +230,23 @@ def generate_rating_layer(rating, content_width):
 
     position_x = 0
 
-    for _ in range(math.floor(rating)):
-        rating_layer_mask.alpha_composite(icon_star_full, (position_x, 0))
-        position_x = position_x + icon_size + icon_margin
+    try:
+        with Image.open(path_star_full) as icon_star_full:
+            for _ in range(math.floor(rating)):
+                rating_layer_mask.alpha_composite(icon_star_full, (position_x, 0))
+                position_x = position_x + icon_size + icon_margin
 
-    if math.floor(rating) != math.ceil(rating):
-        rating_layer_mask.alpha_composite(icon_star_half, (position_x, 0))
-        position_x = position_x + icon_size + icon_margin
+        if math.floor(rating) != math.ceil(rating):
+            with Image.open(path_star_half) as icon_star_half:
+                rating_layer_mask.alpha_composite(icon_star_half, (position_x, 0))
+                position_x = position_x + icon_size + icon_margin
 
-    for _ in range(5 - math.ceil(rating)):
-        rating_layer_mask.alpha_composite(icon_star_empty, (position_x, 0))
-        position_x = position_x + icon_size + icon_margin
+        with Image.open(path_star_empty) as icon_star_empty:
+            for _ in range(5 - math.ceil(rating)):
+                rating_layer_mask.alpha_composite(icon_star_empty, (position_x, 0))
+                position_x = position_x + icon_size + icon_margin
+    except FileNotFoundError:
+        return None
 
     rating_layer_mask = rating_layer_mask.getchannel("A")
     rating_layer_mask = ImageOps.invert(rating_layer_mask)
@@ -290,7 +289,8 @@ def generate_preview_image(
     texts = texts or {}
     # Cover
     try:
-        inner_img_layer = Image.open(picture)
+        with Image.open(picture) as inner_img_layer:
+            inner_img_layer.load()
         inner_img_layer.thumbnail(
             (inner_img_width, inner_img_height), Image.Resampling.LANCZOS
         )

bookwyrm/settings.py

@@ -19,7 +19,6 @@ DOMAIN = env("DOMAIN")
 with open("VERSION", encoding="utf-8") as f:
     version = f.read()
     version = version.replace("\n", "")
-f.close()
 
 VERSION = version
 
@@ -102,6 +101,7 @@ INSTALLED_APPS = [
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "django.contrib.humanize",
+    "oauth2_provider",
     "file_resubmit",
     "sass_processor",
     "bookwyrm",
@@ -351,30 +351,34 @@ USE_L10N = True
 
 USE_TZ = True
 
-
-USER_AGENT = f"BookWyrm (BookWyrm/{VERSION}; +https://{DOMAIN}/)"
-
 # Imagekit generated thumbnails
 ENABLE_THUMBNAIL_GENERATION = env.bool("ENABLE_THUMBNAIL_GENERATION", False)
 IMAGEKIT_CACHEFILE_DIR = "thumbnails"
 IMAGEKIT_DEFAULT_CACHEFILE_STRATEGY = "bookwyrm.thumbnail_generation.Strategy"
 
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/3.2/howto/static-files/
-
 PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
 CSP_ADDITIONAL_HOSTS = env.list("CSP_ADDITIONAL_HOSTS", [])
 
-# Storage
-
 PROTOCOL = "http"
 if USE_HTTPS:
     PROTOCOL = "https"
     SESSION_COOKIE_SECURE = True
     CSRF_COOKIE_SECURE = True
 
+PORT = env.int("PORT", 443 if USE_HTTPS else 80)
+if (USE_HTTPS and PORT == 443) or (not USE_HTTPS and PORT == 80):
+    NETLOC = DOMAIN
+else:
+    NETLOC = f"{DOMAIN}:{PORT}"
+BASE_URL = f"{PROTOCOL}://{NETLOC}"
+
+USER_AGENT = f"BookWyrm (BookWyrm/{VERSION}; +{BASE_URL})"
+
+# Storage
+
 USE_S3 = env.bool("USE_S3", False)
 USE_AZURE = env.bool("USE_AZURE", False)
+S3_SIGNED_URL_EXPIRY = env.int("S3_SIGNED_URL_EXPIRY", 900)
 
 if USE_S3:
     # AWS settings
@@ -386,19 +390,34 @@ if USE_S3:
     AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL", None)
     AWS_DEFAULT_ACL = "public-read"
     AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"}
+    AWS_S3_URL_PROTOCOL = env("AWS_S3_URL_PROTOCOL", f"{PROTOCOL}:")
     # S3 Static settings
     STATIC_LOCATION = "static"
-    STATIC_URL = f"{PROTOCOL}://{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/"
+    STATIC_URL = f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/"
+    STATIC_FULL_URL = STATIC_URL
     STATICFILES_STORAGE = "bookwyrm.storage_backends.StaticStorage"
     # S3 Media settings
     MEDIA_LOCATION = "images"
-    MEDIA_URL = f"{PROTOCOL}://{AWS_S3_CUSTOM_DOMAIN}/{MEDIA_LOCATION}/"
+    MEDIA_URL = f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}/{MEDIA_LOCATION}/"
     MEDIA_FULL_URL = MEDIA_URL
-    STATIC_FULL_URL = STATIC_URL
     DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
-    CSP_DEFAULT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
-    CSP_SCRIPT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
+    # S3 Exports settings
+    EXPORTS_STORAGE = "bookwyrm.storage_backends.ExportsS3Storage"
+    # Content Security Policy
+    CSP_DEFAULT_SRC = [
+        "'self'",
+        f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}"
+        if AWS_S3_CUSTOM_DOMAIN
+        else None,
+    ] + CSP_ADDITIONAL_HOSTS
+    CSP_SCRIPT_SRC = [
+        "'self'",
+        f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}"
+        if AWS_S3_CUSTOM_DOMAIN
+        else None,
+    ] + CSP_ADDITIONAL_HOSTS
 elif USE_AZURE:
+    # Azure settings
     AZURE_ACCOUNT_NAME = env("AZURE_ACCOUNT_NAME")
     AZURE_ACCOUNT_KEY = env("AZURE_ACCOUNT_KEY")
     AZURE_CONTAINER = env("AZURE_CONTAINER")
@@ -408,6 +427,7 @@ elif USE_AZURE:
     STATIC_URL = (
         f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{STATIC_LOCATION}/"
     )
+    STATIC_FULL_URL = STATIC_URL
     STATICFILES_STORAGE = "bookwyrm.storage_backends.AzureStaticStorage"
     # Azure Media settings
     MEDIA_LOCATION = "images"
@@ -415,15 +435,24 @@ elif USE_AZURE:
         f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{MEDIA_LOCATION}/"
     )
     MEDIA_FULL_URL = MEDIA_URL
-    STATIC_FULL_URL = STATIC_URL
     DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.AzureImagesStorage"
+    # Azure Exports settings
+    EXPORTS_STORAGE = None  # not implemented yet
+    # Content Security Policy
     CSP_DEFAULT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
     CSP_SCRIPT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
 else:
+    # Static settings
     STATIC_URL = "/static/"
+    STATIC_FULL_URL = BASE_URL + STATIC_URL
+    STATICFILES_STORAGE = "django.contrib.staticfiles.storage.StaticFilesStorage"
+    # Media settings
     MEDIA_URL = "/images/"
-    MEDIA_FULL_URL = f"{PROTOCOL}://{DOMAIN}{MEDIA_URL}"
-    STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}"
+    MEDIA_FULL_URL = BASE_URL + MEDIA_URL
+    DEFAULT_FILE_STORAGE = "django.core.files.storage.FileSystemStorage"
+    # Exports settings
+    EXPORTS_STORAGE = "bookwyrm.storage_backends.ExportsFileStorage"
+    # Content Security Policy
     CSP_DEFAULT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS
     CSP_SCRIPT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS
 

bookwyrm/storage_backends.py

@@ -1,6 +1,7 @@
 """Handles backends for storages"""
 import os
 from tempfile import SpooledTemporaryFile
+from django.core.files.storage import FileSystemStorage
 from storages.backends.s3boto3 import S3Boto3Storage
 from storages.backends.azure_storage import AzureStorage
 
@@ -61,3 +62,18 @@ class AzureImagesStorage(AzureStorage):  # pylint: disable=abstract-method
 
     location = "images"
     overwrite_files = False
+
+
+class ExportsFileStorage(FileSystemStorage):  # pylint: disable=abstract-method
+    """Storage class for exports contents with local files"""
+
+    location = "exports"
+    overwrite_files = False
+
+
+class ExportsS3Storage(S3Boto3Storage):  # pylint: disable=abstract-method
+    """Storage class for exports contents with S3"""
+
+    location = "exports"
+    default_acl = None
+    overwrite_files = False

bookwyrm/templates/email/html_layout.html

@@ -2,10 +2,10 @@
 <div style="font-family: BlinkMacSystemFont,-apple-system,'Segoe UI',Roboto,Oxygen,Ubuntu,Cantarell,'Fira Sans','Droid Sans','Helvetica Neue',Helvetica,Arial,sans-serif; border-radius: 6px; background-color: #efefef; max-width: 632px;">
     <div style="padding: 1rem; overflow: auto;">
         <div style="float: left; margin-right: 1rem;">
-            <a style="color: #3273dc;" href="https://{{ domain }}" style="text-decoration: none;"><img src="{{ logo }}" alt="logo" loading="lazy" decoding="async"></a>
+            <a style="color: #3273dc;" href="{{ base_url }}" style="text-decoration: none;"><img src="{{ logo }}" alt="logo" loading="lazy" decoding="async"></a>
         </div>
         <div>
-            <a style="color: black; text-decoration: none" href="https://{{ domain }}" style="text-decoration: none;"><strong>{{ site_name }}</strong><br>
+            <a style="color: black; text-decoration: none" href="{{ base_url }}" style="text-decoration: none;"><strong>{{ site_name }}</strong><br>
             {{ domain }}</a>
         </div>
     </div>
@@ -18,9 +18,9 @@
     </div>
 
     <div style="padding: 1rem; font-size: 0.8rem;">
-        <p style="margin: 0; color: #333;">{% blocktrans %}BookWyrm hosted on <a style="color: #3273dc;" href="https://{{ domain }}">{{ site_name }}</a>{% endblocktrans %}</p>
+        <p style="margin: 0; color: #333;">{% blocktrans %}BookWyrm hosted on <a style="color: #3273dc;" href="{{ base_url }}">{{ site_name }}</a>{% endblocktrans %}</p>
         {% if user %}
-        <p style="margin: 0; color: #333;"><a style="color: #3273dc;" href="https://{{ domain }}{% url 'prefs-profile' %}">{% trans "Email preference" %}</a></p>
+        <p style="margin: 0; color: #333;"><a style="color: #3273dc;" href="{{ base_url }}{% url 'prefs-profile' %}">{% trans "Email preference" %}</a></p>
         {% endif %}
     </div>
 </div>

bookwyrm/templates/email/invite/html_content.html

@@ -12,6 +12,6 @@
 <p>
     {% url 'code-of-conduct' as coc_path %}
     {% url 'about' as about_path %}
-    {% blocktrans %}Learn more <a href="https://{{ domain }}{{ about_path }}">about {{ site_name }}</a>.{% endblocktrans %}
+    {% blocktrans %}Learn more <a href="{{ base_url }}{{ about_path }}">about {{ site_name }}</a>.{% endblocktrans %}
 </p>
 {% endblock %}

bookwyrm/templates/email/invite/text_content.html

@@ -5,6 +5,6 @@
 
 {{ invite_link }}
 
-{% blocktrans %}Learn more about {{ site_name }}:{% endblocktrans %} https://{{ domain }}{% url 'about' %}
+{% blocktrans %}Learn more about {{ site_name }}:{% endblocktrans %} {{ base_url }}{% url 'about' %}
 
 {% endblock %}

bookwyrm/templates/opensearch.xml

@@ -10,6 +10,6 @@
     <Image width="16" height="16" type="image/x-icon">{{ image }}</Image>
     <Url
         type="text/html"
-        template="https://{{ DOMAIN }}{% url 'search' %}?q={searchTerms}"
+        template="{{ BASE_URL }}{% url 'search' %}?q={searchTerms}"
     />
 </OpenSearchDescription>

bookwyrm/templates/preferences/export-user.html

@@ -97,25 +97,25 @@
                 </td>
             </tr>
             {% endif %}
-            {% for job in jobs %}
+            {% for export in jobs %}
             <tr>
-                <td>{{ job.updated_date }}</td>
+                <td>{{ export.job.updated_date }}</td>
                 <td>
                     <span
-                        {% if job.status == "stopped" or job.status == "failed" %}
+                        {% if export.job.status == "stopped" or export.job.status == "failed" %}
                         class="tag is-danger"
-                        {% elif job.status == "pending" %}
+                        {% elif export.job.status == "pending" %}
                         class="tag is-warning"
-                        {% elif job.complete %}
+                        {% elif export.job.complete %}
                         class="tag"
                         {% else %}
                         class="tag is-success"
                         {% endif %}
                     >
-                        {% if job.status %}
-                        {{ job.status }}
-                            {{ job.status_display }}
-                        {% elif job.complete %}
+                        {% if export.job.status %}
+                        {{ export.job.status }}
+                            {{ export.job.status_display }}
+                        {% elif export.job.complete %}
                             {% trans "Complete" %}
                         {% else %}
                             {% trans "Active" %}
@@ -123,18 +123,20 @@
                     </span>
                 </td>
                 <td>
-                    <span>{{ job.export_data|get_file_size }}</span>
+                    {% if export.size %}
+                    <span>{{ export.size|get_file_size }}</span>
+                    {% endif %}
                 </td>
                 <td>
-                    {% if job.complete and not job.status == "stopped" and not job.status == "failed" %}
-                    <p>
-                        <a download="" href="/preferences/user-export/{{ job.task_id }}">
-                            <span class="icon icon-download" aria-hidden="true"></span>
-                            <span class="is-hidden-mobile">
-                                {% trans "Download your export" %}
-                            </span>
-                        </a>
-                    </p>
+                    {% if export.url %}
+                    <a href="{{ export.url }}">
+                        <span class="icon icon-download" aria-hidden="true"></span>
+                        <span class="is-hidden-mobile">
+                            {% trans "Download your export" %}
+                        </span>
+                    </a>
+                    {% elif export.unavailable %}
+                    {% trans "Archive is no longer available" %}
                     {% endif %}
                 </td>
             </tr>

bookwyrm/templates/settings/imports/imports.html

@@ -157,13 +157,13 @@
     >
         <div class="notification is-danger is-light">
             <p class="my-2">{% trans "Users are currently unable to start new user exports. This is the default setting." %}</p>
-            {% if use_s3 %}
-            <p>{% trans "It is not currently possible to provide user exports when using s3 storage. The BookWyrm development team are working on a fix for this." %}</p>
+            {% if use_azure %}
+            <p>{% trans "It is not currently possible to provide user exports when using Azure storage." %}</p>
             {% endif %}
         </div>
         {% csrf_token %}
         <div class="control">
-            <button type="submit" class="button is-success" {% if use_s3 %}disabled{% endif %}>
+            <button type="submit" class="button is-success" {% if use_azure %}disabled{% endif %}>
                 {% trans "Enable user exports" %}
             </button>
         </div>

bookwyrm/templatetags/utilities.py

@@ -120,7 +120,7 @@ def id_to_username(user_id):
     """given an arbitrary remote id, return the username"""
     if user_id:
         url = urlparse(user_id)
-        domain = url.netloc
+        domain = url.hostname
         parts = url.path.split("/")
         name = parts[-1]
         value = f"{name}@{domain}"
@@ -130,11 +130,14 @@ def id_to_username(user_id):
 
 
 @register.filter(name="get_file_size")
-def get_file_size(file):
+def get_file_size(nbytes):
     """display the size of a file in human readable terms"""
 
     try:
-        raw_size = os.stat(file.path).st_size
+        raw_size = float(nbytes)
+    except (ValueError, TypeError):
+        return repr(nbytes)
+    else:
         if raw_size < 1024:
             return f"{raw_size} bytes"
         if raw_size < 1024**2:
@@ -142,8 +145,6 @@ def get_file_size(file):
         if raw_size < 1024**3:
             return f"{raw_size/1024**2:.2f} MB"
         return f"{raw_size/1024**3:.2f} GB"
-    except Exception:  # pylint: disable=broad-except
-        return ""
 
 
 @register.filter(name="get_user_permission")

bookwyrm/tests/activitypub/test_author.py

@@ -7,13 +7,13 @@ class Author(TestCase):
     """serialize author tests"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """initial data"""
-        self.book = models.Edition.objects.create(
+        cls.book = models.Edition.objects.create(
             title="Example Edition",
             remote_id="https://example.com/book/1",
         )
-        self.author = models.Author.objects.create(
+        cls.author = models.Author.objects.create(
             name="Author fullname",
             aliases=["One", "Two"],
             bio="bio bio bio",

bookwyrm/tests/activitypub/test_base_activity.py

@@ -1,12 +1,10 @@
 """ tests the base functionality for activitypub dataclasses """
-from io import BytesIO
 import json
 import pathlib
 from unittest.mock import patch
 
 from dataclasses import dataclass
 from django.test import TestCase
-from PIL import Image
 import responses
 
 from bookwyrm import activitypub
@@ -29,16 +27,18 @@ class BaseActivity(TestCase):
     """the super class for model-linked activitypub dataclasses"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """we're probably going to re-use this so why copy/paste"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "mouseword", local=True, localname="mouse"
             )
-        self.user.remote_id = "http://example.com/a/b"
-        self.user.save(broadcast=False, update_fields=["remote_id"])
+        cls.user.remote_id = "http://example.com/a/b"
+        cls.user.save(broadcast=False, update_fields=["remote_id"])
 
     def setUp(self):
         datafile = pathlib.Path(__file__).parent.joinpath("../data/ap_user.json")
@@ -46,13 +46,11 @@ class BaseActivity(TestCase):
         # don't try to load the user icon
         del self.userdata["icon"]
 
-        image_file = pathlib.Path(__file__).parent.joinpath(
+        image_path = pathlib.Path(__file__).parent.joinpath(
             "../../static/images/default_avi.jpg"
         )
-        image = Image.open(image_file)
-        output = BytesIO()
-        image.save(output, format=image.format)
-        self.image_data = output.getvalue()
+        with open(image_path, "rb") as image_file:
+            self.image_data = image_file.read()
 
     def test_get_representative_not_existing(self, *_):
         """test that an instance representative actor is created if it does not exist"""
@@ -232,10 +230,12 @@ class BaseActivity(TestCase):
         )
 
         # sets the celery task call to the function call
-        with patch("bookwyrm.activitypub.base_activity.set_related_field.delay"):
-            with patch("bookwyrm.models.status.Status.ignore_activity") as discarder:
-                discarder.return_value = False
-                update_data.to_model(model=models.Status, instance=status)
+        with (
+            patch("bookwyrm.activitypub.base_activity.set_related_field.delay"),
+            patch("bookwyrm.models.status.Status.ignore_activity") as discarder,
+        ):
+            discarder.return_value = False
+            update_data.to_model(model=models.Status, instance=status)
         self.assertIsNone(status.attachments.first())
 
     @responses.activate

bookwyrm/tests/activitypub/test_note.py

@@ -11,18 +11,20 @@ class Note(TestCase):
     """the model-linked ActivityPub dataclass for Note-based types"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """create a shared user"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "mouseword", local=True, localname="mouse"
             )
-        self.user.remote_id = "https://test-instance.org/user/critic"
-        self.user.save(broadcast=False, update_fields=["remote_id"])
+        cls.user.remote_id = "https://test-instance.org/user/critic"
+        cls.user.save(broadcast=False, update_fields=["remote_id"])
 
-        self.book = models.Edition.objects.create(
+        cls.book = models.Edition.objects.create(
             title="Test Edition", remote_id="http://book.com/book"
         )
 

bookwyrm/tests/activitypub/test_quotation.py

@@ -11,10 +11,10 @@ class Quotation(TestCase):
     """we have hecka ways to create statuses"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """model objects we'll need"""
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.user = models.User.objects.create_user(
+            cls.user = models.User.objects.create_user(
                 "mouse",
                 "mouse@mouse.mouse",
                 "mouseword",
@@ -23,7 +23,7 @@ class Quotation(TestCase):
                 outbox="https://example.com/user/mouse/outbox",
                 remote_id="https://example.com/user/mouse",
             )
-        self.book = models.Edition.objects.create(
+        cls.book = models.Edition.objects.create(
             title="Example Edition",
             remote_id="https://example.com/book/1",
         )

bookwyrm/tests/activitystreams/test_abstractstream.py

@@ -16,15 +16,17 @@ class Activitystreams(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """use a test csv"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
-            self.another_user = models.User.objects.create_user(
+            cls.another_user = models.User.objects.create_user(
                 "nutria",
                 "nutria@nutria.nutria",
                 "password",
@@ -32,7 +34,7 @@ class Activitystreams(TestCase):
                 localname="nutria",
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -42,7 +44,7 @@ class Activitystreams(TestCase):
                 outbox="https://example.com/users/rat/outbox",
             )
         work = models.Work.objects.create(title="test work")
-        self.book = models.Edition.objects.create(title="test book", parent_work=work)
+        cls.book = models.Edition.objects.create(title="test book", parent_work=work)
 
     def setUp(self):
         """per-test setUp"""
@@ -105,9 +107,11 @@ class Activitystreams(TestCase):
             privacy="direct",
             book=self.book,
         )
-        with patch("bookwyrm.activitystreams.r.set"), patch(
-            "bookwyrm.activitystreams.r.delete"
-        ), patch("bookwyrm.activitystreams.ActivityStream.get_store") as redis_mock:
+        with (
+            patch("bookwyrm.activitystreams.r.set"),
+            patch("bookwyrm.activitystreams.r.delete"),
+            patch("bookwyrm.activitystreams.ActivityStream.get_store") as redis_mock,
+        ):
             redis_mock.return_value = [status.id, status2.id]
             result = self.test_stream.get_activity_stream(self.local_user)
         self.assertEqual(result.count(), 2)

bookwyrm/tests/activitystreams/test_booksstream.py

@@ -15,16 +15,18 @@ class Activitystreams(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """use a test csv"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -34,7 +36,7 @@ class Activitystreams(TestCase):
                 outbox="https://example.com/users/rat/outbox",
             )
         work = models.Work.objects.create(title="test work")
-        self.book = models.Edition.objects.create(title="test book", parent_work=work)
+        cls.book = models.Edition.objects.create(title="test book", parent_work=work)
 
     def test_get_statuses_for_user_books(self, *_):
         """create a stream for a user"""

bookwyrm/tests/activitystreams/test_homestream.py

@@ -13,15 +13,17 @@ class Activitystreams(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """use a test csv"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
-            self.another_user = models.User.objects.create_user(
+            cls.another_user = models.User.objects.create_user(
                 "nutria",
                 "nutria@nutria.nutria",
                 "password",
@@ -29,7 +31,7 @@ class Activitystreams(TestCase):
                 localname="nutria",
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",

bookwyrm/tests/activitystreams/test_localstream.py

@@ -13,15 +13,17 @@ class Activitystreams(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """use a test csv"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
-            self.another_user = models.User.objects.create_user(
+            cls.another_user = models.User.objects.create_user(
                 "nutria",
                 "nutria@nutria.nutria",
                 "password",
@@ -29,7 +31,7 @@ class Activitystreams(TestCase):
                 localname="nutria",
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -39,7 +41,7 @@ class Activitystreams(TestCase):
                 outbox="https://example.com/users/rat/outbox",
             )
         work = models.Work.objects.create(title="test work")
-        self.book = models.Edition.objects.create(title="test book", parent_work=work)
+        cls.book = models.Edition.objects.create(title="test book", parent_work=work)
 
     def test_localstream_get_audience_remote_status(self, *_):
         """get a list of users that should see a status"""

bookwyrm/tests/activitystreams/test_signals.py

@@ -15,16 +15,18 @@ class ActivitystreamsSignals(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """use a test csv"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",

bookwyrm/tests/activitystreams/test_tasks.py

@@ -8,15 +8,17 @@ class Activitystreams(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """use a test csv"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
-            self.another_user = models.User.objects.create_user(
+            cls.another_user = models.User.objects.create_user(
                 "nutria",
                 "nutria@nutria.nutria",
                 "password",
@@ -24,7 +26,7 @@ class Activitystreams(TestCase):
                 localname="nutria",
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -34,11 +36,9 @@ class Activitystreams(TestCase):
                 outbox="https://example.com/users/rat/outbox",
             )
         work = models.Work.objects.create(title="test work")
-        self.book = models.Edition.objects.create(title="test book", parent_work=work)
+        cls.book = models.Edition.objects.create(title="test book", parent_work=work)
         with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
-            self.status = models.Status.objects.create(
-                content="hi", user=self.local_user
-            )
+            cls.status = models.Status.objects.create(content="hi", user=cls.local_user)
 
     def test_add_book_statuses_task(self):
         """statuses related to a book"""

bookwyrm/tests/connectors/test_abstract_connector.py

@@ -6,14 +6,14 @@ import responses
 from bookwyrm import models
 from bookwyrm.connectors import abstract_connector, ConnectorException
 from bookwyrm.connectors.abstract_connector import Mapping, get_data
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 
 
 class AbstractConnector(TestCase):
     """generic code for connecting to outside data sources"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """we need an example connector in the database"""
         models.Connector.objects.create(
             identifier="example.com",
@@ -23,7 +23,7 @@ class AbstractConnector(TestCase):
             covers_url="https://example.com/covers",
             search_url="https://example.com/search?q=",
         )
-        self.book = models.Edition.objects.create(
+        cls.book = models.Edition.objects.create(
             title="Test Book",
             remote_id="https://example.com/book/1234",
             openlibrary_key="OL1234M",
@@ -86,7 +86,7 @@ class AbstractConnector(TestCase):
     def test_get_or_create_book_existing(self):
         """find an existing book by remote/origin id"""
         self.assertEqual(models.Book.objects.count(), 1)
-        self.assertEqual(self.book.remote_id, f"https://{DOMAIN}/book/{self.book.id}")
+        self.assertEqual(self.book.remote_id, f"{BASE_URL}/book/{self.book.id}")
         self.assertEqual(self.book.origin_id, "https://example.com/book/1234")
 
         # dedupe by origin id
@@ -95,9 +95,7 @@ class AbstractConnector(TestCase):
         self.assertEqual(result, self.book)
 
         # dedupe by remote id
-        result = self.connector.get_or_create_book(
-            f"https://{DOMAIN}/book/{self.book.id}"
-        )
+        result = self.connector.get_or_create_book(f"{BASE_URL}/book/{self.book.id}")
 
         self.assertEqual(models.Book.objects.count(), 1)
         self.assertEqual(result, self.book)

bookwyrm/tests/connectors/test_abstract_minimal_connector.py

@@ -10,9 +10,9 @@ class AbstractConnector(TestCase):
     """generic code for connecting to outside data sources"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """we need an example connector in the database"""
-        self.connector_info = models.Connector.objects.create(
+        cls.connector_info = models.Connector.objects.create(
             identifier="example.com",
             connector_file="openlibrary",
             base_url="https://example.com",

bookwyrm/tests/connectors/test_bookwyrm_connector.py

@@ -12,7 +12,7 @@ class BookWyrmConnector(TestCase):
     """this connector doesn't do much, just search"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """create bookwrym_connector in the database"""
         models.Connector.objects.create(
             identifier="example.com",

bookwyrm/tests/connectors/test_connector_manager.py

@@ -11,18 +11,18 @@ class ConnectorManager(TestCase):
     """interface between the app and various connectors"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """we'll need some books and a connector info entry"""
-        self.work = models.Work.objects.create(title="Example Work")
+        cls.work = models.Work.objects.create(title="Example Work")
 
         models.Edition.objects.create(
-            title="Example Edition", parent_work=self.work, isbn_10="0000000000"
+            title="Example Edition", parent_work=cls.work, isbn_10="0000000000"
         )
-        self.edition = models.Edition.objects.create(
-            title="Another Edition", parent_work=self.work, isbn_10="1111111111"
+        cls.edition = models.Edition.objects.create(
+            title="Another Edition", parent_work=cls.work, isbn_10="1111111111"
         )
 
-        self.remote_connector = models.Connector.objects.create(
+        cls.remote_connector = models.Connector.objects.create(
             identifier="test_connector_remote",
             priority=1,
             connector_file="bookwyrm_connector",

bookwyrm/tests/connectors/test_inventaire_connector.py

@@ -15,7 +15,7 @@ class Inventaire(TestCase):
     """test loading data from inventaire.io"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """creates the connector in the database"""
         models.Connector.objects.create(
             identifier="inventaire.io",
@@ -212,11 +212,14 @@ class Inventaire(TestCase):
             json={"entities": {}},
         )
         data = {"uri": "blah"}
-        with patch(
-            "bookwyrm.connectors.inventaire.Connector.load_edition_data"
-        ) as loader_mock, patch(
-            "bookwyrm.connectors.inventaire.Connector.get_book_data"
-        ) as getter_mock:
+        with (
+            patch(
+                "bookwyrm.connectors.inventaire.Connector.load_edition_data"
+            ) as loader_mock,
+            patch(
+                "bookwyrm.connectors.inventaire.Connector.get_book_data"
+            ) as getter_mock,
+        ):
             loader_mock.return_value = {"uris": ["hello"]}
             self.connector.get_edition_from_work_data(data)
         self.assertTrue(getter_mock.called)

bookwyrm/tests/connectors/test_openlibrary_connector.py

@@ -19,7 +19,7 @@ class Openlibrary(TestCase):
     """test loading data from openlibrary.org"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """creates the connector in the database"""
         models.Connector.objects.create(
             identifier="openlibrary.org",

bookwyrm/tests/data/ap_user_move.json

@@ -0,0 +1,42 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {
+            "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+            "schema": "http://schema.org#",
+            "PropertyValue": "schema:PropertyValue",
+            "value": "schema:value"
+        }
+    ],
+    "id": "https://example.com/user/mouse",
+    "type": "Person",
+    "preferredUsername": "mouse",
+    "name": "MOUSE?? MOUSE!!",
+    "inbox": "https://example.com/user/mouse/inbox",
+    "outbox": "https://example.com/user/mouse/outbox",
+    "followers": "https://example.com/user/mouse/followers",
+    "following": "https://example.com/user/mouse/following",
+    "summary": "",
+    "publicKey": {
+        "id": "https://example.com/user/mouse/#main-key",
+        "owner": "https://example.com/user/mouse",
+        "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC6QisDrjOQvkRo/MqNmSYPwqtt\nCxg/8rCW+9jKbFUKvqjTeKVotEE85122v/DCvobCCdfQuYIFdVMk+dB1xJ0iPGPg\nyU79QHY22NdV9mFKA2qtXVVxb5cxpA4PlwOHM6PM/k8B+H09OUrop2aPUAYwy+vg\n+MXyz8bAXrIS1kq6fQIDAQAB\n-----END PUBLIC KEY-----"
+    },
+    "endpoints": {
+        "sharedInbox": "https://example.com/inbox"
+    },
+    "bookwyrmUser": true,
+    "manuallyApprovesFollowers": false,
+    "discoverable": false,
+    "alsoKnownAs": [
+        "https://your.domain.here:4242/user/rat"
+    ],
+    "devices": "https://friend.camp/users/tripofmice/collections/devices",
+    "tag": [],
+    "icon": {
+        "type": "Image",
+        "mediaType": "image/png",
+        "url": "https://example.com/images/avatars/AL-2-crop-50.png"
+    }
+}

bookwyrm/tests/data/user_import.json

@@ -214,7 +214,7 @@
                     "attributedTo": "https://www.example.com//user/rat",
                     "content": "<p>I like it</p>",
                     "to": [
-                        "https://your.domain.here/user/rat/followers"
+                        "https://your.domain.here:4242/user/rat/followers"
                     ],
                     "cc": [],
                     "replies": {
@@ -395,7 +395,7 @@
         "https://local.lists/9999"
     ],
     "follows": [
-        "https://your.domain.here/user/rat"
+        "https://your.domain.here:4242/user/rat"
     ],
-    "blocks": ["https://your.domain.here/user/badger"]
+    "blocks": ["https://your.domain.here:4242/user/badger"]
 }

bookwyrm/tests/importers/test_calibre_import.py

@@ -9,7 +9,6 @@ from bookwyrm.importers import CalibreImporter
 from bookwyrm.models.import_job import handle_imported_book
 
 
-# pylint: disable=consider-using-with
 @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
 @patch("bookwyrm.activitystreams.populate_stream_task.delay")
 @patch("bookwyrm.activitystreams.add_book_statuses_task.delay")
@@ -20,20 +19,27 @@ class CalibreImport(TestCase):
         """use a test csv"""
         self.importer = CalibreImporter()
         datafile = pathlib.Path(__file__).parent.joinpath("../data/calibre.csv")
+        # pylint: disable-next=consider-using-with
         self.csv = open(datafile, "r", encoding=self.importer.encoding)
 
+    def tearDown(self):
+        """close test csv"""
+        self.csv.close()
+
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """populate database"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
         models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
-        self.book = models.Edition.objects.create(
+        cls.book = models.Edition.objects.create(
             title="Example Edition",
             remote_id="https://example.com/book/1",
             parent_work=work,

bookwyrm/tests/importers/test_goodreads_import.py

@@ -16,7 +16,6 @@ def make_date(*args):
     return datetime.datetime(*args, tzinfo=pytz.UTC)
 
 
-# pylint: disable=consider-using-with
 @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
 @patch("bookwyrm.activitystreams.populate_stream_task.delay")
 @patch("bookwyrm.activitystreams.add_book_statuses_task.delay")
@@ -27,20 +26,27 @@ class GoodreadsImport(TestCase):
         """use a test csv"""
         self.importer = GoodreadsImporter()
         datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv")
+        # pylint: disable-next=consider-using-with
         self.csv = open(datafile, "r", encoding=self.importer.encoding)
 
+    def tearDown(self):
+        """close test csv"""
+        self.csv.close()
+
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """populate database"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
         models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
-        self.book = models.Edition.objects.create(
+        cls.book = models.Edition.objects.create(
             title="Example Edition",
             remote_id="https://example.com/book/1",
             parent_work=work,

bookwyrm/tests/importers/test_importer.py

@@ -19,7 +19,6 @@ def make_date(*args):
     return datetime.datetime(*args, tzinfo=pytz.UTC)
 
 
-# pylint: disable=consider-using-with
 @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
 @patch("bookwyrm.activitystreams.populate_stream_task.delay")
 @patch("bookwyrm.activitystreams.add_book_statuses_task.delay")
@@ -30,20 +29,27 @@ class GenericImporter(TestCase):
         """use a test csv"""
         self.importer = Importer()
         datafile = pathlib.Path(__file__).parent.joinpath("../data/generic.csv")
+        # pylint: disable-next=consider-using-with
         self.csv = open(datafile, "r", encoding=self.importer.encoding)
 
+    def tearDown(self):
+        """close test csv"""
+        self.csv.close()
+
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """populate database"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
         models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
-        self.book = models.Edition.objects.create(
+        cls.book = models.Edition.objects.create(
             title="Example Edition",
             remote_id="https://example.com/book/1",
             parent_work=work,
@@ -266,9 +272,11 @@ class GenericImporter(TestCase):
         import_item.book = self.book
         import_item.save()
 
-        with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
-            with patch("bookwyrm.models.Status.broadcast") as broadcast_mock:
-                handle_imported_book(import_item)
+        with (
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.models.Status.broadcast") as broadcast_mock,
+        ):
+            handle_imported_book(import_item)
         kwargs = broadcast_mock.call_args.kwargs
         self.assertEqual(kwargs["software"], "bookwyrm")
         review = models.Review.objects.get(book=self.book, user=self.local_user)

bookwyrm/tests/importers/test_librarything_import.py

@@ -16,7 +16,6 @@ def make_date(*args):
     return datetime.datetime(*args, tzinfo=pytz.UTC)
 
 
-# pylint: disable=consider-using-with
 @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
 @patch("bookwyrm.activitystreams.populate_stream_task.delay")
 @patch("bookwyrm.activitystreams.add_book_statuses_task.delay")
@@ -29,20 +28,27 @@ class LibrarythingImport(TestCase):
         datafile = pathlib.Path(__file__).parent.joinpath("../data/librarything.tsv")
 
         # Librarything generates latin encoded exports...
+        # pylint: disable-next=consider-using-with
         self.csv = open(datafile, "r", encoding=self.importer.encoding)
 
+    def tearDown(self):
+        """close test csv"""
+        self.csv.close()
+
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """populate database"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mmai", "mmai@mmai.mmai", "password", local=True
             )
         models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
-        self.book = models.Edition.objects.create(
+        cls.book = models.Edition.objects.create(
             title="Example Edition",
             remote_id="https://example.com/book/1",
             parent_work=work,

bookwyrm/tests/importers/test_openlibrary_import.py

@@ -16,7 +16,6 @@ def make_date(*args):
     return datetime.datetime(*args, tzinfo=pytz.UTC)
 
 
-# pylint: disable=consider-using-with
 @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
 @patch("bookwyrm.activitystreams.populate_stream_task.delay")
 @patch("bookwyrm.activitystreams.add_book_statuses_task.delay")
@@ -27,20 +26,27 @@ class OpenLibraryImport(TestCase):
         """use a test csv"""
         self.importer = OpenLibraryImporter()
         datafile = pathlib.Path(__file__).parent.joinpath("../data/openlibrary.csv")
+        # pylint: disable-next=consider-using-with
         self.csv = open(datafile, "r", encoding=self.importer.encoding)
 
+    def tearDown(self):
+        """close test csv"""
+        self.csv.close()
+
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """populate database"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
         models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
-        self.book = models.Edition.objects.create(
+        cls.book = models.Edition.objects.create(
             title="Example Edition",
             remote_id="https://example.com/book/1",
             parent_work=work,

bookwyrm/tests/importers/test_storygraph_import.py

@@ -16,7 +16,6 @@ def make_date(*args):
     return datetime.datetime(*args, tzinfo=pytz.UTC)
 
 
-# pylint: disable=consider-using-with
 @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
 @patch("bookwyrm.activitystreams.populate_stream_task.delay")
 @patch("bookwyrm.activitystreams.add_book_statuses_task.delay")
@@ -27,20 +26,27 @@ class StorygraphImport(TestCase):
         """use a test csv"""
         self.importer = StorygraphImporter()
         datafile = pathlib.Path(__file__).parent.joinpath("../data/storygraph.csv")
+        # pylint: disable-next=consider-using-with
         self.csv = open(datafile, "r", encoding=self.importer.encoding)
 
+    def tearDown(self):
+        """close test csv"""
+        self.csv.close()
+
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """populate database"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
         models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
-        self.book = models.Edition.objects.create(
+        cls.book = models.Edition.objects.create(
             title="Example Edition",
             remote_id="https://example.com/book/1",
             parent_work=work,

bookwyrm/tests/lists_stream/test_signals.py

@@ -9,19 +9,21 @@ class ListsStreamSignals(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """database setup"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
-            self.another_user = models.User.objects.create_user(
+            cls.another_user = models.User.objects.create_user(
                 "fish", "fish@fish.fish", "password", local=True, localname="fish"
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",

bookwyrm/tests/lists_stream/test_stream.py

@@ -16,15 +16,17 @@ class ListsStream(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """database setup"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
-            self.another_user = models.User.objects.create_user(
+            cls.another_user = models.User.objects.create_user(
                 "nutria",
                 "nutria@nutria.nutria",
                 "password",
@@ -32,7 +34,7 @@ class ListsStream(TestCase):
                 localname="nutria",
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -41,7 +43,7 @@ class ListsStream(TestCase):
                 inbox="https://example.com/users/rat/inbox",
                 outbox="https://example.com/users/rat/outbox",
             )
-        self.stream = lists_stream.ListsStream()
+        cls.stream = lists_stream.ListsStream()
 
     def test_lists_stream_ids(self, *_):
         """the abstract base class for stream objects"""

bookwyrm/tests/lists_stream/test_tasks.py

@@ -11,15 +11,17 @@ class Activitystreams(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """database setup"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
-            self.another_user = models.User.objects.create_user(
+            cls.another_user = models.User.objects.create_user(
                 "nutria",
                 "nutria@nutria.nutria",
                 "password",
@@ -27,7 +29,7 @@ class Activitystreams(TestCase):
                 localname="nutria",
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -36,11 +38,12 @@ class Activitystreams(TestCase):
                 inbox="https://example.com/users/rat/inbox",
                 outbox="https://example.com/users/rat/outbox",
             )
-        with patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch("bookwyrm.lists_stream.remove_list_task.delay"):
-            self.list = models.List.objects.create(
-                user=self.local_user, name="hi", privacy="public"
+        with (
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.lists_stream.remove_list_task.delay"),
+        ):
+            cls.list = models.List.objects.create(
+                user=cls.local_user, name="hi", privacy="public"
             )
 
     def test_populate_lists_task(self, *_):

bookwyrm/tests/management/test_populate_lists_streams.py

@@ -13,15 +13,17 @@ class Activitystreams(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """we need some stuff"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
-            self.another_user = models.User.objects.create_user(
+            cls.another_user = models.User.objects.create_user(
                 "nutria",
                 "nutria@nutria.nutria",
                 "password",
@@ -37,7 +39,7 @@ class Activitystreams(TestCase):
                 is_active=False,
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -46,7 +48,7 @@ class Activitystreams(TestCase):
                 inbox="https://example.com/users/rat/inbox",
                 outbox="https://example.com/users/rat/outbox",
             )
-        self.book = models.Edition.objects.create(title="test book")
+        cls.book = models.Edition.objects.create(title="test book")
 
     def test_populate_streams(self, *_):
         """make sure the function on the redis manager gets called"""

bookwyrm/tests/management/test_populate_streams.py

@@ -11,15 +11,17 @@ class Activitystreams(TestCase):
     """using redis to build activity streams"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """we need some stuff"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse"
             )
-            self.another_user = models.User.objects.create_user(
+            cls.another_user = models.User.objects.create_user(
                 "nutria",
                 "nutria@nutria.nutria",
                 "password",
@@ -35,7 +37,7 @@ class Activitystreams(TestCase):
                 is_active=False,
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -44,7 +46,7 @@ class Activitystreams(TestCase):
                 inbox="https://example.com/users/rat/inbox",
                 outbox="https://example.com/users/rat/outbox",
             )
-        self.book = models.Edition.objects.create(title="test book")
+        cls.book = models.Edition.objects.create(title="test book")
 
     def test_populate_streams(self, _):
         """make sure the function on the redis manager gets called"""
@@ -53,11 +55,10 @@ class Activitystreams(TestCase):
                 user=self.local_user, content="hi", book=self.book
             )
 
-        with patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ) as redis_mock, patch(
-            "bookwyrm.lists_stream.populate_lists_task.delay"
-        ) as list_mock:
+        with (
+            patch("bookwyrm.activitystreams.populate_stream_task.delay") as redis_mock,
+            patch("bookwyrm.lists_stream.populate_lists_task.delay") as list_mock,
+        ):
             populate_streams()
         self.assertEqual(redis_mock.call_count, 6)  # 2 users x 3 streams
         self.assertEqual(list_mock.call_count, 2)  # 2 users

bookwyrm/tests/models/test_activitypub_mixin.py

@@ -27,18 +27,20 @@ class ActivitypubMixins(TestCase):
     """functionality shared across models"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """shared data"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.com", "mouseword", local=True, localname="mouse"
             )
-        self.local_user.remote_id = "http://example.com/a/b"
-        self.local_user.save(broadcast=False, update_fields=["remote_id"])
+        cls.local_user.remote_id = "http://example.com/a/b"
+        cls.local_user.save(broadcast=False, update_fields=["remote_id"])
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",

bookwyrm/tests/models/test_automod.py

@@ -15,12 +15,14 @@ class AutomodModel(TestCase):
     """every response to a get request, html or json"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """we need basic test data and mocks"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse@local.com",
                 "mouse@mouse.mouse",
                 "password",

bookwyrm/tests/models/test_base_model.py

@@ -5,7 +5,7 @@ from django.test import TestCase
 
 from bookwyrm import models
 from bookwyrm.models import base_model
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 
 
 # pylint: disable=attribute-defined-outside-init
@@ -13,16 +13,18 @@ class BaseModel(TestCase):
     """functionality shared across models"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """shared data"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.com", "mouseword", local=True, localname="mouse"
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -42,14 +44,14 @@ class BaseModel(TestCase):
         """these should be generated"""
         self.test_model.id = 1  # pylint: disable=invalid-name
         expected = self.test_model.get_remote_id()
-        self.assertEqual(expected, f"https://{DOMAIN}/bookwyrmtestmodel/1")
+        self.assertEqual(expected, f"{BASE_URL}/bookwyrmtestmodel/1")
 
     def test_remote_id_with_user(self):
         """format of remote id when there's a user object"""
         self.test_model.user = self.local_user
         self.test_model.id = 1
         expected = self.test_model.get_remote_id()
-        self.assertEqual(expected, f"https://{DOMAIN}/user/mouse/bookwyrmtestmodel/1")
+        self.assertEqual(expected, f"{BASE_URL}/user/mouse/bookwyrmtestmodel/1")
 
     def test_set_remote_id(self):
         """this function sets remote ids after creation"""
@@ -58,7 +60,7 @@ class BaseModel(TestCase):
         instance = models.Work.objects.create(title="work title")
         instance.remote_id = None
         base_model.set_remote_id(None, instance, True)
-        self.assertEqual(instance.remote_id, f"https://{DOMAIN}/book/{instance.id}")
+        self.assertEqual(instance.remote_id, f"{BASE_URL}/book/{instance.id}")
 
         # shouldn't set remote_id if it's not created
         instance.remote_id = None

bookwyrm/tests/models/test_book_model.py

@@ -1,12 +1,9 @@
 """ testing models """
-from io import BytesIO
 import pathlib
 
 import pytest
 
 from dateutil.parser import parse
-from PIL import Image
-from django.core.files.base import ContentFile
 from django.test import TestCase
 from django.utils import timezone
 
@@ -19,22 +16,22 @@ class Book(TestCase):
     """not too much going on in the books model but here we are"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """we'll need some books"""
-        self.work = models.Work.objects.create(
+        cls.work = models.Work.objects.create(
             title="Example Work", remote_id="https://example.com/book/1"
         )
-        self.first_edition = models.Edition.objects.create(
-            title="Example Edition", parent_work=self.work
+        cls.first_edition = models.Edition.objects.create(
+            title="Example Edition", parent_work=cls.work
         )
-        self.second_edition = models.Edition.objects.create(
+        cls.second_edition = models.Edition.objects.create(
             title="Another Example Edition",
-            parent_work=self.work,
+            parent_work=cls.work,
         )
 
     def test_remote_id(self):
         """fanciness with remote/origin ids"""
-        remote_id = f"https://{settings.DOMAIN}/book/{self.work.id}"
+        remote_id = f"{settings.BASE_URL}/book/{self.work.id}"
         self.assertEqual(self.work.get_remote_id(), remote_id)
         self.assertEqual(self.work.remote_id, remote_id)
 
@@ -130,15 +127,13 @@ class Book(TestCase):
     )
     def test_thumbnail_fields(self):
         """Just hit them"""
-        image_file = pathlib.Path(__file__).parent.joinpath(
+        image_path = pathlib.Path(__file__).parent.joinpath(
             "../../static/images/default_avi.jpg"
         )
-        image = Image.open(image_file)
-        output = BytesIO()
-        image.save(output, format=image.format)
 
         book = models.Edition.objects.create(title="hello")
-        book.cover.save("test.jpg", ContentFile(output.getvalue()))
+        with open(image_path, "rb") as image_file:
+            book.cover.save("test.jpg", image_file)
 
         self.assertIsNotNone(book.cover_bw_book_xsmall_webp.url)
         self.assertIsNotNone(book.cover_bw_book_xsmall_jpg.url)

bookwyrm/tests/models/test_bookwyrm_export_job.py

@@ -1,31 +1,30 @@
 """test bookwyrm user export functions"""
 import datetime
 import json
+import pathlib
+
 from unittest.mock import patch
 
-from django.core.serializers.json import DjangoJSONEncoder
-from django.test import TestCase
 from django.utils import timezone
+from django.test import TestCase
 
 from bookwyrm import models
-import bookwyrm.models.bookwyrm_export_job as export_job
+from bookwyrm.utils.tar import BookwyrmTarFile
 
 
-class BookwyrmExport(TestCase):
+class BookwyrmExportJob(TestCase):
     """testing user export functions"""
 
     def setUp(self):
         """lots of stuff to set up for a user export"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"), patch(
-            "bookwyrm.suggested_users.rerank_user_task.delay"
-        ), patch(
-            "bookwyrm.lists_stream.remove_list_task.delay"
-        ), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch(
-            "bookwyrm.activitystreams.add_book_statuses_task"
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+            patch("bookwyrm.suggested_users.rerank_user_task.delay"),
+            patch("bookwyrm.lists_stream.remove_list_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.activitystreams.add_book_statuses_task"),
         ):
 
             self.local_user = models.User.objects.create_user(
@@ -44,6 +43,11 @@ class BookwyrmExport(TestCase):
                 preferred_timezone="America/Los Angeles",
                 default_post_privacy="followers",
             )
+            avatar_path = pathlib.Path(__file__).parent.joinpath(
+                "../../static/images/default_avi.jpg"
+            )
+            with open(avatar_path, "rb") as avatar_file:
+                self.local_user.avatar.save("mouse-avatar.jpg", avatar_file)
 
             self.rat_user = models.User.objects.create_user(
                 "rat", "rat@rat.rat", "ratword", local=True, localname="rat"
@@ -89,6 +93,13 @@ class BookwyrmExport(TestCase):
                 title="Example Edition", parent_work=self.work
             )
 
+            # edition cover
+            cover_path = pathlib.Path(__file__).parent.joinpath(
+                "../../static/images/default_avi.jpg"
+            )
+            with open(cover_path, "rb") as cover_file:
+                self.edition.cover.save("tèst.jpg", cover_file)
+
             self.edition.authors.add(self.author)
 
             # readthrough
@@ -141,91 +152,105 @@ class BookwyrmExport(TestCase):
                 book=self.edition,
             )
 
-    def test_json_export_user_settings(self):
-        """Test the json export function for basic user info"""
-        data = export_job.json_export(self.local_user)
-        user_data = json.loads(data)
-        self.assertEqual(user_data["preferredUsername"], "mouse")
-        self.assertEqual(user_data["name"], "Mouse")
-        self.assertEqual(user_data["summary"], "<p>I'm a real bookmouse</p>")
-        self.assertEqual(user_data["manuallyApprovesFollowers"], False)
-        self.assertEqual(user_data["hideFollows"], False)
-        self.assertEqual(user_data["discoverable"], True)
-        self.assertEqual(user_data["settings"]["show_goal"], False)
-        self.assertEqual(user_data["settings"]["show_suggested_users"], False)
+            self.job = models.BookwyrmExportJob.objects.create(user=self.local_user)
+
+            # run the first stage of the export
+            with patch("bookwyrm.models.bookwyrm_export_job.create_archive_task.delay"):
+                models.bookwyrm_export_job.create_export_json_task(job_id=self.job.id)
+            self.job.refresh_from_db()
+
+    def test_add_book_to_user_export_job(self):
+        """does AddBookToUserExportJob ...add the book to the export?"""
+        self.assertIsNotNone(self.job.export_json["books"])
+        self.assertEqual(len(self.job.export_json["books"]), 1)
+        book = self.job.export_json["books"][0]
+
+        self.assertEqual(book["work"]["id"], self.work.remote_id)
+        self.assertEqual(len(book["authors"]), 1)
+        self.assertEqual(len(book["shelves"]), 1)
+        self.assertEqual(len(book["lists"]), 1)
+        self.assertEqual(len(book["comments"]), 1)
+        self.assertEqual(len(book["reviews"]), 1)
+        self.assertEqual(len(book["quotations"]), 1)
+        self.assertEqual(len(book["readthroughs"]), 1)
+
+        self.assertEqual(book["edition"]["id"], self.edition.remote_id)
         self.assertEqual(
-            user_data["settings"]["preferred_timezone"], "America/Los Angeles"
-        )
-        self.assertEqual(user_data["settings"]["default_post_privacy"], "followers")
-
-    def test_json_export_extended_user_data(self):
-        """Test the json export function for other non-book user info"""
-        data = export_job.json_export(self.local_user)
-        json_data = json.loads(data)
-
-        # goal
-        self.assertEqual(len(json_data["goals"]), 1)
-        self.assertEqual(json_data["goals"][0]["goal"], 128937123)
-        self.assertEqual(json_data["goals"][0]["year"], timezone.now().year)
-        self.assertEqual(json_data["goals"][0]["privacy"], "followers")
-
-        # saved lists
-        self.assertEqual(len(json_data["saved_lists"]), 1)
-        self.assertEqual(json_data["saved_lists"][0], "https://local.lists/9999")
-
-        # follows
-        self.assertEqual(len(json_data["follows"]), 1)
-        self.assertEqual(json_data["follows"][0], "https://your.domain.here/user/rat")
-        # blocked users
-        self.assertEqual(len(json_data["blocks"]), 1)
-        self.assertEqual(json_data["blocks"][0], "https://your.domain.here/user/badger")
-
-    def test_json_export_books(self):
-        """Test the json export function for extended user info"""
-
-        data = export_job.json_export(self.local_user)
-        json_data = json.loads(data)
-        start_date = json_data["books"][0]["readthroughs"][0]["start_date"]
-
-        self.assertEqual(len(json_data["books"]), 1)
-        self.assertEqual(json_data["books"][0]["edition"]["title"], "Example Edition")
-        self.assertEqual(len(json_data["books"][0]["authors"]), 1)
-        self.assertEqual(json_data["books"][0]["authors"][0]["name"], "Sam Zhu")
-
-        self.assertEqual(
-            f'"{start_date}"', DjangoJSONEncoder().encode(self.readthrough_start)
+            book["edition"]["cover"]["url"], f"images/{self.edition.cover.name}"
         )
 
-        self.assertEqual(json_data["books"][0]["shelves"][0]["name"], "Read")
+    def test_start_export_task(self):
+        """test saved list task saves initial json and data"""
+        self.assertIsNotNone(self.job.export_data)
+        self.assertIsNotNone(self.job.export_json)
+        self.assertEqual(self.job.export_json["name"], self.local_user.name)
 
-        self.assertEqual(len(json_data["books"][0]["lists"]), 1)
-        self.assertEqual(json_data["books"][0]["lists"][0]["name"], "My excellent list")
+    def test_export_saved_lists_task(self):
+        """test export_saved_lists_task adds the saved lists"""
+        self.assertIsNotNone(self.job.export_json["saved_lists"])
         self.assertEqual(
-            json_data["books"][0]["lists"][0]["list_item"]["book"],
-            self.edition.remote_id,
-            self.edition.id,
+            self.job.export_json["saved_lists"][0], self.saved_list.remote_id
         )
 
-        self.assertEqual(len(json_data["books"][0]["reviews"]), 1)
-        self.assertEqual(len(json_data["books"][0]["comments"]), 1)
-        self.assertEqual(len(json_data["books"][0]["quotations"]), 1)
+    def test_export_follows_task(self):
+        """test export_follows_task adds the follows"""
+        self.assertIsNotNone(self.job.export_json["follows"])
+        self.assertEqual(self.job.export_json["follows"][0], self.rat_user.remote_id)
 
-        self.assertEqual(json_data["books"][0]["reviews"][0]["name"], "my review")
-        self.assertEqual(
-            json_data["books"][0]["reviews"][0]["content"], "<p>awesome</p>"
-        )
-        self.assertEqual(json_data["books"][0]["reviews"][0]["rating"], 5.0)
+    def test_export_blocks_task(self):
+        """test export_blocks_task adds the blocks"""
+        self.assertIsNotNone(self.job.export_json["blocks"])
+        self.assertEqual(self.job.export_json["blocks"][0], self.badger_user.remote_id)
 
-        self.assertEqual(
-            json_data["books"][0]["comments"][0]["content"], "<p>ok so far</p>"
-        )
-        self.assertEqual(json_data["books"][0]["comments"][0]["progress"], 15)
-        self.assertEqual(json_data["books"][0]["comments"][0]["progress_mode"], "PG")
+    def test_export_reading_goals_task(self):
+        """test export_reading_goals_task adds the goals"""
+        self.assertIsNotNone(self.job.export_json["goals"])
+        self.assertEqual(self.job.export_json["goals"][0]["goal"], 128937123)
 
+    def test_json_export(self):
+        """test json_export job adds settings"""
+        self.assertIsNotNone(self.job.export_json["settings"])
+        self.assertFalse(self.job.export_json["settings"]["show_goal"])
         self.assertEqual(
-            json_data["books"][0]["quotations"][0]["content"], "<p>check this out</p>"
+            self.job.export_json["settings"]["preferred_timezone"],
+            "America/Los Angeles",
         )
         self.assertEqual(
-            json_data["books"][0]["quotations"][0]["quote"],
-            "<p>A rose by any other name</p>",
+            self.job.export_json["settings"]["default_post_privacy"], "followers"
         )
+        self.assertFalse(self.job.export_json["settings"]["show_suggested_users"])
+
+    def test_get_books_for_user(self):
+        """does get_books_for_user get all the books"""
+
+        data = models.bookwyrm_export_job.get_books_for_user(self.local_user)
+
+        self.assertEqual(len(data), 1)
+        self.assertEqual(data[0].title, "Example Edition")
+
+    def test_archive(self):
+        """actually create the TAR file"""
+        models.bookwyrm_export_job.create_archive_task(job_id=self.job.id)
+        self.job.refresh_from_db()
+
+        with (
+            self.job.export_data.open("rb") as tar_file,
+            BookwyrmTarFile.open(mode="r", fileobj=tar_file) as tar,
+        ):
+            archive_json_file = tar.extractfile("archive.json")
+            data = json.load(archive_json_file)
+
+            # JSON from the archive should be what we want it to be
+            self.assertEqual(data, self.job.export_json)
+
+            # User avatar should be present in archive
+            with self.local_user.avatar.open() as expected_avatar:
+                archive_avatar = tar.extractfile(data["icon"]["url"])
+                self.assertEqual(expected_avatar.read(), archive_avatar.read())
+
+            # Edition cover should be present in archive
+            with self.edition.cover.open() as expected_cover:
+                archive_cover = tar.extractfile(
+                    data["books"][0]["edition"]["cover"]["url"]
+                )
+                self.assertEqual(expected_cover.read(), archive_cover.read())

bookwyrm/tests/models/test_bookwyrm_import_job.py

@@ -18,12 +18,12 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
 
     def setUp(self):
         """setting stuff up"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"), patch(
-            "bookwyrm.suggested_users.rerank_user_task.delay"
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+            patch("bookwyrm.suggested_users.rerank_user_task.delay"),
         ):
-
             self.local_user = models.User.objects.create_user(
                 "mouse",
                 "mouse@mouse.mouse",
@@ -78,16 +78,18 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
     def test_update_user_profile(self):
         """Test update the user's profile from import data"""
 
-        with patch("bookwyrm.suggested_users.remove_user_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch("bookwyrm.suggested_users.rerank_user_task.delay"):
-
-            with open(self.archive_file, "rb") as fileobj:
-                with BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile:
-
-                    models.bookwyrm_import_job.update_user_profile(
-                        self.local_user, tarfile, self.json_data
-                    )
+        with (
+            patch("bookwyrm.suggested_users.remove_user_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.suggested_users.rerank_user_task.delay"),
+        ):
+            with (
+                open(self.archive_file, "rb") as fileobj,
+                BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile,
+            ):
+                models.bookwyrm_import_job.update_user_profile(
+                    self.local_user, tarfile, self.json_data
+                )
 
             self.local_user.refresh_from_db()
 
@@ -103,10 +105,11 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
     def test_update_user_settings(self):
         """Test updating the user's settings from import data"""
 
-        with patch("bookwyrm.suggested_users.remove_user_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch("bookwyrm.suggested_users.rerank_user_task.delay"):
-
+        with (
+            patch("bookwyrm.suggested_users.remove_user_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.suggested_users.rerank_user_task.delay"),
+        ):
             models.bookwyrm_import_job.update_user_settings(
                 self.local_user, self.json_data
             )
@@ -145,8 +148,9 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
     def test_upsert_saved_lists_existing(self):
         """Test upserting an existing saved list"""
 
-        with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        with (
+            patch("bookwyrm.lists_stream.remove_list_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
         ):
             book_list = models.List.objects.create(
                 name="My cool list",
@@ -172,8 +176,9 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
     def test_upsert_saved_lists_not_existing(self):
         """Test upserting a new saved list"""
 
-        with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        with (
+            patch("bookwyrm.lists_stream.remove_list_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
         ):
             book_list = models.List.objects.create(
                 name="My cool list",
@@ -199,9 +204,11 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
 
         self.assertFalse(before_follow)
 
-        with patch("bookwyrm.activitystreams.add_user_statuses_task.delay"), patch(
-            "bookwyrm.lists_stream.add_user_lists_task.delay"
-        ), patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
+        with (
+            patch("bookwyrm.activitystreams.add_user_statuses_task.delay"),
+            patch("bookwyrm.lists_stream.add_user_lists_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+        ):
             models.bookwyrm_import_job.upsert_follows(
                 self.local_user, self.json_data.get("follows")
             )
@@ -222,10 +229,11 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
         ).exists()
         self.assertFalse(blocked_before)
 
-        with patch("bookwyrm.suggested_users.remove_suggestion_task.delay"), patch(
-            "bookwyrm.activitystreams.remove_user_statuses_task.delay"
-        ), patch("bookwyrm.lists_stream.remove_user_lists_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        with (
+            patch("bookwyrm.suggested_users.remove_suggestion_task.delay"),
+            patch("bookwyrm.activitystreams.remove_user_statuses_task.delay"),
+            patch("bookwyrm.lists_stream.remove_user_lists_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
         ):
             models.bookwyrm_import_job.upsert_user_blocks(
                 self.local_user, self.json_data.get("blocks")
@@ -246,14 +254,15 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
 
         self.assertEqual(models.Edition.objects.count(), 1)
 
-        with open(self.archive_file, "rb") as fileobj:
-            with BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile:
+        with (
+            open(self.archive_file, "rb") as fileobj,
+            BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile,
+        ):
+            bookwyrm_import_job.get_or_create_edition(
+                self.json_data["books"][1], tarfile
+            )  # Sand Talk
 
-                bookwyrm_import_job.get_or_create_edition(
-                    self.json_data["books"][1], tarfile
-                )  # Sand Talk
-
-                self.assertEqual(models.Edition.objects.count(), 1)
+            self.assertEqual(models.Edition.objects.count(), 1)
 
     def test_get_or_create_edition_not_existing(self):
         """Test take a JSON string of books and editions,
@@ -262,12 +271,13 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
 
         self.assertEqual(models.Edition.objects.count(), 1)
 
-        with open(self.archive_file, "rb") as fileobj:
-            with BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile:
-
-                bookwyrm_import_job.get_or_create_edition(
-                    self.json_data["books"][0], tarfile
-                )  # Seeing like a state
+        with (
+            open(self.archive_file, "rb") as fileobj,
+            BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile,
+        ):
+            bookwyrm_import_job.get_or_create_edition(
+                self.json_data["books"][0], tarfile
+            )  # Seeing like a state
 
         self.assertTrue(models.Edition.objects.filter(isbn_13="9780300070163").exists())
         self.assertEqual(models.Edition.objects.count(), 2)
@@ -312,10 +322,10 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
 
         self.assertEqual(models.Review.objects.filter(user=self.local_user).count(), 0)
         reviews = self.json_data["books"][0]["reviews"]
-        with patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True):
-
+        with (
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True),
+        ):
             bookwyrm_import_job.upsert_statuses(
                 self.local_user, models.Review, reviews, self.book.remote_id
             )
@@ -349,10 +359,10 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
         self.assertEqual(models.Comment.objects.filter(user=self.local_user).count(), 0)
         comments = self.json_data["books"][1]["comments"]
 
-        with patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True):
-
+        with (
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True),
+        ):
             bookwyrm_import_job.upsert_statuses(
                 self.local_user, models.Comment, comments, self.book.remote_id
             )
@@ -378,9 +388,10 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
             models.Quotation.objects.filter(user=self.local_user).count(), 0
         )
         quotes = self.json_data["books"][1]["quotations"]
-        with patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True):
+        with (
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True),
+        ):
 
             bookwyrm_import_job.upsert_statuses(
                 self.local_user, models.Quotation, quotes, self.book.remote_id
@@ -411,9 +422,10 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
             models.Quotation.objects.filter(user=self.local_user).count(), 0
         )
         quotes = self.json_data["books"][1]["quotations"]
-        with patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=False):
+        with (
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=False),
+        ):
 
             bookwyrm_import_job.upsert_statuses(
                 self.local_user, models.Quotation, quotes, self.book.remote_id
@@ -432,8 +444,9 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
             title="Another Book", remote_id="https://example.com/book/9876"
         )
 
-        with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        with (
+            patch("bookwyrm.lists_stream.remove_list_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
         ):
             book_list = models.List.objects.create(
                 name="my list of books", user=self.local_user
@@ -452,8 +465,9 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
             1,
         )
 
-        with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        with (
+            patch("bookwyrm.lists_stream.remove_list_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
         ):
             bookwyrm_import_job.upsert_lists(
                 self.local_user,
@@ -479,8 +493,9 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
         self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 0)
         self.assertFalse(models.ListItem.objects.filter(book=self.book.id).exists())
 
-        with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        with (
+            patch("bookwyrm.lists_stream.remove_list_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
         ):
             bookwyrm_import_job.upsert_lists(
                 self.local_user,
@@ -503,16 +518,18 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
 
         shelf = models.Shelf.objects.get(name="Read", user=self.local_user)
 
-        with patch("bookwyrm.activitystreams.add_book_statuses_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        with (
+            patch("bookwyrm.activitystreams.add_book_statuses_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
         ):
             models.ShelfBook.objects.create(
                 book=self.book, shelf=shelf, user=self.local_user
             )
 
         book_data = self.json_data["books"][0]
-        with patch("bookwyrm.activitystreams.add_book_statuses_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        with (
+            patch("bookwyrm.activitystreams.add_book_statuses_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
         ):
             bookwyrm_import_job.upsert_shelves(self.book, self.local_user, book_data)
 
@@ -530,8 +547,9 @@ class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
 
         book_data = self.json_data["books"][0]
 
-        with patch("bookwyrm.activitystreams.add_book_statuses_task.delay"), patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        with (
+            patch("bookwyrm.activitystreams.add_book_statuses_task.delay"),
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
         ):
             bookwyrm_import_job.upsert_shelves(self.book, self.local_user, book_data)
 

bookwyrm/tests/models/test_fields.py

@@ -1,16 +1,14 @@
 """ testing models """
-from io import BytesIO
 from collections import namedtuple
 from dataclasses import dataclass
 import datetime
 import json
 import pathlib
-import re
+from urllib.parse import urlparse
 from typing import List
 from unittest import expectedFailure
 from unittest.mock import patch
 
-from PIL import Image
 import responses
 
 from django.core.exceptions import ValidationError
@@ -24,7 +22,7 @@ from bookwyrm.activitypub.base_activity import ActivityObject
 from bookwyrm.models import fields, User, Status, Edition
 from bookwyrm.models.base_model import BookWyrmModel
 from bookwyrm.models.activitypub_mixin import ActivitypubMixin
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import PROTOCOL, NETLOC
 
 # pylint: disable=too-many-public-methods
 @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
@@ -420,23 +418,19 @@ class ModelFields(TestCase):
         user = User.objects.create_user(
             "mouse", "mouse@mouse.mouse", "mouseword", local=True, localname="mouse"
         )
-        image_file = pathlib.Path(__file__).parent.joinpath(
+        image_path = pathlib.Path(__file__).parent.joinpath(
             "../../static/images/default_avi.jpg"
         )
-        image = Image.open(image_file)
-        output = BytesIO()
-        image.save(output, format=image.format)
-        user.avatar.save("test.jpg", ContentFile(output.getvalue()))
+        with open(image_path, "rb") as image_file:
+            user.avatar.save("test.jpg", image_file)
 
         instance = fields.ImageField()
 
         output = instance.field_to_activity(user.avatar)
-        self.assertIsNotNone(
-            re.match(
-                rf"https:\/\/{DOMAIN}\/.*\.jpg",
-                output.url,
-            )
-        )
+        parsed_url = urlparse(output.url)
+        self.assertEqual(parsed_url.scheme, PROTOCOL)
+        self.assertEqual(parsed_url.netloc, NETLOC)
+        self.assertRegex(parsed_url.path, r"\.jpg$")
         self.assertEqual(output.name, "")
         self.assertEqual(output.type, "Image")
 
@@ -516,30 +510,25 @@ class ModelFields(TestCase):
     @responses.activate
     def test_image_field_set_field_from_activity_no_overwrite_with_cover(self, *_):
         """update a model instance from an activitypub object"""
-        image_file = pathlib.Path(__file__).parent.joinpath(
+        image_path = pathlib.Path(__file__).parent.joinpath(
             "../../static/images/default_avi.jpg"
         )
-        image = Image.open(image_file)
-        output = BytesIO()
-        image.save(output, format=image.format)
-
-        another_image_file = pathlib.Path(__file__).parent.joinpath(
+        another_image_path = pathlib.Path(__file__).parent.joinpath(
             "../../static/images/logo.png"
         )
-        another_image = Image.open(another_image_file)
-        another_output = BytesIO()
-        another_image.save(another_output, format=another_image.format)
 
         instance = fields.ImageField(activitypub_field="cover", name="cover")
 
-        responses.add(
-            responses.GET,
-            "http://www.example.com/image.jpg",
-            body=another_image.tobytes(),
-            status=200,
-        )
+        with open(another_image_path, "rb") as another_image_file:
+            responses.add(
+                responses.GET,
+                "http://www.example.com/image.jpg",
+                body=another_image_file.read(),
+                status=200,
+            )
         book = Edition.objects.create(title="hello")
-        book.cover.save("test.jpg", ContentFile(output.getvalue()))
+        with open(image_path, "rb") as image_file:
+            book.cover.save("test.jpg", image_file)
         cover_size = book.cover.size
         self.assertIsNotNone(cover_size)
 
@@ -553,24 +542,22 @@ class ModelFields(TestCase):
     @responses.activate
     def test_image_field_set_field_from_activity_with_overwrite_with_cover(self, *_):
         """update a model instance from an activitypub object"""
-        image_file = pathlib.Path(__file__).parent.joinpath(
+        image_path = pathlib.Path(__file__).parent.joinpath(
             "../../static/images/default_avi.jpg"
         )
-        image = Image.open(image_file)
-        output = BytesIO()
-        image.save(output, format=image.format)
         book = Edition.objects.create(title="hello")
-        book.cover.save("test.jpg", ContentFile(output.getvalue()))
+        with open(image_path, "rb") as image_file:
+            book.cover.save("test.jpg", image_file)
         cover_size = book.cover.size
         self.assertIsNotNone(cover_size)
 
-        another_image_file = pathlib.Path(__file__).parent.joinpath(
+        another_image_path = pathlib.Path(__file__).parent.joinpath(
             "../../static/images/logo.png"
         )
 
         instance = fields.ImageField(activitypub_field="cover", name="cover")
 
-        with open(another_image_file, "rb") as another_image:
+        with open(another_image_path, "rb") as another_image:
             responses.add(
                 responses.GET,
                 "http://www.example.com/image.jpg",

bookwyrm/tests/models/test_group.py

@@ -10,21 +10,23 @@ class Group(TestCase):
     """some activitypub oddness ahead"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """Set up for tests"""
 
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.owner_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.owner_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "mouseword", local=True, localname="mouse"
             )
 
-            self.rat = models.User.objects.create_user(
+            cls.rat = models.User.objects.create_user(
                 "rat", "rat@rat.rat", "ratword", local=True, localname="rat"
             )
 
-            self.badger = models.User.objects.create_user(
+            cls.badger = models.User.objects.create_user(
                 "badger",
                 "badger@badger.badger",
                 "badgerword",
@@ -32,7 +34,7 @@ class Group(TestCase):
                 localname="badger",
             )
 
-            self.capybara = models.User.objects.create_user(
+            cls.capybara = models.User.objects.create_user(
                 "capybara",
                 "capybara@capybara.capybara",
                 "capybaraword",
@@ -40,32 +42,32 @@ class Group(TestCase):
                 localname="capybara",
             )
 
-        self.public_group = models.Group.objects.create(
+        cls.public_group = models.Group.objects.create(
             name="Public Group",
             description="Initial description",
-            user=self.owner_user,
+            user=cls.owner_user,
             privacy="public",
         )
 
-        self.private_group = models.Group.objects.create(
+        cls.private_group = models.Group.objects.create(
             name="Private Group",
             description="Top secret",
-            user=self.owner_user,
+            user=cls.owner_user,
             privacy="direct",
         )
 
-        self.followers_only_group = models.Group.objects.create(
+        cls.followers_only_group = models.Group.objects.create(
             name="Followers Group",
             description="No strangers",
-            user=self.owner_user,
+            user=cls.owner_user,
             privacy="followers",
         )
 
-        models.GroupMember.objects.create(group=self.private_group, user=self.badger)
+        models.GroupMember.objects.create(group=cls.private_group, user=cls.badger)
         models.GroupMember.objects.create(
-            group=self.followers_only_group, user=self.badger
+            group=cls.followers_only_group, user=cls.badger
         )
-        models.GroupMember.objects.create(group=self.public_group, user=self.capybara)
+        models.GroupMember.objects.create(group=cls.public_group, user=cls.capybara)
 
     def test_group_members_can_see_private_groups(self, _):
         """direct privacy group should not be excluded from group listings for group
@@ -81,9 +83,10 @@ class Group(TestCase):
         """follower-only group booklists should not be excluded from group booklist
         listing for group members who do not follower list owner"""
 
-        with patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch("bookwyrm.lists_stream.remove_list_task.delay"):
+        with (
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.lists_stream.remove_list_task.delay"),
+        ):
             followers_list = models.List.objects.create(
                 name="Followers List",
                 curation="group",
@@ -104,9 +107,10 @@ class Group(TestCase):
         """private group booklists should not be excluded from group booklist listing
         for group members"""
 
-        with patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch("bookwyrm.lists_stream.remove_list_task.delay"):
+        with (
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.lists_stream.remove_list_task.delay"),
+        ):
             private_list = models.List.objects.create(
                 name="Private List",
                 privacy="direct",

bookwyrm/tests/models/test_import_model.py

@@ -17,12 +17,14 @@ class ImportJob(TestCase):
     """this is a fancy one!!!"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """data is from a goodreads export of The Raven Tower"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
 
@@ -192,14 +194,16 @@ class ImportJob(TestCase):
             status=200,
         )
 
-        with patch("bookwyrm.connectors.abstract_connector.load_more_data.delay"):
-            with patch(
+        with (
+            patch("bookwyrm.connectors.abstract_connector.load_more_data.delay"),
+            patch(
                 "bookwyrm.connectors.connector_manager.first_search_result"
-            ) as search:
-                search.return_value = result
-                with patch(
-                    "bookwyrm.connectors.openlibrary.Connector.get_authors_from_data"
-                ):
-                    book = item.get_book_from_identifier()
+            ) as search,
+        ):
+            search.return_value = result
+            with patch(
+                "bookwyrm.connectors.openlibrary.Connector.get_authors_from_data"
+            ):
+                book = item.get_book_from_identifier()
 
         self.assertEqual(book.title, "Sabriel")

bookwyrm/tests/models/test_list.py

@@ -12,21 +12,23 @@ class List(TestCase):
     """some activitypub oddness ahead"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """look, a list"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "mouseword", local=True, localname="mouse"
             )
         work = models.Work.objects.create(title="hello")
-        self.book = models.Edition.objects.create(title="hi", parent_work=work)
+        cls.book = models.Edition.objects.create(title="hi", parent_work=work)
 
     def test_remote_id(self, *_):
         """shelves use custom remote ids"""
         book_list = models.List.objects.create(name="Test List", user=self.local_user)
-        expected_id = f"https://{settings.DOMAIN}/list/{book_list.id}"
+        expected_id = f"{settings.BASE_URL}/list/{book_list.id}"
         self.assertEqual(book_list.get_remote_id(), expected_id)
 
     def test_to_activity(self, *_):

bookwyrm/tests/models/test_move.py

@@ -0,0 +1,60 @@
+""" testing move models """
+from unittest.mock import patch
+from django.core.exceptions import PermissionDenied
+from django.test import TestCase
+
+from bookwyrm import models
+
+
+class MoveUser(TestCase):
+    """move your account to another identity"""
+
+    @classmethod
+    def setUpTestData(cls):
+        """we need some users for this"""
+        with patch("bookwyrm.models.user.set_remote_server.delay"):
+            cls.target_user = models.User.objects.create_user(
+                "rat",
+                "rat@rat.com",
+                "ratword",
+                local=False,
+                remote_id="https://example.com/users/rat",
+                inbox="https://example.com/users/rat/inbox",
+                outbox="https://example.com/users/rat/outbox",
+            )
+
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.origin_user = models.User.objects.create_user(
+                "mouse", "mouse@mouse.com", "mouseword", local=True, localname="mouse"
+            )
+        cls.origin_user.remote_id = "http://local.com/user/mouse"
+        cls.origin_user.save(broadcast=False, update_fields=["remote_id"])
+
+    def test_user_move_unauthorized(self):
+        """attempt a user move without alsoKnownAs set"""
+
+        with self.assertRaises(PermissionDenied):
+            models.MoveUser.objects.create(
+                user=self.origin_user,
+                object=self.origin_user.remote_id,
+                target=self.target_user,
+            )
+
+    @patch("bookwyrm.suggested_users.remove_user_task.delay")
+    @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async")
+    def test_user_move(self, *_):
+        """move user"""
+
+        self.target_user.also_known_as.add(self.origin_user.id)
+        self.target_user.save(broadcast=False)
+
+        models.MoveUser.objects.create(
+            user=self.origin_user,
+            object=self.origin_user.remote_id,
+            target=self.target_user,
+        )
+        self.assertEqual(self.origin_user.moved_to, self.target_user.remote_id)

bookwyrm/tests/models/test_notification.py

@@ -8,19 +8,21 @@ class Notification(TestCase):
     """let people know things"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """useful things for creating a notification"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "mouseword", local=True, localname="mouse"
             )
-            self.another_user = models.User.objects.create_user(
+            cls.another_user = models.User.objects.create_user(
                 "rat", "rat@rat.rat", "ratword", local=True, localname="rat"
             )
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -29,14 +31,14 @@ class Notification(TestCase):
                 inbox="https://example.com/users/rat/inbox",
                 outbox="https://example.com/users/rat/outbox",
             )
-        self.work = models.Work.objects.create(title="Test Work")
-        self.book = models.Edition.objects.create(
+        cls.work = models.Work.objects.create(title="Test Work")
+        cls.book = models.Edition.objects.create(
             title="Test Book",
             isbn_13="1234567890123",
             remote_id="https://example.com/book/1",
-            parent_work=self.work,
+            parent_work=cls.work,
         )
-        self.another_book = models.Edition.objects.create(
+        cls.another_book = models.Edition.objects.create(
             title="Second Test Book",
             parent_work=models.Work.objects.create(title="Test Work"),
         )
@@ -199,12 +201,14 @@ class NotifyInviteRequest(TestCase):
     """let admins know of invite requests"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """ensure there is one admin"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse@local.com",
                 "mouse@mouse.mouse",
                 "password",
@@ -264,9 +268,11 @@ class NotifyInviteRequest(TestCase):
 
     def test_notify_multiple_admins(self):
         """all admins are notified"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
             self.local_user = models.User.objects.create_user(
                 "admin@local.com",
                 "admin@example.com",

bookwyrm/tests/models/test_readthrough_model.py

@@ -12,18 +12,20 @@ class ReadThrough(TestCase):
     """some activitypub oddness ahead"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """look, a shelf"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "mouseword", local=True, localname="mouse"
             )
 
-        self.work = models.Work.objects.create(title="Example Work")
-        self.edition = models.Edition.objects.create(
-            title="Example Edition", parent_work=self.work
+        cls.work = models.Work.objects.create(title="Example Work")
+        cls.edition = models.Edition.objects.create(
+            title="Example Edition", parent_work=cls.work
         )
 
     def test_valid_date(self):

bookwyrm/tests/models/test_relationship_models.py

@@ -15,10 +15,10 @@ class Relationship(TestCase):
     """following, blocking, stuff like that"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """we need some users for this"""
         with patch("bookwyrm.models.user.set_remote_server.delay"):
-            self.remote_user = models.User.objects.create_user(
+            cls.remote_user = models.User.objects.create_user(
                 "rat",
                 "rat@rat.com",
                 "ratword",
@@ -27,14 +27,16 @@ class Relationship(TestCase):
                 inbox="https://example.com/users/rat/inbox",
                 outbox="https://example.com/users/rat/outbox",
             )
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.com", "mouseword", local=True, localname="mouse"
             )
-        self.local_user.remote_id = "http://local.com/user/mouse"
-        self.local_user.save(broadcast=False, update_fields=["remote_id"])
+        cls.local_user.remote_id = "http://local.com/user/mouse"
+        cls.local_user.save(broadcast=False, update_fields=["remote_id"])
 
     def test_user_follows(self, *_):
         """basic functionality of user follows"""

bookwyrm/tests/models/test_shelf_model.py

@@ -16,16 +16,18 @@ class Shelf(TestCase):
     """some activitypub oddness ahead"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """look, a shelf"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "mouseword", local=True, localname="mouse"
             )
         work = models.Work.objects.create(title="Test Work")
-        self.book = models.Edition.objects.create(title="test book", parent_work=work)
+        cls.book = models.Edition.objects.create(title="test book", parent_work=work)
 
     def test_remote_id(self, *_):
         """shelves use custom remote ids"""
@@ -33,7 +35,7 @@ class Shelf(TestCase):
             shelf = models.Shelf.objects.create(
                 name="Test Shelf", identifier="test-shelf", user=self.local_user
             )
-        expected_id = f"https://{settings.DOMAIN}/user/mouse/books/test-shelf"
+        expected_id = f"{settings.BASE_URL}/user/mouse/books/test-shelf"
         self.assertEqual(shelf.get_remote_id(), expected_id)
 
     def test_to_activity(self, *_):

bookwyrm/tests/models/test_site.py

@@ -13,12 +13,14 @@ class SiteModels(TestCase):
     """tests for site models"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(cls):
         """we need basic test data and mocks"""
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.local_user = models.User.objects.create_user(
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
                 "mouse@local.com",
                 "mouse@mouse.com",
                 "mouseword",
@@ -77,7 +79,7 @@ class SiteModels(TestCase):
     def test_site_invite_link(self):
         """invite link generator"""
         invite = models.SiteInvite.objects.create(user=self.local_user, code="hello")
-        self.assertEqual(invite.link, f"https://{settings.DOMAIN}/invite/hello")
+        self.assertEqual(invite.link, f"{settings.BASE_URL}/invite/hello")
 
     def test_invite_request(self):
         """someone wants an invite"""
@@ -93,7 +95,7 @@ class SiteModels(TestCase):
         """password reset token"""
         token = models.PasswordReset.objects.create(user=self.local_user, code="hello")
         self.assertTrue(token.valid())
-        self.assertEqual(token.link, f"https://{settings.DOMAIN}/password-reset/hello")
+        self.assertEqual(token.link, f"{settings.BASE_URL}/password-reset/hello")
 
     @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
     @patch("bookwyrm.suggested_users.remove_user_task.delay")