Merge branch 'main' into pre_commit

.env.example

@@ -16,6 +16,11 @@ DEFAULT_LANGUAGE="English"
 ## Leave unset to allow all hosts
 # ALLOWED_HOSTS="localhost,127.0.0.1,[::1]"
 
+# Specify when the site is served from a port that is not the default
+# for the protocol (80 for HTTP or 443 for HTTPS).
+# Probably only necessary in development.
+# PORT=1333
+
 MEDIA_ROOT=images/
 
 # Database configuration
@@ -71,14 +76,20 @@ ENABLE_THUMBNAIL_GENERATION=true
 USE_S3=false
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
+# seconds for signed S3 urls to expire
+# this is currently only used for user export files
+S3_SIGNED_URL_EXPIRY=900
 
 # Commented are example values if you use a non-AWS, S3-compatible service
 # AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
 # non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
-# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL.
+# AWS_S3_URL_PROTOCOL must end in ":" and defaults to the same protocol as
+# the BookWyrm instance ("http:" or "https:", based on USE_SSL).
 
 # AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
 # AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_URL_PROTOCOL=None        # "http:"
 # AWS_S3_REGION_NAME=None         # "fr-par"
 # AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
 
@@ -133,9 +144,9 @@ HTTP_X_FORWARDED_PROTO=false
 TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2
 TWO_FACTOR_LOGIN_MAX_SECONDS=60
 
-# Additional hosts to allow in the Content-Security-Policy, "self" (should be DOMAIN)
-# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
-# Value should be a comma-separated list of host names.
+# Additional hosts to allow in the Content-Security-Policy, "self" (should be
+# DOMAIN with optionally ":" + PORT) and AWS_S3_CUSTOM_DOMAIN (if used) are
+# added by default.  Value should be a comma-separated list of host names.
 CSP_ADDITIONAL_HOSTS=
 
 # Time before being logged out (in seconds)

.github/pull_request_template.md

@@ -0,0 +1,68 @@
+<!--
+Thanks for contributing! This template has some checkboxes that help keep track of what changes go into a release.
+
+To check (tick) a list item, replace the space between square brackets with an x, like this:
+
+- [x] I have checked the box
+
+You can find more information and tips for BookWyrm contributors at https://docs.joinbookwyrm.com/contributing.html
+-->
+## Description
+<!--
+Describe what your pull request does here
+-->
+
+
+<!--
+For pull requests that relate or close an issue, please include them
+below.  We like to follow [Github's guidance on linking issues to pull requests](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue).
+
+For example having the text: "closes #1234" would connect the current pull
+request to issue 1234.  And when we merge the pull request, Github will
+automatically close the issue.
+-->
+
+- Related Issue #
+- Closes #
+
+## What type of Pull Request is this?
+<!-- Check all that apply -->
+
+- [ ] Bug Fix
+- [ ] Enhancement
+- [ ] Plumbing / Internals / Dependencies
+- [ ] Refactor
+
+## Does this PR change settings or dependencies, or break something?
+<!-- Check all that apply -->
+
+- [ ] This PR changes or adds default settings, configuration, or .env values
+- [ ] This PR changes or adds dependencies
+- [ ] This PR introduces other breaking changes
+
+### Details of breaking or configuration changes (if any of above checked)
+
+
+## Documentation
+<!--
+Documentation for users, admins, and developers is an important way to keep the BookWyrm community welcoming and make Bookwyrm easy to use.
+Our documentation is maintained in a separate repository at https://github.com/bookwyrm-social/documentation
+-->
+
+<!-- Check all that apply -->
+
+- [ ] New or amended documentation will be required if this PR is merged
+- [ ] I have created a matching pull request in the Documentation repository
+- [ ] I intend to create a matching pull request in the Documentation repository after this PR is merged
+
+<!-- Amazing! Thanks for filling that out. Your PR will need to have passing tests and happy linters before we can merge
+You will need to check your code with `black`, `pylint`, and `mypy`, or `./bw-dev formatters`
+-->
+
+### Tests
+<!-- Check one -->
+
+- [ ] My changes do not need new tests
+- [ ] All tests I have added are passing
+- [ ] I have written tests but need help to make them pass
+- [ ] I have not written tests and need help to write them

.github/release.yml

@@ -0,0 +1,26 @@
+changelog:
+  exclude:
+    labels:
+      - ignore-for-release
+  categories:
+    - title: ‼️ Breaking Changes & New Settings ⚙️
+      labels:
+        - breaking-change
+        - config-change
+    - title: Updated Dependencies 🧸
+      labels:
+        - dependencies
+    - title: New Features 🎉
+      labels:
+        - enhancement
+    - title: Bug Fixes 🐛
+      labels:
+        - fix
+        - bug
+    - title: Internals/Plumbing 👩‍🔧
+        - plumbing
+        - tests
+        - deployment
+    - title: Other Changes
+      labels:
+        - "*"

.github/workflows/codeql-analysis.yml

@@ -40,7 +40,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,7 +51,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -65,4 +65,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/lint-frontend.yaml

@@ -22,7 +22,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install modules
-        run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
+      #  run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
+        run: npm install eslint@^8.9.0
 
       # See .stylelintignore for files that are not linted.
       # - name: Run stylelint

.github/workflows/python.yml

@@ -48,7 +48,7 @@ jobs:
     - name: Set up .env
       run: cp .env.example .env
     - name: Check migrations up-to-date
-      run: python ./manage.py makemigrations --check
+      run: python ./manage.py makemigrations --check -v 3
     - name: Run Tests
       run: pytest -n 3
 

.gitignore

@@ -16,6 +16,7 @@
 # BookWyrm
 .env
 /images/
+/exports/
 /static/
 bookwyrm/static/css/bookwyrm.css
 bookwyrm/static/css/themes/
@@ -37,3 +38,6 @@ nginx/default.conf
 
 #macOS
 **/.DS_Store
+
+# Docker
+docker-compose.override.yml

.pylintrc

@@ -3,7 +3,19 @@ ignore=migrations
 load-plugins=pylint.extensions.no_self_use
 
 [MESSAGES CONTROL]
-disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801,C3001,import-error
+disable =
+    cyclic-import,
+    duplicate-code,
+    fixme,
+    no-member,
+    raise-missing-from,
+    too-few-public-methods,
+    too-many-ancestors,
+    too-many-instance-attributes,
+    unnecessary-lambda-assignment,
+    unsubscriptable-object,
+enable =
+    useless-suppression
 
 [FORMAT]
 max-line-length=88

README.md

@@ -10,7 +10,6 @@ BookWyrm is a social network for tracking your reading, talking about books, wri
 ## Links
 
 [![Mastodon Follow](https://img.shields.io/mastodon/follow/000146121?domain=https%3A%2F%2Ftech.lgbt&style=social)](https://tech.lgbt/@bookwyrm)
-[![Twitter Follow](https://img.shields.io/twitter/follow/BookWyrmSocial?style=social)](https://twitter.com/BookWyrmSocial)
 
  - [Project homepage](https://joinbookwyrm.com/)
  - [Support](https://patreon.com/bookwyrm)

bookwyrm/activitypub/base_activity.py

@@ -250,7 +250,10 @@ class ActivityObject:
                 pass
         data = {k: v for (k, v) in data.items() if v is not None and k not in omit}
         if "@context" not in omit:
-            data["@context"] = "https://www.w3.org/ns/activitystreams"
+            data["@context"] = [
+                "https://www.w3.org/ns/activitystreams",
+                {"Hashtag": "as:Hashtag"},
+            ]
         return data
 
 
@@ -400,11 +403,11 @@ def get_representative():
     to sign outgoing HTTP GET requests"""
     return models.User.objects.get_or_create(
         username=f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}",
-        defaults=dict(
-            email="bookwyrm@localhost",
-            local=True,
-            localname=INSTANCE_ACTOR_USERNAME,
-        ),
+        defaults={
+            "email": "bookwyrm@localhost",
+            "local": True,
+            "localname": INSTANCE_ACTOR_USERNAME,
+        },
     )[0]
 
 

bookwyrm/activitypub/book.py

@@ -67,7 +67,6 @@ class Edition(Book):
     type: str = "Edition"
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Work(Book):
     """work instance of a book object"""

bookwyrm/activitypub/ordered_collection.py

@@ -18,7 +18,6 @@ class OrderedCollection(ActivityObject):
     type: str = "OrderedCollection"
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class OrderedCollectionPrivate(OrderedCollection):
     """an ordered collection with privacy settings"""

bookwyrm/activitypub/verbs.py

@@ -22,7 +22,6 @@ class Verb(ActivityObject):
             self.object.to_model(allow_external_connections=allow_external_connections)
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Create(Verb):
     """Create activity"""
@@ -33,7 +32,6 @@ class Create(Verb):
     type: str = "Create"
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Delete(Verb):
     """Create activity"""
@@ -63,7 +61,6 @@ class Delete(Verb):
         # if we can't find it, we don't need to delete it because we don't have it
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Update(Verb):
     """Update activity"""
@@ -227,7 +224,6 @@ class Like(Verb):
         self.to_model(allow_external_connections=allow_external_connections)
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Announce(Verb):
     """boosting a status"""

bookwyrm/activitystreams.py

@@ -32,7 +32,7 @@ class ActivityStream(RedisStore):
         stream_id = self.stream_id(user_id)
         return f"{stream_id}-unread-by-type"
 
-    def get_rank(self, obj):  # pylint: disable=no-self-use
+    def get_rank(self, obj):
         """statuses are sorted by date published"""
         return obj.published_date.timestamp()
 
@@ -139,14 +139,14 @@ class ActivityStream(RedisStore):
                 | (
                     Q(following=status.user) & Q(following=status.reply_parent.user)
                 )  # if the user is following both authors
-            ).distinct()
+            )
 
         # only visible to the poster's followers and tagged users
         elif status.privacy == "followers":
             audience = audience.filter(
                 Q(following=status.user)  # if the user is following the author
             )
-        return audience.distinct()
+        return audience.distinct("id")
 
     @tracer.start_as_current_span("ActivityStream.get_audience")
     def get_audience(self, status):
@@ -156,7 +156,7 @@ class ActivityStream(RedisStore):
         status_author = models.User.objects.filter(
             is_active=True, local=True, id=status.user.id
         ).values_list("id", flat=True)
-        return list(set(list(audience) + list(status_author)))
+        return list(set(audience) | set(status_author))
 
     def get_stores_for_users(self, user_ids):
         """convert a list of user ids into redis store ids"""
@@ -183,15 +183,13 @@ class HomeStream(ActivityStream):
     def get_audience(self, status):
         trace.get_current_span().set_attribute("stream_id", self.key)
         audience = super()._get_audience(status)
-        if not audience:
-            return []
         # if the user is following the author
         audience = audience.filter(following=status.user).values_list("id", flat=True)
         # if the user is the post's author
         status_author = models.User.objects.filter(
             is_active=True, local=True, id=status.user.id
         ).values_list("id", flat=True)
-        return list(set(list(audience) + list(status_author)))
+        return list(set(audience) | set(status_author))
 
     def get_statuses_for_user(self, user):
         return models.Status.privacy_filter(
@@ -239,9 +237,7 @@ class BooksStream(ActivityStream):
         )
 
         audience = super()._get_audience(status)
-        if not audience:
-            return models.User.objects.none()
-        return audience.filter(shelfbook__book__parent_work=work).distinct()
+        return audience.filter(shelfbook__book__parent_work=work)
 
     def get_audience(self, status):
         # only show public statuses on the books feed,

bookwyrm/apps.py

@@ -33,7 +33,6 @@ class BookwyrmConfig(AppConfig):
     name = "bookwyrm"
     verbose_name = "BookWyrm"
 
-    # pylint: disable=no-self-use
     def ready(self):
         """set up OTLP and preview image files, if desired"""
         if settings.OTEL_EXPORTER_OTLP_ENDPOINT or settings.OTEL_EXPORTER_CONSOLE:

bookwyrm/book_search.py

@@ -36,7 +36,6 @@ def search(
     ...
 
 
-# pylint: disable=arguments-differ
 def search(
     query: str,
     *,

bookwyrm/connectors/connector_manager.py

@@ -118,9 +118,11 @@ def get_connectors() -> Iterator[abstract_connector.AbstractConnector]:
 def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnector:
     """get the connector related to the object's server"""
     url = urlparse(remote_id)
-    identifier = url.netloc
+    identifier = url.hostname
     if not identifier:
-        raise ValueError("Invalid remote id")
+        raise ValueError(f"Invalid remote id: {remote_id}")
+
+    base_url = f"{url.scheme}://{url.netloc}"
 
     try:
         connector_info = models.Connector.objects.get(identifier=identifier)
@@ -128,10 +130,10 @@ def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnec
         connector_info = models.Connector.objects.create(
             identifier=identifier,
             connector_file="bookwyrm_connector",
-            base_url=f"https://{identifier}",
-            books_url=f"https://{identifier}/book",
-            covers_url=f"https://{identifier}/images/covers",
-            search_url=f"https://{identifier}/search?q=",
+            base_url=base_url,
+            books_url=f"{base_url}/book",
+            covers_url=f"{base_url}/images/covers",
+            search_url=f"{base_url}/search?q=",
             priority=2,
         )
 
@@ -143,7 +145,9 @@ def load_more_data(connector_id: str, book_id: str) -> None:
     """background the work of getting all 10,000 editions of LoTR"""
     connector_info = models.Connector.objects.get(id=connector_id)
     connector = load_connector(connector_info)
-    book = models.Book.objects.select_subclasses().get(id=book_id)
+    book = models.Book.objects.select_subclasses().get(  # type: ignore[no-untyped-call]
+        id=book_id
+    )
     connector.expand_book_data(book)
 
 
@@ -154,7 +158,9 @@ def create_edition_task(
     """separate task for each of the 10,000 editions of LoTR"""
     connector_info = models.Connector.objects.get(id=connector_id)
     connector = load_connector(connector_info)
-    work = models.Work.objects.select_subclasses().get(id=work_id)
+    work = models.Work.objects.select_subclasses().get(  # type: ignore[no-untyped-call]
+        id=work_id
+    )
     connector.create_edition_from_data(work, data)
 
 
@@ -188,8 +194,11 @@ def raise_not_valid_url(url: str) -> None:
     if not parsed.scheme in ["http", "https"]:
         raise ConnectorException("Invalid scheme: ", url)
 
+    if not parsed.hostname:
+        raise ConnectorException("Hostname missing: ", url)
+
     try:
-        ipaddress.ip_address(parsed.netloc)
+        ipaddress.ip_address(parsed.hostname)
         raise ConnectorException("Provided url is an IP address: ", url)
     except ValueError:
         # it's not an IP address, which is good

bookwyrm/connectors/inventaire.py

@@ -229,7 +229,7 @@ class Connector(AbstractConnector):
             data = get_data(url)
         except ConnectorException:
             return ""
-        return data.get("extract", "")
+        return str(data.get("extract", ""))
 
     def get_remote_id_from_model(self, obj: models.BookDataModel) -> str:
         """use get_remote_id to figure out the link from a model obj"""

bookwyrm/context_processors.py

@@ -2,7 +2,7 @@
 from bookwyrm import models, settings
 
 
-def site_settings(request):  # pylint: disable=unused-argument
+def site_settings(request):
     """include the custom info about the site"""
     request_protocol = "https://"
     if not request.is_secure():

bookwyrm/emailing.py

@@ -4,7 +4,7 @@ from django.template.loader import get_template
 
 from bookwyrm import models, settings
 from bookwyrm.tasks import app, EMAIL
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import DOMAIN, BASE_URL
 
 
 def email_data():
@@ -14,6 +14,7 @@ def email_data():
         "site_name": site.name,
         "logo": site.logo_small_url,
         "domain": DOMAIN,
+        "base_url": BASE_URL,
         "user": None,
     }
 

bookwyrm/forms/custom_form.py

@@ -15,9 +15,9 @@ class StyledForm(ModelForm):
         css_classes["number"] = "input"
         css_classes["checkbox"] = "checkbox"
         css_classes["textarea"] = "textarea"
-        # pylint: disable=super-with-arguments
         super().__init__(*args, **kwargs)
         for visible in self.visible_fields():
+            input_type = ""
             if hasattr(visible.field.widget, "input_type"):
                 input_type = visible.field.widget.input_type
             if isinstance(visible.field.widget, Textarea):

bookwyrm/forms/edit_user.py

@@ -18,6 +18,7 @@ class EditUserForm(CustomForm):
             "email",
             "summary",
             "show_goal",
+            "show_ratings",
             "show_suggested_users",
             "manually_approves_followers",
             "default_post_privacy",

bookwyrm/forms/landing.py

@@ -34,7 +34,6 @@ class LoginForm(CustomForm):
 
     def add_invalid_password_error(self):
         """We don't want to be too specific about this"""
-        # pylint: disable=attribute-defined-outside-init
         self.non_field_errors = _("Username or password are incorrect")
 
 

bookwyrm/forms/links.py

@@ -26,7 +26,7 @@ class FileLinkForm(CustomForm):
         url = cleaned_data.get("url")
         filetype = cleaned_data.get("filetype")
         book = cleaned_data.get("book")
-        domain = urlparse(url).netloc
+        domain = urlparse(url).hostname
         if models.LinkDomain.objects.filter(domain=domain).exists():
             status = models.LinkDomain.objects.get(domain=domain).status
             if status == "blocked":

bookwyrm/forms/widgets.py

@@ -5,8 +5,6 @@ from django import forms
 class ArrayWidget(forms.widgets.TextInput):
     """Inputs for postgres array fields"""
 
-    # pylint: disable=unused-argument
-    # pylint: disable=no-self-use
     def value_from_datadict(self, data, files, name):
         """get all values for this name"""
         return [i for i in data.getlist(name) if i]

bookwyrm/importers/__init__.py

@@ -1,7 +1,7 @@
 """ import classes """
 
 from .importer import Importer
-from .bookwyrm_import import BookwyrmImporter
+from .bookwyrm_import import BookwyrmImporter, BookwyrmBooksImporter
 from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter

bookwyrm/importers/bookwyrm_import.py

@@ -3,6 +3,7 @@ from django.http import QueryDict
 
 from bookwyrm.models import User
 from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
+from . import Importer
 
 
 class BookwyrmImporter:
@@ -22,3 +23,17 @@ class BookwyrmImporter:
             user=user, archive_file=archive_file, required=required
         )
         return job
+
+
+class BookwyrmBooksImporter(Importer):
+    """
+    Handle reading a csv from BookWyrm.
+    Goodreads is the default importer, we basically just use the same structure
+    But BookWyrm has additional attributes in the csv
+    """
+
+    service = "BookWyrm"
+    row_mappings_guesses = Importer.row_mappings_guesses + [
+        ("shelf_name", ["shelf_name"]),
+        ("review_published", ["review_published"]),
+    ]

bookwyrm/importers/calibre_import.py

@@ -14,15 +14,10 @@ class CalibreImporter(Importer):
     def __init__(self, *args: Any, **kwargs: Any):
         # Add timestamp to row_mappings_guesses for date_added to avoid
         # integrity error
-        row_mappings_guesses = []
-
-        for field, mapping in self.row_mappings_guesses:
-            if field in ("date_added",):
-                row_mappings_guesses.append((field, mapping + ["timestamp"]))
-            else:
-                row_mappings_guesses.append((field, mapping))
-
-        self.row_mappings_guesses = row_mappings_guesses
+        self.row_mappings_guesses = [
+            (field, mapping + (["timestamp"] if field == "date_added" else []))
+            for field, mapping in self.row_mappings_guesses
+        ]
         super().__init__(*args, **kwargs)
 
     def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:

bookwyrm/importers/importer.py

@@ -18,17 +18,26 @@ class Importer:
     row_mappings_guesses = [
         ("id", ["id", "book id"]),
         ("title", ["title"]),
-        ("authors", ["author", "authors", "primary author"]),
-        ("isbn_10", ["isbn10", "isbn", "isbn/uid"]),
-        ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]),
+        ("authors", ["author_text", "author", "authors", "primary author"]),
+        ("isbn_10", ["isbn_10", "isbn10", "isbn", "isbn/uid"]),
+        ("isbn_13", ["isbn_13", "isbn13", "isbn", "isbns", "isbn/uid"]),
         ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]),
-        ("review_name", ["review name"]),
-        ("review_body", ["my review", "review"]),
+        ("review_name", ["review_name", "review name"]),
+        ("review_body", ["review_content", "my review", "review"]),
         ("rating", ["my rating", "rating", "star rating"]),
-        ("date_added", ["date added", "entry date", "added"]),
-        ("date_started", ["date started", "started"]),
-        ("date_finished", ["date finished", "last date read", "date read", "finished"]),
+        (
+            "date_added",
+            ["shelf_date", "date_added", "date added", "entry date", "added"],
+        ),
+        ("date_started", ["start_date", "date started", "started"]),
+        (
+            "date_finished",
+            ["finish_date", "date finished", "last date read", "date read", "finished"],
+        ),
     ]
+
+    # TODO: stopped
+
     date_fields = ["date_added", "date_started", "date_finished"]
     shelf_mapping_guesses = {
         "to-read": ["to-read", "want to read"],
@@ -36,9 +45,14 @@ class Importer:
         "reading": ["currently-reading", "reading", "currently reading"],
     }
 
-    # pylint: disable=too-many-locals
+    # pylint: disable=too-many-arguments
     def create_job(
-        self, user: User, csv_file: Iterable[str], include_reviews: bool, privacy: str
+        self,
+        user: User,
+        csv_file: Iterable[str],
+        include_reviews: bool,
+        privacy: str,
+        create_shelves: bool = True,
     ) -> ImportJob:
         """check over a csv and creates a database entry for the job"""
         csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
@@ -55,6 +69,7 @@ class Importer:
         job = ImportJob.objects.create(
             user=user,
             include_reviews=include_reviews,
+            create_shelves=create_shelves,
             privacy=privacy,
             mappings=mappings,
             source=self.service,
@@ -114,7 +129,7 @@ class Importer:
         shelf = [
             s for (s, gs) in self.shelf_mapping_guesses.items() if shelf_name in gs
         ]
-        return shelf[0] if shelf else None
+        return shelf[0] if shelf else normalized_row.get("shelf") or None
 
     # pylint: disable=no-self-use
     def normalize_row(
@@ -149,6 +164,7 @@ class Importer:
         job = ImportJob.objects.create(
             user=user,
             include_reviews=original_job.include_reviews,
+            create_shelves=original_job.create_shelves,
             privacy=original_job.privacy,
             source=original_job.source,
             # TODO: allow users to adjust mappings

bookwyrm/importers/librarything_import.py

@@ -20,7 +20,7 @@ class LibrarythingImporter(Importer):
 
     def normalize_row(
         self, entry: dict[str, str], mappings: dict[str, Optional[str]]
-    ) -> dict[str, Optional[str]]:  # pylint: disable=no-self-use
+    ) -> dict[str, Optional[str]]:
         """use the dataclass to create the formatted row of data"""
         normalized = {
             k: _remove_brackets(entry.get(v) if v else None)

bookwyrm/lists_stream.py

@@ -18,7 +18,7 @@ class ListsStream(RedisStore):
             return f"{user}-lists"
         return f"{user.id}-lists"
 
-    def get_rank(self, obj):  # pylint: disable=no-self-use
+    def get_rank(self, obj):
         """lists are sorted by updated date"""
         return obj.updated_date.timestamp()
 

bookwyrm/management/commands/deduplicate_book_data.py

@@ -1,13 +1,14 @@
 """ PROCEED WITH CAUTION: uses deduplication fields to permanently
 merge book data objects """
+
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from bookwyrm import models
-from bookwyrm.management.merge import merge_objects
 
 
-def dedupe_model(model):
+def dedupe_model(model, dry_run=False):
     """combine duplicate editions and update related models"""
+    print(f"deduplicating {model.__name__}:")
     fields = model._meta.get_fields()
     dedupe_fields = [
         f for f in fields if hasattr(f, "deduplication_field") and f.deduplication_field
@@ -16,30 +17,42 @@ def dedupe_model(model):
         dupes = (
             model.objects.values(field.name)
             .annotate(Count(field.name))
-            .filter(**{"%s__count__gt" % field.name: 1})
+            .filter(**{f"{field.name}__count__gt": 1})
+            .exclude(**{field.name: ""})
+            .exclude(**{f"{field.name}__isnull": True})
         )
 
         for dupe in dupes:
             value = dupe[field.name]
-            if not value or value == "":
-                continue
             print("----------")
-            print(dupe)
             objs = model.objects.filter(**{field.name: value}).order_by("id")
             canonical = objs.first()
-            print("keeping", canonical.remote_id)
+            action = "would merge" if dry_run else "merging"
+            print(
+                f"{action} into {model.__name__} {canonical.remote_id} based on {field.name} {value}:"
+            )
             for obj in objs[1:]:
-                print(obj.remote_id)
-                merge_objects(canonical, obj)
+                print(f"- {obj.remote_id}")
+                absorbed_fields = obj.merge_into(canonical, dry_run=dry_run)
+                print(f"  absorbed fields: {absorbed_fields}")
 
 
 class Command(BaseCommand):
     """deduplicate allllll the book data models"""
 
     help = "merges duplicate book data"
+
+    def add_arguments(self, parser):
+        """add the arguments for this command"""
+        parser.add_argument(
+            "--dry_run",
+            action="store_true",
+            help="don't actually merge, only print what would happen",
+        )
+
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
         """run deduplications"""
-        dedupe_model(models.Edition)
-        dedupe_model(models.Work)
-        dedupe_model(models.Author)
+        dedupe_model(models.Edition, dry_run=options["dry_run"])
+        dedupe_model(models.Work, dry_run=options["dry_run"])
+        dedupe_model(models.Author, dry_run=options["dry_run"])

bookwyrm/management/merge.py

@@ -1,50 +0,0 @@
-from django.db.models import ManyToManyField
-
-
-def update_related(canonical, obj):
-    """update all the models with fk to the object being removed"""
-    # move related models to canonical
-    related_models = [
-        (r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
-    ]
-    for (related_field, related_model) in related_models:
-        # Skip the ManyToMany fields that aren’t auto-created. These
-        # should have a corresponding OneToMany field in the model for
-        # the linking table anyway. If we update it through that model
-        # instead then we won’t lose the extra fields in the linking
-        # table.
-        related_field_obj = related_model._meta.get_field(related_field)
-        if isinstance(related_field_obj, ManyToManyField):
-            through = related_field_obj.remote_field.through
-            if not through._meta.auto_created:
-                continue
-        related_objs = related_model.objects.filter(**{related_field: obj})
-        for related_obj in related_objs:
-            print("replacing in", related_model.__name__, related_field, related_obj.id)
-            try:
-                setattr(related_obj, related_field, canonical)
-                related_obj.save()
-            except TypeError:
-                getattr(related_obj, related_field).add(canonical)
-                getattr(related_obj, related_field).remove(obj)
-
-
-def copy_data(canonical, obj):
-    """try to get the most data possible"""
-    for data_field in obj._meta.get_fields():
-        if not hasattr(data_field, "activitypub_field"):
-            continue
-        data_value = getattr(obj, data_field.name)
-        if not data_value:
-            continue
-        if not getattr(canonical, data_field.name):
-            print("setting data field", data_field.name, data_value)
-            setattr(canonical, data_field.name, data_value)
-    canonical.save()
-
-
-def merge_objects(canonical, obj):
-    copy_data(canonical, obj)
-    update_related(canonical, obj)
-    # remove the outdated entry
-    obj.delete()

bookwyrm/management/merge_command.py

@@ -1,4 +1,3 @@
-from bookwyrm.management.merge import merge_objects
 from django.core.management.base import BaseCommand
 
 
@@ -9,6 +8,11 @@ class MergeCommand(BaseCommand):
         """add the arguments for this command"""
         parser.add_argument("--canonical", type=int, required=True)
         parser.add_argument("--other", type=int, required=True)
+        parser.add_argument(
+            "--dry_run",
+            action="store_true",
+            help="don't actually merge, only print what would happen",
+        )
 
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
@@ -26,4 +30,8 @@ class MergeCommand(BaseCommand):
             print("other book doesn’t exist!")
             return
 
-        merge_objects(canonical, other)
+        absorbed_fields = other.merge_into(canonical, dry_run=options["dry_run"])
+
+        action = "would be" if options["dry_run"] else "has been"
+        print(f"{other.remote_id} {action} merged into {canonical.remote_id}")
+        print(f"absorbed fields: {absorbed_fields}")

bookwyrm/middleware/timezone_middleware.py

@@ -1,5 +1,5 @@
 """ Makes the app aware of the users timezone """
-import pytz
+import zoneinfo
 
 from django.utils import timezone
 
@@ -12,9 +12,7 @@ class TimezoneMiddleware:
 
     def __call__(self, request):
         if request.user.is_authenticated:
-            timezone.activate(pytz.timezone(request.user.preferred_timezone))
+            timezone.activate(zoneinfo.ZoneInfo(request.user.preferred_timezone))
         else:
-            timezone.activate(pytz.utc)
-        response = self.get_response(request)
-        timezone.deactivate()
-        return response
+            timezone.deactivate()
+        return self.get_response(request)

bookwyrm/migrations/0171_alter_user_preferred_timezone.py

@@ -10,6 +10,7 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        # The new timezones are "Factory" and "localtime"
         migrations.AlterField(
             model_name="user",
             name="preferred_timezone",

bookwyrm/migrations/0189_importjob_create_shelves.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2023-11-25 05:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importjob",
+            name="create_shelves",
+            field=models.BooleanField(default=True),
+        ),
+    ]

bookwyrm/migrations/0193_auto_20240128_0249.py

@@ -0,0 +1,92 @@
+# Generated by Django 3.2.23 on 2024-01-28 02:49
+
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+from django.core.files.storage import storages
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0192_sitesettings_user_exports_enabled"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="bookwyrmexportjob",
+            name="export_json",
+            field=models.JSONField(
+                encoder=django.core.serializers.json.DjangoJSONEncoder, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="bookwyrmexportjob",
+            name="json_completed",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name="bookwyrmexportjob",
+            name="export_data",
+            field=models.FileField(
+                null=True,
+                storage=storages["exports"],
+                upload_to="",
+            ),
+        ),
+        migrations.CreateModel(
+            name="AddFileToTar",
+            fields=[
+                (
+                    "childjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.childjob",
+                    ),
+                ),
+                (
+                    "parent_export_job",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="child_edition_export_jobs",
+                        to="bookwyrm.bookwyrmexportjob",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.childjob",),
+        ),
+        migrations.CreateModel(
+            name="AddBookToUserExportJob",
+            fields=[
+                (
+                    "childjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.childjob",
+                    ),
+                ),
+                (
+                    "edition",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="bookwyrm.edition",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.childjob",),
+        ),
+    ]

bookwyrm/migrations/0196_merge_20240318_1737.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2024-03-18 17:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0193_auto_20240128_0249"),
+        ("bookwyrm", "0195_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0197_merge_20240324_0235.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-03-24 02:35
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0196_merge_20240318_1737"),
+        ("bookwyrm", "0196_merge_pr3134_into_main"),
+    ]
+
+    operations = []

bookwyrm/migrations/0197_mergedauthor_mergedbook.py

@@ -0,0 +1,48 @@
+# Generated by Django 3.2.24 on 2024-02-28 21:30
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0196_merge_pr3134_into_main"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="MergedBook",
+            fields=[
+                ("deleted_id", models.IntegerField(primary_key=True, serialize=False)),
+                (
+                    "merged_into",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="absorbed",
+                        to="bookwyrm.book",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="MergedAuthor",
+            fields=[
+                ("deleted_id", models.IntegerField(primary_key=True, serialize=False)),
+                (
+                    "merged_into",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="absorbed",
+                        to="bookwyrm.author",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

bookwyrm/migrations/0198_alter_bookwyrmexportjob_export_data.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.25 on 2024-03-26 11:37
+
+import bookwyrm.models.bookwyrm_export_job
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0197_merge_20240324_0235"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="bookwyrmexportjob",
+            name="export_data",
+            field=models.FileField(
+                null=True,
+                storage=bookwyrm.models.bookwyrm_export_job.select_exports_storage,
+                upload_to="",
+            ),
+        ),
+    ]

bookwyrm/migrations/0199_alter_userblocks_user_object_and_more.py

@@ -0,0 +1,70 @@
+# Generated by Django 4.2.11 on 2024-03-29 19:25
+
+import bookwyrm.models.fields
+from django.conf import settings
+from django.db import migrations
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0198_book_search_vector_author_aliases"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="userblocks",
+            name="user_object",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_object",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userblocks",
+            name="user_subject",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_subject",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userfollowrequest",
+            name="user_object",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_object",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userfollowrequest",
+            name="user_subject",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_subject",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userfollows",
+            name="user_object",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_object",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userfollows",
+            name="user_subject",
+            field=bookwyrm.models.fields.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="%(class)s_user_subject",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+    ]

bookwyrm/migrations/0199_merge_20240326_1217.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-03-26 12:17
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0198_alter_bookwyrmexportjob_export_data"),
+        ("bookwyrm", "0198_book_search_vector_author_aliases"),
+    ]
+
+    operations = []

bookwyrm/migrations/0199_status_bookwyrm_st_remote__06aeba_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-02 19:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0198_book_search_vector_author_aliases"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="status",
+            index=models.Index(
+                fields=["remote_id"], name="bookwyrm_st_remote__06aeba_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0200_alter_user_preferred_timezone.py

@@ -0,0 +1,633 @@
+# Generated by Django 4.2.11 on 2024-04-01 20:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0199_alter_userblocks_user_object_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_timezone",
+            field=models.CharField(
+                choices=[
+                    ("Africa/Abidjan", "Africa/Abidjan"),
+                    ("Africa/Accra", "Africa/Accra"),
+                    ("Africa/Addis_Ababa", "Africa/Addis_Ababa"),
+                    ("Africa/Algiers", "Africa/Algiers"),
+                    ("Africa/Asmara", "Africa/Asmara"),
+                    ("Africa/Asmera", "Africa/Asmera"),
+                    ("Africa/Bamako", "Africa/Bamako"),
+                    ("Africa/Bangui", "Africa/Bangui"),
+                    ("Africa/Banjul", "Africa/Banjul"),
+                    ("Africa/Bissau", "Africa/Bissau"),
+                    ("Africa/Blantyre", "Africa/Blantyre"),
+                    ("Africa/Brazzaville", "Africa/Brazzaville"),
+                    ("Africa/Bujumbura", "Africa/Bujumbura"),
+                    ("Africa/Cairo", "Africa/Cairo"),
+                    ("Africa/Casablanca", "Africa/Casablanca"),
+                    ("Africa/Ceuta", "Africa/Ceuta"),
+                    ("Africa/Conakry", "Africa/Conakry"),
+                    ("Africa/Dakar", "Africa/Dakar"),
+                    ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"),
+                    ("Africa/Djibouti", "Africa/Djibouti"),
+                    ("Africa/Douala", "Africa/Douala"),
+                    ("Africa/El_Aaiun", "Africa/El_Aaiun"),
+                    ("Africa/Freetown", "Africa/Freetown"),
+                    ("Africa/Gaborone", "Africa/Gaborone"),
+                    ("Africa/Harare", "Africa/Harare"),
+                    ("Africa/Johannesburg", "Africa/Johannesburg"),
+                    ("Africa/Juba", "Africa/Juba"),
+                    ("Africa/Kampala", "Africa/Kampala"),
+                    ("Africa/Khartoum", "Africa/Khartoum"),
+                    ("Africa/Kigali", "Africa/Kigali"),
+                    ("Africa/Kinshasa", "Africa/Kinshasa"),
+                    ("Africa/Lagos", "Africa/Lagos"),
+                    ("Africa/Libreville", "Africa/Libreville"),
+                    ("Africa/Lome", "Africa/Lome"),
+                    ("Africa/Luanda", "Africa/Luanda"),
+                    ("Africa/Lubumbashi", "Africa/Lubumbashi"),
+                    ("Africa/Lusaka", "Africa/Lusaka"),
+                    ("Africa/Malabo", "Africa/Malabo"),
+                    ("Africa/Maputo", "Africa/Maputo"),
+                    ("Africa/Maseru", "Africa/Maseru"),
+                    ("Africa/Mbabane", "Africa/Mbabane"),
+                    ("Africa/Mogadishu", "Africa/Mogadishu"),
+                    ("Africa/Monrovia", "Africa/Monrovia"),
+                    ("Africa/Nairobi", "Africa/Nairobi"),
+                    ("Africa/Ndjamena", "Africa/Ndjamena"),
+                    ("Africa/Niamey", "Africa/Niamey"),
+                    ("Africa/Nouakchott", "Africa/Nouakchott"),
+                    ("Africa/Ouagadougou", "Africa/Ouagadougou"),
+                    ("Africa/Porto-Novo", "Africa/Porto-Novo"),
+                    ("Africa/Sao_Tome", "Africa/Sao_Tome"),
+                    ("Africa/Timbuktu", "Africa/Timbuktu"),
+                    ("Africa/Tripoli", "Africa/Tripoli"),
+                    ("Africa/Tunis", "Africa/Tunis"),
+                    ("Africa/Windhoek", "Africa/Windhoek"),
+                    ("America/Adak", "America/Adak"),
+                    ("America/Anchorage", "America/Anchorage"),
+                    ("America/Anguilla", "America/Anguilla"),
+                    ("America/Antigua", "America/Antigua"),
+                    ("America/Araguaina", "America/Araguaina"),
+                    (
+                        "America/Argentina/Buenos_Aires",
+                        "America/Argentina/Buenos_Aires",
+                    ),
+                    ("America/Argentina/Catamarca", "America/Argentina/Catamarca"),
+                    (
+                        "America/Argentina/ComodRivadavia",
+                        "America/Argentina/ComodRivadavia",
+                    ),
+                    ("America/Argentina/Cordoba", "America/Argentina/Cordoba"),
+                    ("America/Argentina/Jujuy", "America/Argentina/Jujuy"),
+                    ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"),
+                    ("America/Argentina/Mendoza", "America/Argentina/Mendoza"),
+                    (
+                        "America/Argentina/Rio_Gallegos",
+                        "America/Argentina/Rio_Gallegos",
+                    ),
+                    ("America/Argentina/Salta", "America/Argentina/Salta"),
+                    ("America/Argentina/San_Juan", "America/Argentina/San_Juan"),
+                    ("America/Argentina/San_Luis", "America/Argentina/San_Luis"),
+                    ("America/Argentina/Tucuman", "America/Argentina/Tucuman"),
+                    ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"),
+                    ("America/Aruba", "America/Aruba"),
+                    ("America/Asuncion", "America/Asuncion"),
+                    ("America/Atikokan", "America/Atikokan"),
+                    ("America/Atka", "America/Atka"),
+                    ("America/Bahia", "America/Bahia"),
+                    ("America/Bahia_Banderas", "America/Bahia_Banderas"),
+                    ("America/Barbados", "America/Barbados"),
+                    ("America/Belem", "America/Belem"),
+                    ("America/Belize", "America/Belize"),
+                    ("America/Blanc-Sablon", "America/Blanc-Sablon"),
+                    ("America/Boa_Vista", "America/Boa_Vista"),
+                    ("America/Bogota", "America/Bogota"),
+                    ("America/Boise", "America/Boise"),
+                    ("America/Buenos_Aires", "America/Buenos_Aires"),
+                    ("America/Cambridge_Bay", "America/Cambridge_Bay"),
+                    ("America/Campo_Grande", "America/Campo_Grande"),
+                    ("America/Cancun", "America/Cancun"),
+                    ("America/Caracas", "America/Caracas"),
+                    ("America/Catamarca", "America/Catamarca"),
+                    ("America/Cayenne", "America/Cayenne"),
+                    ("America/Cayman", "America/Cayman"),
+                    ("America/Chicago", "America/Chicago"),
+                    ("America/Chihuahua", "America/Chihuahua"),
+                    ("America/Ciudad_Juarez", "America/Ciudad_Juarez"),
+                    ("America/Coral_Harbour", "America/Coral_Harbour"),
+                    ("America/Cordoba", "America/Cordoba"),
+                    ("America/Costa_Rica", "America/Costa_Rica"),
+                    ("America/Creston", "America/Creston"),
+                    ("America/Cuiaba", "America/Cuiaba"),
+                    ("America/Curacao", "America/Curacao"),
+                    ("America/Danmarkshavn", "America/Danmarkshavn"),
+                    ("America/Dawson", "America/Dawson"),
+                    ("America/Dawson_Creek", "America/Dawson_Creek"),
+                    ("America/Denver", "America/Denver"),
+                    ("America/Detroit", "America/Detroit"),
+                    ("America/Dominica", "America/Dominica"),
+                    ("America/Edmonton", "America/Edmonton"),
+                    ("America/Eirunepe", "America/Eirunepe"),
+                    ("America/El_Salvador", "America/El_Salvador"),
+                    ("America/Ensenada", "America/Ensenada"),
+                    ("America/Fort_Nelson", "America/Fort_Nelson"),
+                    ("America/Fort_Wayne", "America/Fort_Wayne"),
+                    ("America/Fortaleza", "America/Fortaleza"),
+                    ("America/Glace_Bay", "America/Glace_Bay"),
+                    ("America/Godthab", "America/Godthab"),
+                    ("America/Goose_Bay", "America/Goose_Bay"),
+                    ("America/Grand_Turk", "America/Grand_Turk"),
+                    ("America/Grenada", "America/Grenada"),
+                    ("America/Guadeloupe", "America/Guadeloupe"),
+                    ("America/Guatemala", "America/Guatemala"),
+                    ("America/Guayaquil", "America/Guayaquil"),
+                    ("America/Guyana", "America/Guyana"),
+                    ("America/Halifax", "America/Halifax"),
+                    ("America/Havana", "America/Havana"),
+                    ("America/Hermosillo", "America/Hermosillo"),
+                    ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"),
+                    ("America/Indiana/Knox", "America/Indiana/Knox"),
+                    ("America/Indiana/Marengo", "America/Indiana/Marengo"),
+                    ("America/Indiana/Petersburg", "America/Indiana/Petersburg"),
+                    ("America/Indiana/Tell_City", "America/Indiana/Tell_City"),
+                    ("America/Indiana/Vevay", "America/Indiana/Vevay"),
+                    ("America/Indiana/Vincennes", "America/Indiana/Vincennes"),
+                    ("America/Indiana/Winamac", "America/Indiana/Winamac"),
+                    ("America/Indianapolis", "America/Indianapolis"),
+                    ("America/Inuvik", "America/Inuvik"),
+                    ("America/Iqaluit", "America/Iqaluit"),
+                    ("America/Jamaica", "America/Jamaica"),
+                    ("America/Jujuy", "America/Jujuy"),
+                    ("America/Juneau", "America/Juneau"),
+                    ("America/Kentucky/Louisville", "America/Kentucky/Louisville"),
+                    ("America/Kentucky/Monticello", "America/Kentucky/Monticello"),
+                    ("America/Knox_IN", "America/Knox_IN"),
+                    ("America/Kralendijk", "America/Kralendijk"),
+                    ("America/La_Paz", "America/La_Paz"),
+                    ("America/Lima", "America/Lima"),
+                    ("America/Los_Angeles", "America/Los_Angeles"),
+                    ("America/Louisville", "America/Louisville"),
+                    ("America/Lower_Princes", "America/Lower_Princes"),
+                    ("America/Maceio", "America/Maceio"),
+                    ("America/Managua", "America/Managua"),
+                    ("America/Manaus", "America/Manaus"),
+                    ("America/Marigot", "America/Marigot"),
+                    ("America/Martinique", "America/Martinique"),
+                    ("America/Matamoros", "America/Matamoros"),
+                    ("America/Mazatlan", "America/Mazatlan"),
+                    ("America/Mendoza", "America/Mendoza"),
+                    ("America/Menominee", "America/Menominee"),
+                    ("America/Merida", "America/Merida"),
+                    ("America/Metlakatla", "America/Metlakatla"),
+                    ("America/Mexico_City", "America/Mexico_City"),
+                    ("America/Miquelon", "America/Miquelon"),
+                    ("America/Moncton", "America/Moncton"),
+                    ("America/Monterrey", "America/Monterrey"),
+                    ("America/Montevideo", "America/Montevideo"),
+                    ("America/Montreal", "America/Montreal"),
+                    ("America/Montserrat", "America/Montserrat"),
+                    ("America/Nassau", "America/Nassau"),
+                    ("America/New_York", "America/New_York"),
+                    ("America/Nipigon", "America/Nipigon"),
+                    ("America/Nome", "America/Nome"),
+                    ("America/Noronha", "America/Noronha"),
+                    ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"),
+                    ("America/North_Dakota/Center", "America/North_Dakota/Center"),
+                    (
+                        "America/North_Dakota/New_Salem",
+                        "America/North_Dakota/New_Salem",
+                    ),
+                    ("America/Nuuk", "America/Nuuk"),
+                    ("America/Ojinaga", "America/Ojinaga"),
+                    ("America/Panama", "America/Panama"),
+                    ("America/Pangnirtung", "America/Pangnirtung"),
+                    ("America/Paramaribo", "America/Paramaribo"),
+                    ("America/Phoenix", "America/Phoenix"),
+                    ("America/Port-au-Prince", "America/Port-au-Prince"),
+                    ("America/Port_of_Spain", "America/Port_of_Spain"),
+                    ("America/Porto_Acre", "America/Porto_Acre"),
+                    ("America/Porto_Velho", "America/Porto_Velho"),
+                    ("America/Puerto_Rico", "America/Puerto_Rico"),
+                    ("America/Punta_Arenas", "America/Punta_Arenas"),
+                    ("America/Rainy_River", "America/Rainy_River"),
+                    ("America/Rankin_Inlet", "America/Rankin_Inlet"),
+                    ("America/Recife", "America/Recife"),
+                    ("America/Regina", "America/Regina"),
+                    ("America/Resolute", "America/Resolute"),
+                    ("America/Rio_Branco", "America/Rio_Branco"),
+                    ("America/Rosario", "America/Rosario"),
+                    ("America/Santa_Isabel", "America/Santa_Isabel"),
+                    ("America/Santarem", "America/Santarem"),
+                    ("America/Santiago", "America/Santiago"),
+                    ("America/Santo_Domingo", "America/Santo_Domingo"),
+                    ("America/Sao_Paulo", "America/Sao_Paulo"),
+                    ("America/Scoresbysund", "America/Scoresbysund"),
+                    ("America/Shiprock", "America/Shiprock"),
+                    ("America/Sitka", "America/Sitka"),
+                    ("America/St_Barthelemy", "America/St_Barthelemy"),
+                    ("America/St_Johns", "America/St_Johns"),
+                    ("America/St_Kitts", "America/St_Kitts"),
+                    ("America/St_Lucia", "America/St_Lucia"),
+                    ("America/St_Thomas", "America/St_Thomas"),
+                    ("America/St_Vincent", "America/St_Vincent"),
+                    ("America/Swift_Current", "America/Swift_Current"),
+                    ("America/Tegucigalpa", "America/Tegucigalpa"),
+                    ("America/Thule", "America/Thule"),
+                    ("America/Thunder_Bay", "America/Thunder_Bay"),
+                    ("America/Tijuana", "America/Tijuana"),
+                    ("America/Toronto", "America/Toronto"),
+                    ("America/Tortola", "America/Tortola"),
+                    ("America/Vancouver", "America/Vancouver"),
+                    ("America/Virgin", "America/Virgin"),
+                    ("America/Whitehorse", "America/Whitehorse"),
+                    ("America/Winnipeg", "America/Winnipeg"),
+                    ("America/Yakutat", "America/Yakutat"),
+                    ("America/Yellowknife", "America/Yellowknife"),
+                    ("Antarctica/Casey", "Antarctica/Casey"),
+                    ("Antarctica/Davis", "Antarctica/Davis"),
+                    ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"),
+                    ("Antarctica/Macquarie", "Antarctica/Macquarie"),
+                    ("Antarctica/Mawson", "Antarctica/Mawson"),
+                    ("Antarctica/McMurdo", "Antarctica/McMurdo"),
+                    ("Antarctica/Palmer", "Antarctica/Palmer"),
+                    ("Antarctica/Rothera", "Antarctica/Rothera"),
+                    ("Antarctica/South_Pole", "Antarctica/South_Pole"),
+                    ("Antarctica/Syowa", "Antarctica/Syowa"),
+                    ("Antarctica/Troll", "Antarctica/Troll"),
+                    ("Antarctica/Vostok", "Antarctica/Vostok"),
+                    ("Arctic/Longyearbyen", "Arctic/Longyearbyen"),
+                    ("Asia/Aden", "Asia/Aden"),
+                    ("Asia/Almaty", "Asia/Almaty"),
+                    ("Asia/Amman", "Asia/Amman"),
+                    ("Asia/Anadyr", "Asia/Anadyr"),
+                    ("Asia/Aqtau", "Asia/Aqtau"),
+                    ("Asia/Aqtobe", "Asia/Aqtobe"),
+                    ("Asia/Ashgabat", "Asia/Ashgabat"),
+                    ("Asia/Ashkhabad", "Asia/Ashkhabad"),
+                    ("Asia/Atyrau", "Asia/Atyrau"),
+                    ("Asia/Baghdad", "Asia/Baghdad"),
+                    ("Asia/Bahrain", "Asia/Bahrain"),
+                    ("Asia/Baku", "Asia/Baku"),
+                    ("Asia/Bangkok", "Asia/Bangkok"),
+                    ("Asia/Barnaul", "Asia/Barnaul"),
+                    ("Asia/Beirut", "Asia/Beirut"),
+                    ("Asia/Bishkek", "Asia/Bishkek"),
+                    ("Asia/Brunei", "Asia/Brunei"),
+                    ("Asia/Calcutta", "Asia/Calcutta"),
+                    ("Asia/Chita", "Asia/Chita"),
+                    ("Asia/Choibalsan", "Asia/Choibalsan"),
+                    ("Asia/Chongqing", "Asia/Chongqing"),
+                    ("Asia/Chungking", "Asia/Chungking"),
+                    ("Asia/Colombo", "Asia/Colombo"),
+                    ("Asia/Dacca", "Asia/Dacca"),
+                    ("Asia/Damascus", "Asia/Damascus"),
+                    ("Asia/Dhaka", "Asia/Dhaka"),
+                    ("Asia/Dili", "Asia/Dili"),
+                    ("Asia/Dubai", "Asia/Dubai"),
+                    ("Asia/Dushanbe", "Asia/Dushanbe"),
+                    ("Asia/Famagusta", "Asia/Famagusta"),
+                    ("Asia/Gaza", "Asia/Gaza"),
+                    ("Asia/Harbin", "Asia/Harbin"),
+                    ("Asia/Hebron", "Asia/Hebron"),
+                    ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"),
+                    ("Asia/Hong_Kong", "Asia/Hong_Kong"),
+                    ("Asia/Hovd", "Asia/Hovd"),
+                    ("Asia/Irkutsk", "Asia/Irkutsk"),
+                    ("Asia/Istanbul", "Asia/Istanbul"),
+                    ("Asia/Jakarta", "Asia/Jakarta"),
+                    ("Asia/Jayapura", "Asia/Jayapura"),
+                    ("Asia/Jerusalem", "Asia/Jerusalem"),
+                    ("Asia/Kabul", "Asia/Kabul"),
+                    ("Asia/Kamchatka", "Asia/Kamchatka"),
+                    ("Asia/Karachi", "Asia/Karachi"),
+                    ("Asia/Kashgar", "Asia/Kashgar"),
+                    ("Asia/Kathmandu", "Asia/Kathmandu"),
+                    ("Asia/Katmandu", "Asia/Katmandu"),
+                    ("Asia/Khandyga", "Asia/Khandyga"),
+                    ("Asia/Kolkata", "Asia/Kolkata"),
+                    ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"),
+                    ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"),
+                    ("Asia/Kuching", "Asia/Kuching"),
+                    ("Asia/Kuwait", "Asia/Kuwait"),
+                    ("Asia/Macao", "Asia/Macao"),
+                    ("Asia/Macau", "Asia/Macau"),
+                    ("Asia/Magadan", "Asia/Magadan"),
+                    ("Asia/Makassar", "Asia/Makassar"),
+                    ("Asia/Manila", "Asia/Manila"),
+                    ("Asia/Muscat", "Asia/Muscat"),
+                    ("Asia/Nicosia", "Asia/Nicosia"),
+                    ("Asia/Novokuznetsk", "Asia/Novokuznetsk"),
+                    ("Asia/Novosibirsk", "Asia/Novosibirsk"),
+                    ("Asia/Omsk", "Asia/Omsk"),
+                    ("Asia/Oral", "Asia/Oral"),
+                    ("Asia/Phnom_Penh", "Asia/Phnom_Penh"),
+                    ("Asia/Pontianak", "Asia/Pontianak"),
+                    ("Asia/Pyongyang", "Asia/Pyongyang"),
+                    ("Asia/Qatar", "Asia/Qatar"),
+                    ("Asia/Qostanay", "Asia/Qostanay"),
+                    ("Asia/Qyzylorda", "Asia/Qyzylorda"),
+                    ("Asia/Rangoon", "Asia/Rangoon"),
+                    ("Asia/Riyadh", "Asia/Riyadh"),
+                    ("Asia/Saigon", "Asia/Saigon"),
+                    ("Asia/Sakhalin", "Asia/Sakhalin"),
+                    ("Asia/Samarkand", "Asia/Samarkand"),
+                    ("Asia/Seoul", "Asia/Seoul"),
+                    ("Asia/Shanghai", "Asia/Shanghai"),
+                    ("Asia/Singapore", "Asia/Singapore"),
+                    ("Asia/Srednekolymsk", "Asia/Srednekolymsk"),
+                    ("Asia/Taipei", "Asia/Taipei"),
+                    ("Asia/Tashkent", "Asia/Tashkent"),
+                    ("Asia/Tbilisi", "Asia/Tbilisi"),
+                    ("Asia/Tehran", "Asia/Tehran"),
+                    ("Asia/Tel_Aviv", "Asia/Tel_Aviv"),
+                    ("Asia/Thimbu", "Asia/Thimbu"),
+                    ("Asia/Thimphu", "Asia/Thimphu"),
+                    ("Asia/Tokyo", "Asia/Tokyo"),
+                    ("Asia/Tomsk", "Asia/Tomsk"),
+                    ("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"),
+                    ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"),
+                    ("Asia/Ulan_Bator", "Asia/Ulan_Bator"),
+                    ("Asia/Urumqi", "Asia/Urumqi"),
+                    ("Asia/Ust-Nera", "Asia/Ust-Nera"),
+                    ("Asia/Vientiane", "Asia/Vientiane"),
+                    ("Asia/Vladivostok", "Asia/Vladivostok"),
+                    ("Asia/Yakutsk", "Asia/Yakutsk"),
+                    ("Asia/Yangon", "Asia/Yangon"),
+                    ("Asia/Yekaterinburg", "Asia/Yekaterinburg"),
+                    ("Asia/Yerevan", "Asia/Yerevan"),
+                    ("Atlantic/Azores", "Atlantic/Azores"),
+                    ("Atlantic/Bermuda", "Atlantic/Bermuda"),
+                    ("Atlantic/Canary", "Atlantic/Canary"),
+                    ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"),
+                    ("Atlantic/Faeroe", "Atlantic/Faeroe"),
+                    ("Atlantic/Faroe", "Atlantic/Faroe"),
+                    ("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"),
+                    ("Atlantic/Madeira", "Atlantic/Madeira"),
+                    ("Atlantic/Reykjavik", "Atlantic/Reykjavik"),
+                    ("Atlantic/South_Georgia", "Atlantic/South_Georgia"),
+                    ("Atlantic/St_Helena", "Atlantic/St_Helena"),
+                    ("Atlantic/Stanley", "Atlantic/Stanley"),
+                    ("Australia/ACT", "Australia/ACT"),
+                    ("Australia/Adelaide", "Australia/Adelaide"),
+                    ("Australia/Brisbane", "Australia/Brisbane"),
+                    ("Australia/Broken_Hill", "Australia/Broken_Hill"),
+                    ("Australia/Canberra", "Australia/Canberra"),
+                    ("Australia/Currie", "Australia/Currie"),
+                    ("Australia/Darwin", "Australia/Darwin"),
+                    ("Australia/Eucla", "Australia/Eucla"),
+                    ("Australia/Hobart", "Australia/Hobart"),
+                    ("Australia/LHI", "Australia/LHI"),
+                    ("Australia/Lindeman", "Australia/Lindeman"),
+                    ("Australia/Lord_Howe", "Australia/Lord_Howe"),
+                    ("Australia/Melbourne", "Australia/Melbourne"),
+                    ("Australia/NSW", "Australia/NSW"),
+                    ("Australia/North", "Australia/North"),
+                    ("Australia/Perth", "Australia/Perth"),
+                    ("Australia/Queensland", "Australia/Queensland"),
+                    ("Australia/South", "Australia/South"),
+                    ("Australia/Sydney", "Australia/Sydney"),
+                    ("Australia/Tasmania", "Australia/Tasmania"),
+                    ("Australia/Victoria", "Australia/Victoria"),
+                    ("Australia/West", "Australia/West"),
+                    ("Australia/Yancowinna", "Australia/Yancowinna"),
+                    ("Brazil/Acre", "Brazil/Acre"),
+                    ("Brazil/DeNoronha", "Brazil/DeNoronha"),
+                    ("Brazil/East", "Brazil/East"),
+                    ("Brazil/West", "Brazil/West"),
+                    ("CET", "CET"),
+                    ("CST6CDT", "CST6CDT"),
+                    ("Canada/Atlantic", "Canada/Atlantic"),
+                    ("Canada/Central", "Canada/Central"),
+                    ("Canada/Eastern", "Canada/Eastern"),
+                    ("Canada/Mountain", "Canada/Mountain"),
+                    ("Canada/Newfoundland", "Canada/Newfoundland"),
+                    ("Canada/Pacific", "Canada/Pacific"),
+                    ("Canada/Saskatchewan", "Canada/Saskatchewan"),
+                    ("Canada/Yukon", "Canada/Yukon"),
+                    ("Chile/Continental", "Chile/Continental"),
+                    ("Chile/EasterIsland", "Chile/EasterIsland"),
+                    ("Cuba", "Cuba"),
+                    ("EET", "EET"),
+                    ("EST", "EST"),
+                    ("EST5EDT", "EST5EDT"),
+                    ("Egypt", "Egypt"),
+                    ("Eire", "Eire"),
+                    ("Etc/GMT", "Etc/GMT"),
+                    ("Etc/GMT+0", "Etc/GMT+0"),
+                    ("Etc/GMT+1", "Etc/GMT+1"),
+                    ("Etc/GMT+10", "Etc/GMT+10"),
+                    ("Etc/GMT+11", "Etc/GMT+11"),
+                    ("Etc/GMT+12", "Etc/GMT+12"),
+                    ("Etc/GMT+2", "Etc/GMT+2"),
+                    ("Etc/GMT+3", "Etc/GMT+3"),
+                    ("Etc/GMT+4", "Etc/GMT+4"),
+                    ("Etc/GMT+5", "Etc/GMT+5"),
+                    ("Etc/GMT+6", "Etc/GMT+6"),
+                    ("Etc/GMT+7", "Etc/GMT+7"),
+                    ("Etc/GMT+8", "Etc/GMT+8"),
+                    ("Etc/GMT+9", "Etc/GMT+9"),
+                    ("Etc/GMT-0", "Etc/GMT-0"),
+                    ("Etc/GMT-1", "Etc/GMT-1"),
+                    ("Etc/GMT-10", "Etc/GMT-10"),
+                    ("Etc/GMT-11", "Etc/GMT-11"),
+                    ("Etc/GMT-12", "Etc/GMT-12"),
+                    ("Etc/GMT-13", "Etc/GMT-13"),
+                    ("Etc/GMT-14", "Etc/GMT-14"),
+                    ("Etc/GMT-2", "Etc/GMT-2"),
+                    ("Etc/GMT-3", "Etc/GMT-3"),
+                    ("Etc/GMT-4", "Etc/GMT-4"),
+                    ("Etc/GMT-5", "Etc/GMT-5"),
+                    ("Etc/GMT-6", "Etc/GMT-6"),
+                    ("Etc/GMT-7", "Etc/GMT-7"),
+                    ("Etc/GMT-8", "Etc/GMT-8"),
+                    ("Etc/GMT-9", "Etc/GMT-9"),
+                    ("Etc/GMT0", "Etc/GMT0"),
+                    ("Etc/Greenwich", "Etc/Greenwich"),
+                    ("Etc/UCT", "Etc/UCT"),
+                    ("Etc/UTC", "Etc/UTC"),
+                    ("Etc/Universal", "Etc/Universal"),
+                    ("Etc/Zulu", "Etc/Zulu"),
+                    ("Europe/Amsterdam", "Europe/Amsterdam"),
+                    ("Europe/Andorra", "Europe/Andorra"),
+                    ("Europe/Astrakhan", "Europe/Astrakhan"),
+                    ("Europe/Athens", "Europe/Athens"),
+                    ("Europe/Belfast", "Europe/Belfast"),
+                    ("Europe/Belgrade", "Europe/Belgrade"),
+                    ("Europe/Berlin", "Europe/Berlin"),
+                    ("Europe/Bratislava", "Europe/Bratislava"),
+                    ("Europe/Brussels", "Europe/Brussels"),
+                    ("Europe/Bucharest", "Europe/Bucharest"),
+                    ("Europe/Budapest", "Europe/Budapest"),
+                    ("Europe/Busingen", "Europe/Busingen"),
+                    ("Europe/Chisinau", "Europe/Chisinau"),
+                    ("Europe/Copenhagen", "Europe/Copenhagen"),
+                    ("Europe/Dublin", "Europe/Dublin"),
+                    ("Europe/Gibraltar", "Europe/Gibraltar"),
+                    ("Europe/Guernsey", "Europe/Guernsey"),
+                    ("Europe/Helsinki", "Europe/Helsinki"),
+                    ("Europe/Isle_of_Man", "Europe/Isle_of_Man"),
+                    ("Europe/Istanbul", "Europe/Istanbul"),
+                    ("Europe/Jersey", "Europe/Jersey"),
+                    ("Europe/Kaliningrad", "Europe/Kaliningrad"),
+                    ("Europe/Kiev", "Europe/Kiev"),
+                    ("Europe/Kirov", "Europe/Kirov"),
+                    ("Europe/Kyiv", "Europe/Kyiv"),
+                    ("Europe/Lisbon", "Europe/Lisbon"),
+                    ("Europe/Ljubljana", "Europe/Ljubljana"),
+                    ("Europe/London", "Europe/London"),
+                    ("Europe/Luxembourg", "Europe/Luxembourg"),
+                    ("Europe/Madrid", "Europe/Madrid"),
+                    ("Europe/Malta", "Europe/Malta"),
+                    ("Europe/Mariehamn", "Europe/Mariehamn"),
+                    ("Europe/Minsk", "Europe/Minsk"),
+                    ("Europe/Monaco", "Europe/Monaco"),
+                    ("Europe/Moscow", "Europe/Moscow"),
+                    ("Europe/Nicosia", "Europe/Nicosia"),
+                    ("Europe/Oslo", "Europe/Oslo"),
+                    ("Europe/Paris", "Europe/Paris"),
+                    ("Europe/Podgorica", "Europe/Podgorica"),
+                    ("Europe/Prague", "Europe/Prague"),
+                    ("Europe/Riga", "Europe/Riga"),
+                    ("Europe/Rome", "Europe/Rome"),
+                    ("Europe/Samara", "Europe/Samara"),
+                    ("Europe/San_Marino", "Europe/San_Marino"),
+                    ("Europe/Sarajevo", "Europe/Sarajevo"),
+                    ("Europe/Saratov", "Europe/Saratov"),
+                    ("Europe/Simferopol", "Europe/Simferopol"),
+                    ("Europe/Skopje", "Europe/Skopje"),
+                    ("Europe/Sofia", "Europe/Sofia"),
+                    ("Europe/Stockholm", "Europe/Stockholm"),
+                    ("Europe/Tallinn", "Europe/Tallinn"),
+                    ("Europe/Tirane", "Europe/Tirane"),
+                    ("Europe/Tiraspol", "Europe/Tiraspol"),
+                    ("Europe/Ulyanovsk", "Europe/Ulyanovsk"),
+                    ("Europe/Uzhgorod", "Europe/Uzhgorod"),
+                    ("Europe/Vaduz", "Europe/Vaduz"),
+                    ("Europe/Vatican", "Europe/Vatican"),
+                    ("Europe/Vienna", "Europe/Vienna"),
+                    ("Europe/Vilnius", "Europe/Vilnius"),
+                    ("Europe/Volgograd", "Europe/Volgograd"),
+                    ("Europe/Warsaw", "Europe/Warsaw"),
+                    ("Europe/Zagreb", "Europe/Zagreb"),
+                    ("Europe/Zaporozhye", "Europe/Zaporozhye"),
+                    ("Europe/Zurich", "Europe/Zurich"),
+                    ("Factory", "Factory"),
+                    ("GB", "GB"),
+                    ("GB-Eire", "GB-Eire"),
+                    ("GMT", "GMT"),
+                    ("GMT+0", "GMT+0"),
+                    ("GMT-0", "GMT-0"),
+                    ("GMT0", "GMT0"),
+                    ("Greenwich", "Greenwich"),
+                    ("HST", "HST"),
+                    ("Hongkong", "Hongkong"),
+                    ("Iceland", "Iceland"),
+                    ("Indian/Antananarivo", "Indian/Antananarivo"),
+                    ("Indian/Chagos", "Indian/Chagos"),
+                    ("Indian/Christmas", "Indian/Christmas"),
+                    ("Indian/Cocos", "Indian/Cocos"),
+                    ("Indian/Comoro", "Indian/Comoro"),
+                    ("Indian/Kerguelen", "Indian/Kerguelen"),
+                    ("Indian/Mahe", "Indian/Mahe"),
+                    ("Indian/Maldives", "Indian/Maldives"),
+                    ("Indian/Mauritius", "Indian/Mauritius"),
+                    ("Indian/Mayotte", "Indian/Mayotte"),
+                    ("Indian/Reunion", "Indian/Reunion"),
+                    ("Iran", "Iran"),
+                    ("Israel", "Israel"),
+                    ("Jamaica", "Jamaica"),
+                    ("Japan", "Japan"),
+                    ("Kwajalein", "Kwajalein"),
+                    ("Libya", "Libya"),
+                    ("MET", "MET"),
+                    ("MST", "MST"),
+                    ("MST7MDT", "MST7MDT"),
+                    ("Mexico/BajaNorte", "Mexico/BajaNorte"),
+                    ("Mexico/BajaSur", "Mexico/BajaSur"),
+                    ("Mexico/General", "Mexico/General"),
+                    ("NZ", "NZ"),
+                    ("NZ-CHAT", "NZ-CHAT"),
+                    ("Navajo", "Navajo"),
+                    ("PRC", "PRC"),
+                    ("PST8PDT", "PST8PDT"),
+                    ("Pacific/Apia", "Pacific/Apia"),
+                    ("Pacific/Auckland", "Pacific/Auckland"),
+                    ("Pacific/Bougainville", "Pacific/Bougainville"),
+                    ("Pacific/Chatham", "Pacific/Chatham"),
+                    ("Pacific/Chuuk", "Pacific/Chuuk"),
+                    ("Pacific/Easter", "Pacific/Easter"),
+                    ("Pacific/Efate", "Pacific/Efate"),
+                    ("Pacific/Enderbury", "Pacific/Enderbury"),
+                    ("Pacific/Fakaofo", "Pacific/Fakaofo"),
+                    ("Pacific/Fiji", "Pacific/Fiji"),
+                    ("Pacific/Funafuti", "Pacific/Funafuti"),
+                    ("Pacific/Galapagos", "Pacific/Galapagos"),
+                    ("Pacific/Gambier", "Pacific/Gambier"),
+                    ("Pacific/Guadalcanal", "Pacific/Guadalcanal"),
+                    ("Pacific/Guam", "Pacific/Guam"),
+                    ("Pacific/Honolulu", "Pacific/Honolulu"),
+                    ("Pacific/Johnston", "Pacific/Johnston"),
+                    ("Pacific/Kanton", "Pacific/Kanton"),
+                    ("Pacific/Kiritimati", "Pacific/Kiritimati"),
+                    ("Pacific/Kosrae", "Pacific/Kosrae"),
+                    ("Pacific/Kwajalein", "Pacific/Kwajalein"),
+                    ("Pacific/Majuro", "Pacific/Majuro"),
+                    ("Pacific/Marquesas", "Pacific/Marquesas"),
+                    ("Pacific/Midway", "Pacific/Midway"),
+                    ("Pacific/Nauru", "Pacific/Nauru"),
+                    ("Pacific/Niue", "Pacific/Niue"),
+                    ("Pacific/Norfolk", "Pacific/Norfolk"),
+                    ("Pacific/Noumea", "Pacific/Noumea"),
+                    ("Pacific/Pago_Pago", "Pacific/Pago_Pago"),
+                    ("Pacific/Palau", "Pacific/Palau"),
+                    ("Pacific/Pitcairn", "Pacific/Pitcairn"),
+                    ("Pacific/Pohnpei", "Pacific/Pohnpei"),
+                    ("Pacific/Ponape", "Pacific/Ponape"),
+                    ("Pacific/Port_Moresby", "Pacific/Port_Moresby"),
+                    ("Pacific/Rarotonga", "Pacific/Rarotonga"),
+                    ("Pacific/Saipan", "Pacific/Saipan"),
+                    ("Pacific/Samoa", "Pacific/Samoa"),
+                    ("Pacific/Tahiti", "Pacific/Tahiti"),
+                    ("Pacific/Tarawa", "Pacific/Tarawa"),
+                    ("Pacific/Tongatapu", "Pacific/Tongatapu"),
+                    ("Pacific/Truk", "Pacific/Truk"),
+                    ("Pacific/Wake", "Pacific/Wake"),
+                    ("Pacific/Wallis", "Pacific/Wallis"),
+                    ("Pacific/Yap", "Pacific/Yap"),
+                    ("Poland", "Poland"),
+                    ("Portugal", "Portugal"),
+                    ("ROC", "ROC"),
+                    ("ROK", "ROK"),
+                    ("Singapore", "Singapore"),
+                    ("Turkey", "Turkey"),
+                    ("UCT", "UCT"),
+                    ("US/Alaska", "US/Alaska"),
+                    ("US/Aleutian", "US/Aleutian"),
+                    ("US/Arizona", "US/Arizona"),
+                    ("US/Central", "US/Central"),
+                    ("US/East-Indiana", "US/East-Indiana"),
+                    ("US/Eastern", "US/Eastern"),
+                    ("US/Hawaii", "US/Hawaii"),
+                    ("US/Indiana-Starke", "US/Indiana-Starke"),
+                    ("US/Michigan", "US/Michigan"),
+                    ("US/Mountain", "US/Mountain"),
+                    ("US/Pacific", "US/Pacific"),
+                    ("US/Samoa", "US/Samoa"),
+                    ("UTC", "UTC"),
+                    ("Universal", "Universal"),
+                    ("W-SU", "W-SU"),
+                    ("WET", "WET"),
+                    ("Zulu", "Zulu"),
+                    ("localtime", "localtime"),
+                ],
+                default="UTC",
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0200_auto_20240327_1914.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.25 on 2024-03-27 19:14
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0199_merge_20240326_1217"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="addfiletotar",
+            name="childjob_ptr",
+        ),
+        migrations.RemoveField(
+            model_name="addfiletotar",
+            name="parent_export_job",
+        ),
+        migrations.DeleteModel(
+            name="AddBookToUserExportJob",
+        ),
+        migrations.DeleteModel(
+            name="AddFileToTar",
+        ),
+    ]

bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0199_status_bookwyrm_st_remote__06aeba_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="status",
+            index=models.Index(
+                fields=["thread_id"], name="bookwyrm_st_thread__cf064f_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.2.11 on 2024-04-01 21:09
+
+import bookwyrm.models.fields
+from django.db import migrations, models
+from django.contrib.postgres.operations import CreateCollation
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0200_alter_user_preferred_timezone"),
+    ]
+
+    operations = [
+        CreateCollation(
+            "case_insensitive",
+            provider="icu",
+            locale="und-u-ks-level2",
+            deterministic=False,
+        ),
+        migrations.AlterField(
+            model_name="hashtag",
+            name="name",
+            field=bookwyrm.models.fields.CharField(
+                db_collation="case_insensitive", max_length=256
+            ),
+        ),
+        migrations.AlterField(
+            model_name="user",
+            name="localname",
+            field=models.CharField(
+                db_collation="case_insensitive",
+                max_length=255,
+                null=True,
+                unique=True,
+                validators=[bookwyrm.models.fields.validate_localname],
+            ),
+        ),
+    ]

bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0200_status_bookwyrm_st_thread__cf064f_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="keypair",
+            index=models.Index(
+                fields=["remote_id"], name="bookwyrm_ke_remote__472927_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0201_keypair_bookwyrm_ke_remote__472927_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="user",
+            index=models.Index(
+                fields=["username"], name="bookwyrm_us_usernam_b2546d_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-04-03 19:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0202_user_bookwyrm_us_usernam_b2546d_idx"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="user",
+            index=models.Index(
+                fields=["is_active", "local"], name="bookwyrm_us_is_acti_972dc4_idx"
+            ),
+        ),
+    ]

bookwyrm/migrations/0204_merge_20240409_1042.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-04-09 10:42
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0197_mergedauthor_mergedbook"),
+        ("bookwyrm", "0203_user_bookwyrm_us_is_acti_972dc4_idx"),
+    ]
+
+    operations = []

bookwyrm/migrations/0205_merge_20240410_2022.py

@@ -0,0 +1,13 @@
+# Generated by Django 4.2.11 on 2024-04-10 20:22
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0201_alter_hashtag_name_alter_user_localname"),
+        ("bookwyrm", "0204_merge_20240409_1042"),
+    ]
+
+    operations = []

bookwyrm/migrations/0205_merge_20240413_0232.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.25 on 2024-04-13 02:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0200_auto_20240327_1914"),
+        ("bookwyrm", "0204_merge_20240409_1042"),
+    ]
+
+    operations = []

bookwyrm/migrations/0206_merge_20240415_1537.py

@@ -0,0 +1,13 @@
+# Generated by Django 4.2.11 on 2024-04-15 15:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0205_merge_20240410_2022"),
+        ("bookwyrm", "0205_merge_20240413_0232"),
+    ]
+
+    operations = []

bookwyrm/migrations/0207_merge_20240629_0626.py

@@ -0,0 +1,13 @@
+# Generated by Django 4.2.11 on 2024-06-29 06:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0189_importjob_create_shelves"),
+        ("bookwyrm", "0206_merge_20240415_1537"),
+    ]
+
+    operations = []

bookwyrm/migrations/0207_sqlparse_update.py

@@ -0,0 +1,51 @@
+# Generated by Django 4.2.11 on 2024-07-27 18:18
+
+from django.db import migrations, models
+import pgtrigger.compiler
+import pgtrigger.migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0206_merge_20240415_1537"),
+    ]
+
+    operations = [
+        pgtrigger.migrations.RemoveTrigger(
+            model_name="author",
+            name="reset_book_search_vector_on_author_edit",
+        ),
+        pgtrigger.migrations.RemoveTrigger(
+            model_name="book",
+            name="update_search_vector_on_book_edit",
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="author",
+            trigger=pgtrigger.compiler.Trigger(
+                name="reset_book_search_vector_on_author_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;",
+                    hash="4eeb17d1c9c53f543615bcae1234bd0260adefcc",
+                    operation='UPDATE OF "name", "aliases"',
+                    pgid="pgtrigger_reset_book_search_vector_on_author_edit_a50c7",
+                    table="bookwyrm_author",
+                    when="AFTER",
+                ),
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name="book",
+            trigger=pgtrigger.compiler.Trigger(
+                name="update_search_vector_on_book_edit",
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func="WITH author_names AS (SELECT array_to_string(bookwyrm_author.name || bookwyrm_author.aliases, ' ') AS name_and_aliases FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id) SELECT setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(name_and_aliases), ' '), '')), 'C') FROM author_names) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D') INTO new.search_vector;RETURN NEW;",
+                    hash="676d929ce95beff671544b6add09cf9360b6f299",
+                    operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"',
+                    pgid="pgtrigger_update_search_vector_on_book_edit_bec58",
+                    table="bookwyrm_book",
+                    when="BEFORE",
+                ),
+            ),
+        ),
+    ]

bookwyrm/migrations/0208_merge_0207_merge_20240629_0626_0207_sqlparse_update.py

@@ -0,0 +1,13 @@
+# Generated by Django 4.2.11 on 2024-07-28 11:07
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0207_merge_20240629_0626"),
+        ("bookwyrm", "0207_sqlparse_update"),
+    ]
+
+    operations = []

bookwyrm/migrations/0209_user_show_ratings.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.15 on 2024-08-24 01:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0208_merge_0207_merge_20240629_0626_0207_sqlparse_update"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="show_ratings",
+            field=models.BooleanField(default=True),
+        ),
+    ]

bookwyrm/models/activitypub_mixin.py

@@ -31,7 +31,7 @@ logger = logging.getLogger(__name__)
 
 PropertyField = namedtuple("PropertyField", ("set_activity_from_field"))
 
-# pylint: disable=invalid-name
+
 def set_activity_from_property_field(activity, obj, field):
     """assign a model property value to the activity json"""
     activity[field[1]] = getattr(obj, field[0])
@@ -169,7 +169,7 @@ class ActivitypubMixin:
             # filter users first by whether they're using the desired software
             # this lets us send book updates only to other bw servers
             if software:
-                queryset = queryset.filter(bookwyrm_user=(software == "bookwyrm"))
+                queryset = queryset.filter(bookwyrm_user=software == "bookwyrm")
             # if there's a user, we only want to send to the user's followers
             if user:
                 queryset = queryset.filter(following=user)
@@ -206,14 +206,10 @@ class ObjectMixin(ActivitypubMixin):
         created: Optional[bool] = None,
         software: Any = None,
         priority: str = BROADCAST,
+        broadcast: bool = True,
         **kwargs: Any,
     ) -> None:
         """broadcast created/updated/deleted objects as appropriate"""
-        broadcast = kwargs.get("broadcast", True)
-        # this bonus kwarg would cause an error in the base save method
-        if "broadcast" in kwargs:
-            del kwargs["broadcast"]
-
         created = created or not bool(self.id)
         # first off, we want to save normally no matter what
         super().save(*args, **kwargs)

bookwyrm/models/author.py

@@ -1,22 +1,25 @@
 """ database schema for info about authors """
+
 import re
-from typing import Tuple, Any
+from typing import Any
 
 from django.db import models
 from django.contrib.postgres.indexes import GinIndex
 import pgtrigger
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from bookwyrm.utils.db import format_trigger
 
-from .book import BookDataModel
+from .book import BookDataModel, MergedAuthor
 from . import fields
 
 
 class Author(BookDataModel):
     """basic biographic info"""
 
+    merged_model = MergedAuthor
+
     wikipedia_link = fields.CharField(
         max_length=255, blank=True, null=True, deduplication_field=True
     )
@@ -42,12 +45,12 @@ class Author(BookDataModel):
     )
     bio = fields.HtmlField(null=True, blank=True)
 
-    def save(self, *args: Tuple[Any, ...], **kwargs: dict[str, Any]) -> None:
+    def save(self, *args: Any, **kwargs: Any) -> None:
         """normalize isni format"""
-        if self.isni:
+        if self.isni is not None:
             self.isni = re.sub(r"\s", "", self.isni)
 
-        return super().save(*args, **kwargs)
+        super().save(*args, **kwargs)
 
     @property
     def isni_link(self):
@@ -67,7 +70,7 @@ class Author(BookDataModel):
 
     def get_remote_id(self):
         """editions and works both use "book" instead of model_name"""
-        return f"https://{DOMAIN}/author/{self.id}"
+        return f"{BASE_URL}/author/{self.id}"
 
     class Meta:
         """sets up indexes and triggers"""

bookwyrm/models/base_model.py

@@ -10,7 +10,7 @@ from django.http import Http404
 from django.utils.translation import gettext_lazy as _
 from django.utils.text import slugify
 
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from .fields import RemoteIdField
 
 
@@ -38,7 +38,7 @@ class BookWyrmModel(models.Model):
 
     def get_remote_id(self):
         """generate the url that resolves to the local object, without a slug"""
-        base_path = f"https://{DOMAIN}"
+        base_path = BASE_URL
         if hasattr(self, "user"):
             base_path = f"{base_path}{self.user.local_path}"
 
@@ -53,7 +53,7 @@ class BookWyrmModel(models.Model):
     @property
     def local_path(self):
         """how to link to this object in the local app, with a slug"""
-        local = self.get_remote_id().replace(f"https://{DOMAIN}", "")
+        local = self.get_remote_id().replace(BASE_URL, "")
 
         name = None
         if hasattr(self, "name_field"):

bookwyrm/models/book.py

@@ -1,13 +1,15 @@
 """ database schema for books and shelves """
+
 from itertools import chain
 import re
-from typing import Any
+from typing import Any, Dict, Optional, Iterable
+from typing_extensions import Self
 
 from django.contrib.postgres.search import SearchVectorField
 from django.contrib.postgres.indexes import GinIndex
 from django.core.cache import cache
 from django.db import models, transaction
-from django.db.models import Prefetch
+from django.db.models import Prefetch, ManyToManyField
 from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
 from model_utils import FieldTracker
@@ -19,13 +21,13 @@ from bookwyrm import activitypub
 from bookwyrm.isbn.isbn import hyphenator_singleton as hyphenator
 from bookwyrm.preview_images import generate_edition_preview_image_task
 from bookwyrm.settings import (
-    DOMAIN,
+    BASE_URL,
     DEFAULT_LANGUAGE,
     LANGUAGE_ARTICLES,
     ENABLE_PREVIEW_IMAGES,
     ENABLE_THUMBNAIL_GENERATION,
 )
-from bookwyrm.utils.db import format_trigger
+from bookwyrm.utils.db import format_trigger, add_update_fields
 
 from .activitypub_mixin import OrderedCollectionPageMixin, ObjectMixin
 from .base_model import BookWyrmModel
@@ -94,24 +96,133 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
 
         abstract = True
 
-    def save(self, *args: Any, **kwargs: Any) -> None:
+    def save(
+        self, *args: Any, update_fields: Optional[Iterable[str]] = None, **kwargs: Any
+    ) -> None:
         """ensure that the remote_id is within this instance"""
         if self.id:
             self.remote_id = self.get_remote_id()
+            update_fields = add_update_fields(update_fields, "remote_id")
         else:
             self.origin_id = self.remote_id
             self.remote_id = None
-        return super().save(*args, **kwargs)
+            update_fields = add_update_fields(update_fields, "origin_id", "remote_id")
+
+        super().save(*args, update_fields=update_fields, **kwargs)
 
     # pylint: disable=arguments-differ
     def broadcast(self, activity, sender, software="bookwyrm", **kwargs):
         """only send book data updates to other bookwyrm instances"""
         super().broadcast(activity, sender, software=software, **kwargs)
 
+    def merge_into(self, canonical: Self, dry_run=False) -> Dict[str, Any]:
+        """merge this entity into another entity"""
+        if canonical.id == self.id:
+            raise ValueError(f"Cannot merge {self} into itself")
+
+        absorbed_fields = canonical.absorb_data_from(self, dry_run=dry_run)
+
+        if dry_run:
+            return absorbed_fields
+
+        canonical.save()
+
+        self.merged_model.objects.create(deleted_id=self.id, merged_into=canonical)
+
+        # move related models to canonical
+        related_models = [
+            (r.remote_field.name, r.related_model) for r in self._meta.related_objects
+        ]
+        for related_field, related_model in related_models:
+            # Skip the ManyToMany fields that aren’t auto-created. These
+            # should have a corresponding OneToMany field in the model for
+            # the linking table anyway. If we update it through that model
+            # instead then we won’t lose the extra fields in the linking
+            # table.
+            # pylint: disable=protected-access
+            related_field_obj = related_model._meta.get_field(related_field)
+            if isinstance(related_field_obj, ManyToManyField):
+                through = related_field_obj.remote_field.through
+                if not through._meta.auto_created:
+                    continue
+            related_objs = related_model.objects.filter(**{related_field: self})
+            for related_obj in related_objs:
+                try:
+                    setattr(related_obj, related_field, canonical)
+                    related_obj.save()
+                except TypeError:
+                    getattr(related_obj, related_field).add(canonical)
+                    getattr(related_obj, related_field).remove(self)
+
+        self.delete()
+        return absorbed_fields
+
+    def absorb_data_from(self, other: Self, dry_run=False) -> Dict[str, Any]:
+        """fill empty fields with values from another entity"""
+        absorbed_fields = {}
+        for data_field in self._meta.get_fields():
+            if not hasattr(data_field, "activitypub_field"):
+                continue
+            canonical_value = getattr(self, data_field.name)
+            other_value = getattr(other, data_field.name)
+            if not other_value:
+                continue
+            if isinstance(data_field, fields.ArrayField):
+                if new_values := list(set(other_value) - set(canonical_value)):
+                    # append at the end (in no particular order)
+                    if not dry_run:
+                        setattr(self, data_field.name, canonical_value + new_values)
+                    absorbed_fields[data_field.name] = new_values
+            elif isinstance(data_field, fields.PartialDateField):
+                if (
+                    (not canonical_value)
+                    or (other_value.has_day and not canonical_value.has_day)
+                    or (other_value.has_month and not canonical_value.has_month)
+                ):
+                    if not dry_run:
+                        setattr(self, data_field.name, other_value)
+                    absorbed_fields[data_field.name] = other_value
+            else:
+                if not canonical_value:
+                    if not dry_run:
+                        setattr(self, data_field.name, other_value)
+                    absorbed_fields[data_field.name] = other_value
+        return absorbed_fields
+
+
+class MergedBookDataModel(models.Model):
+    """a BookDataModel instance that has been merged into another instance. kept
+    to be able to redirect old URLs"""
+
+    deleted_id = models.IntegerField(primary_key=True)
+
+    class Meta:
+        """abstract just like BookDataModel"""
+
+        abstract = True
+
+
+class MergedBook(MergedBookDataModel):
+    """an Book that has been merged into another one"""
+
+    merged_into = models.ForeignKey(
+        "Book", on_delete=models.PROTECT, related_name="absorbed"
+    )
+
+
+class MergedAuthor(MergedBookDataModel):
+    """an Author that has been merged into another one"""
+
+    merged_into = models.ForeignKey(
+        "Author", on_delete=models.PROTECT, related_name="absorbed"
+    )
+
 
 class Book(BookDataModel):
     """a generic book, which can mean either an edition or a work"""
 
+    merged_model = MergedBook
+
     connector = models.ForeignKey("Connector", on_delete=models.PROTECT, null=True)
 
     # book/work metadata
@@ -192,9 +303,13 @@ class Book(BookDataModel):
         """properties of this edition, as a string"""
         items = [
             self.physical_format if hasattr(self, "physical_format") else None,
-            f"{self.languages[0]} language"
-            if self.languages and self.languages[0] and self.languages[0] != "English"
-            else None,
+            (
+                f"{self.languages[0]} language"
+                if self.languages
+                and self.languages[0]
+                and self.languages[0] != "English"
+                else None
+            ),
             str(self.published_date.year) if self.published_date else None,
             ", ".join(self.publishers) if hasattr(self, "publishers") else None,
         ]
@@ -212,11 +327,11 @@ class Book(BookDataModel):
         if not isinstance(self, (Edition, Work)):
             raise ValueError("Books should be added as Editions or Works")
 
-        return super().save(*args, **kwargs)
+        super().save(*args, **kwargs)
 
     def get_remote_id(self):
         """editions and works both use "book" instead of model_name"""
-        return f"https://{DOMAIN}/book/{self.id}"
+        return f"{BASE_URL}/book/{self.id}"
 
     def guess_sort_title(self):
         """Get a best-guess sort title for the current book"""
@@ -289,10 +404,11 @@ class Work(OrderedCollectionPageMixin, Book):
 
     def save(self, *args, **kwargs):
         """set some fields on the edition object"""
+        super().save(*args, **kwargs)
+
         # set rank
         for edition in self.editions.all():
             edition.save()
-        return super().save(*args, **kwargs)
 
     @property
     def default_edition(self):
@@ -398,33 +514,48 @@ class Edition(Book):
         # max rank is 9
         return rank
 
-    def save(self, *args: Any, **kwargs: Any) -> None:
+    def save(
+        self, *args: Any, update_fields: Optional[Iterable[str]] = None, **kwargs: Any
+    ) -> None:
         """set some fields on the edition object"""
         # calculate isbn 10/13
-        if self.isbn_13 and self.isbn_13[:3] == "978" and not self.isbn_10:
+        if (
+            self.isbn_10 is None
+            and self.isbn_13 is not None
+            and self.isbn_13[:3] == "978"
+        ):
             self.isbn_10 = isbn_13_to_10(self.isbn_13)
-        if self.isbn_10 and not self.isbn_13:
+            update_fields = add_update_fields(update_fields, "isbn_10")
+        if self.isbn_13 is None and self.isbn_10 is not None:
             self.isbn_13 = isbn_10_to_13(self.isbn_10)
+            update_fields = add_update_fields(update_fields, "isbn_13")
 
         # normalize isbn format
-        if self.isbn_10:
+        if self.isbn_10 is not None:
             self.isbn_10 = normalize_isbn(self.isbn_10)
-        if self.isbn_13:
+        if self.isbn_13 is not None:
             self.isbn_13 = normalize_isbn(self.isbn_13)
 
         # set rank
-        self.edition_rank = self.get_rank()
-
-        # clear author cache
-        if self.id:
-            for author_id in self.authors.values_list("id", flat=True):
-                cache.delete(f"author-books-{author_id}")
+        if (new := self.get_rank()) != self.edition_rank:
+            self.edition_rank = new
+            update_fields = add_update_fields(update_fields, "edition_rank")
 
         # Create sort title by removing articles from title
         if self.sort_title in [None, ""]:
             self.sort_title = self.guess_sort_title()
+            update_fields = add_update_fields(update_fields, "sort_title")
 
-        return super().save(*args, **kwargs)
+        super().save(*args, update_fields=update_fields, **kwargs)
+
+        # clear author cache
+        if self.id:
+            cache.delete_many(
+                [
+                    f"author-books-{author_id}"
+                    for author_id in self.authors.values_list("id", flat=True)
+                ]
+            )
 
     @transaction.atomic
     def repair(self):

bookwyrm/models/bookwyrm_export_job.py

@@ -1,229 +1,341 @@
 """Export user account to tar.gz file for import into another Bookwyrm instance"""
 
-import dataclasses
 import logging
-from uuid import uuid4
+import os
 
-from django.db.models import FileField
-from django.db.models import Q
+from boto3.session import Session as BotoSession
+from s3_tar import S3Tar
+
+from django.db.models import BooleanField, FileField, JSONField
 from django.core.serializers.json import DjangoJSONEncoder
 from django.core.files.base import ContentFile
+from django.core.files.storage import storages
 
-from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, List, ListItem
+from bookwyrm import settings
+
+from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, ListItem
 from bookwyrm.models import Review, Comment, Quotation
 from bookwyrm.models import Edition
 from bookwyrm.models import UserFollows, User, UserBlocks
-from bookwyrm.models.job import ParentJob, ParentTask
+from bookwyrm.models.job import ParentJob
 from bookwyrm.tasks import app, IMPORTS
 from bookwyrm.utils.tar import BookwyrmTarFile
 
 logger = logging.getLogger(__name__)
 
 
+class BookwyrmAwsSession(BotoSession):
+    """a boto session that always uses settings.AWS_S3_ENDPOINT_URL"""
+
+    def client(self, *args, **kwargs):
+        kwargs["endpoint_url"] = settings.AWS_S3_ENDPOINT_URL
+        return super().client("s3", *args, **kwargs)
+
+
+def select_exports_storage():
+    """callable to allow for dependency on runtime configuration"""
+    return storages["exports"]
+
+
 class BookwyrmExportJob(ParentJob):
     """entry for a specific request to export a bookwyrm user"""
 
-    export_data = FileField(null=True)
+    export_data = FileField(null=True, storage=select_exports_storage)
+    export_json = JSONField(null=True, encoder=DjangoJSONEncoder)
+    json_completed = BooleanField(default=False)
 
     def start_job(self):
-        """Start the job"""
-        start_export_task.delay(job_id=self.id, no_children=True)
+        """schedule the first task"""
 
-        return self
+        task = create_export_json_task.delay(job_id=self.id)
+        self.task_id = task.id
+        self.save(update_fields=["task_id"])
 
 
-@app.task(queue=IMPORTS, base=ParentTask)
-def start_export_task(**kwargs):
-    """trigger the child tasks for each row"""
-    job = BookwyrmExportJob.objects.get(id=kwargs["job_id"])
+@app.task(queue=IMPORTS)
+def create_export_json_task(job_id):
+    """create the JSON data for the export"""
+
+    job = BookwyrmExportJob.objects.get(id=job_id)
 
     # don't start the job if it was stopped from the UI
     if job.complete:
         return
+
     try:
-        # This is where ChildJobs get made
-        job.export_data = ContentFile(b"", str(uuid4()))
-        json_data = json_export(job.user)
-        tar_export(json_data, job.user, job.export_data)
-        job.save(update_fields=["export_data"])
+        job.set_status("active")
+
+        # generate JSON structure
+        job.export_json = export_json(job.user)
+        job.save(update_fields=["export_json"])
+
+        # create archive in separate task
+        create_archive_task.delay(job_id=job.id)
     except Exception as err:  # pylint: disable=broad-except
-        logger.exception("User Export Job %s Failed with error: %s", job.id, err)
+        logger.exception(
+            "create_export_json_task for %s failed with error: %s", job, err
+        )
         job.set_status("failed")
 
-    job.set_status("complete")
+
+def archive_file_location(file, directory="") -> str:
+    """get the relative location of a file inside the archive"""
+    return os.path.join(directory, file.name)
 
 
-def tar_export(json_data: str, user, file):
-    """wrap the export information in a tar file"""
-    file.open("wb")
-    with BookwyrmTarFile.open(mode="w:gz", fileobj=file) as tar:
-        tar.write_bytes(json_data.encode("utf-8"))
+def add_file_to_s3_tar(s3_tar: S3Tar, storage, file, directory=""):
+    """
+    add file to S3Tar inside directory, keeping any directories under its
+    storage location
+    """
+    s3_tar.add_file(
+        os.path.join(storage.location, file.name),
+        folder=os.path.dirname(archive_file_location(file, directory=directory)),
+    )
 
-        # Add avatar image if present
-        if getattr(user, "avatar", False):
-            tar.add_image(user.avatar, filename="avatar")
 
+@app.task(queue=IMPORTS)
+def create_archive_task(job_id):
+    """create the archive containing the JSON file and additional files"""
+
+    job = BookwyrmExportJob.objects.get(id=job_id)
+
+    # don't start the job if it was stopped from the UI
+    if job.complete:
+        return
+
+    try:
+        export_task_id = str(job.task_id)
+        archive_filename = f"{export_task_id}.tar.gz"
+        export_json_bytes = DjangoJSONEncoder().encode(job.export_json).encode("utf-8")
+
+        user = job.user
         editions = get_books_for_user(user)
-        for book in editions:
-            if getattr(book, "cover", False):
-                tar.add_image(book.cover)
 
-    file.close()
+        if settings.USE_S3:
+            # Storage for writing temporary files
+            exports_storage = storages["exports"]
+
+            # Handle for creating the final archive
+            s3_tar = S3Tar(
+                exports_storage.bucket_name,
+                os.path.join(exports_storage.location, archive_filename),
+                session=BookwyrmAwsSession(),
+            )
+
+            # Save JSON file to a temporary location
+            export_json_tmp_file = os.path.join(export_task_id, "archive.json")
+            exports_storage.save(
+                export_json_tmp_file,
+                ContentFile(export_json_bytes),
+            )
+            s3_tar.add_file(
+                os.path.join(exports_storage.location, export_json_tmp_file)
+            )
+
+            # Add images to TAR
+            images_storage = storages["default"]
+
+            if user.avatar:
+                add_file_to_s3_tar(s3_tar, images_storage, user.avatar)
+
+            for edition in editions:
+                if edition.cover:
+                    add_file_to_s3_tar(
+                        s3_tar, images_storage, edition.cover, directory="images"
+                    )
+
+            # Create archive and store file name
+            s3_tar.tar()
+            job.export_data = archive_filename
+            job.save(update_fields=["export_data"])
+
+            # Delete temporary files
+            exports_storage.delete(export_json_tmp_file)
+
+        else:
+            job.export_data = archive_filename
+            with job.export_data.open("wb") as tar_file:
+                with BookwyrmTarFile.open(mode="w:gz", fileobj=tar_file) as tar:
+                    # save json file
+                    tar.write_bytes(export_json_bytes)
+
+                    # Add avatar image if present
+                    if user.avatar:
+                        tar.add_image(user.avatar)
+
+                    for edition in editions:
+                        if edition.cover:
+                            tar.add_image(edition.cover, directory="images")
+            job.save(update_fields=["export_data"])
+
+        job.set_status("completed")
+
+    except Exception as err:  # pylint: disable=broad-except
+        logger.exception("create_archive_task for %s failed with error: %s", job, err)
+        job.set_status("failed")
 
 
-def json_export(
-    user,
-):  # pylint: disable=too-many-locals, too-many-statements, too-many-branches
-    """Generate an export for a user"""
+def export_json(user: User):
+    """create export JSON"""
+    data = export_user(user)  # in the root of the JSON structure
+    data["settings"] = export_settings(user)
+    data["goals"] = export_goals(user)
+    data["books"] = export_books(user)
+    data["saved_lists"] = export_saved_lists(user)
+    data["follows"] = export_follows(user)
+    data["blocks"] = export_blocks(user)
+    return data
 
-    # User as AP object
-    exported_user = user.to_activity()
-    # I don't love this but it prevents a JSON encoding error
-    # when there is no user image
-    if exported_user.get("icon") in (None, dataclasses.MISSING):
-        exported_user["icon"] = {}
+
+def export_user(user: User):
+    """export user data"""
+    data = user.to_activity()
+    if user.avatar:
+        data["icon"]["url"] = archive_file_location(user.avatar)
     else:
-        # change the URL to be relative to the JSON file
-        file_type = exported_user["icon"]["url"].rsplit(".", maxsplit=1)[-1]
-        filename = f"avatar.{file_type}"
-        exported_user["icon"]["url"] = filename
+        data["icon"] = {}
+    return data
 
-    # Additional settings - can't be serialized as AP
+
+def export_settings(user: User):
+    """Additional settings - can't be serialized as AP"""
     vals = [
         "show_goal",
         "preferred_timezone",
         "default_post_privacy",
         "show_suggested_users",
     ]
-    exported_user["settings"] = {}
-    for k in vals:
-        exported_user["settings"][k] = getattr(user, k)
+    return {k: getattr(user, k) for k in vals}
 
-    # Reading goals - can't be serialized as AP
-    reading_goals = AnnualGoal.objects.filter(user=user).distinct()
-    exported_user["goals"] = []
-    for goal in reading_goals:
-        exported_user["goals"].append(
-            {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy}
-        )
 
-    # Reading history - can't be serialized as AP
-    readthroughs = ReadThrough.objects.filter(user=user).distinct().values()
-    readthroughs = list(readthroughs)
+def export_saved_lists(user: User):
+    """add user saved lists to export JSON"""
+    return [l.remote_id for l in user.saved_lists.all()]
 
-    # Books
-    editions = get_books_for_user(user)
-    exported_user["books"] = []
 
-    for edition in editions:
-        book = {}
-        book["work"] = edition.parent_work.to_activity()
-        book["edition"] = edition.to_activity()
-
-        if book["edition"].get("cover"):
-            # change the URL to be relative to the JSON file
-            filename = book["edition"]["cover"]["url"].rsplit("/", maxsplit=1)[-1]
-            book["edition"]["cover"]["url"] = f"covers/{filename}"
-
-        # authors
-        book["authors"] = []
-        for author in edition.authors.all():
-            book["authors"].append(author.to_activity())
-
-        # Shelves this book is on
-        # Every ShelfItem is this book so we don't other serializing
-        book["shelves"] = []
-        shelf_books = (
-            ShelfBook.objects.select_related("shelf")
-            .filter(user=user, book=edition)
-            .distinct()
-        )
-
-        for shelfbook in shelf_books:
-            book["shelves"].append(shelfbook.shelf.to_activity())
-
-        # Lists and ListItems
-        # ListItems include "notes" and "approved" so we need them
-        # even though we know it's this book
-        book["lists"] = []
-        list_items = ListItem.objects.filter(book=edition, user=user).distinct()
-
-        for item in list_items:
-            list_info = item.book_list.to_activity()
-            list_info[
-                "privacy"
-            ] = item.book_list.privacy  # this isn't serialized so we add it
-            list_info["list_item"] = item.to_activity()
-            book["lists"].append(list_info)
-
-        # Statuses
-        # Can't use select_subclasses here because
-        # we need to filter on the "book" value,
-        # which is not available on an ordinary Status
-        for status in ["comments", "quotations", "reviews"]:
-            book[status] = []
-
-        comments = Comment.objects.filter(user=user, book=edition).all()
-        for status in comments:
-            obj = status.to_activity()
-            obj["progress"] = status.progress
-            obj["progress_mode"] = status.progress_mode
-            book["comments"].append(obj)
-
-        quotes = Quotation.objects.filter(user=user, book=edition).all()
-        for status in quotes:
-            obj = status.to_activity()
-            obj["position"] = status.position
-            obj["endposition"] = status.endposition
-            obj["position_mode"] = status.position_mode
-            book["quotations"].append(obj)
-
-        reviews = Review.objects.filter(user=user, book=edition).all()
-        for status in reviews:
-            obj = status.to_activity()
-            book["reviews"].append(obj)
-
-        # readthroughs can't be serialized to activity
-        book_readthroughs = (
-            ReadThrough.objects.filter(user=user, book=edition).distinct().values()
-        )
-        book["readthroughs"] = list(book_readthroughs)
-
-        # append everything
-        exported_user["books"].append(book)
-
-    # saved book lists - just the remote id
-    saved_lists = List.objects.filter(id__in=user.saved_lists.all()).distinct()
-    exported_user["saved_lists"] = [l.remote_id for l in saved_lists]
-
-    # follows - just the remote id
+def export_follows(user: User):
+    """add user follows to export JSON"""
     follows = UserFollows.objects.filter(user_subject=user).distinct()
     following = User.objects.filter(userfollows_user_object__in=follows).distinct()
-    exported_user["follows"] = [f.remote_id for f in following]
+    return [f.remote_id for f in following]
 
-    # blocks - just the remote id
+
+def export_blocks(user: User):
+    """add user blocks to export JSON"""
     blocks = UserBlocks.objects.filter(user_subject=user).distinct()
     blocking = User.objects.filter(userblocks_user_object__in=blocks).distinct()
+    return [b.remote_id for b in blocking]
 
-    exported_user["blocks"] = [b.remote_id for b in blocking]
 
-    return DjangoJSONEncoder().encode(exported_user)
+def export_goals(user: User):
+    """add user reading goals to export JSON"""
+    reading_goals = AnnualGoal.objects.filter(user=user).distinct()
+    return [
+        {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy}
+        for goal in reading_goals
+    ]
+
+
+def export_books(user: User):
+    """add books to export JSON"""
+    editions = get_books_for_user(user)
+    return [export_book(user, edition) for edition in editions]
+
+
+def export_book(user: User, edition: Edition):
+    """add book to export JSON"""
+    data = {}
+    data["work"] = edition.parent_work.to_activity()
+    data["edition"] = edition.to_activity()
+
+    if edition.cover:
+        data["edition"]["cover"]["url"] = archive_file_location(
+            edition.cover, directory="images"
+        )
+
+    # authors
+    data["authors"] = [author.to_activity() for author in edition.authors.all()]
+
+    # Shelves this book is on
+    # Every ShelfItem is this book so we don't other serializing
+    shelf_books = (
+        ShelfBook.objects.select_related("shelf")
+        .filter(user=user, book=edition)
+        .distinct()
+    )
+    data["shelves"] = [shelfbook.shelf.to_activity() for shelfbook in shelf_books]
+
+    # Lists and ListItems
+    # ListItems include "notes" and "approved" so we need them
+    # even though we know it's this book
+    list_items = ListItem.objects.filter(book=edition, user=user).distinct()
+
+    data["lists"] = []
+    for item in list_items:
+        list_info = item.book_list.to_activity()
+        list_info[
+            "privacy"
+        ] = item.book_list.privacy  # this isn't serialized so we add it
+        list_info["list_item"] = item.to_activity()
+        data["lists"].append(list_info)
+
+    # Statuses
+    # Can't use select_subclasses here because
+    # we need to filter on the "book" value,
+    # which is not available on an ordinary Status
+    for status in ["comments", "quotations", "reviews"]:
+        data[status] = []
+
+    comments = Comment.objects.filter(user=user, book=edition).all()
+    for status in comments:
+        obj = status.to_activity()
+        obj["progress"] = status.progress
+        obj["progress_mode"] = status.progress_mode
+        data["comments"].append(obj)
+
+    quotes = Quotation.objects.filter(user=user, book=edition).all()
+    for status in quotes:
+        obj = status.to_activity()
+        obj["position"] = status.position
+        obj["endposition"] = status.endposition
+        obj["position_mode"] = status.position_mode
+        data["quotations"].append(obj)
+
+    reviews = Review.objects.filter(user=user, book=edition).all()
+    data["reviews"] = [status.to_activity() for status in reviews]
+
+    # readthroughs can't be serialized to activity
+    book_readthroughs = (
+        ReadThrough.objects.filter(user=user, book=edition).distinct().values()
+    )
+    data["readthroughs"] = list(book_readthroughs)
+    return data
 
 
 def get_books_for_user(user):
-    """Get all the books and editions related to a user"""
+    """
+    Get all the books and editions related to a user.
 
-    editions = (
-        Edition.objects.select_related("parent_work")
-        .filter(
-            Q(shelves__user=user)
-            | Q(readthrough__user=user)
-            | Q(review__user=user)
-            | Q(list__user=user)
-            | Q(comment__user=user)
-            | Q(quotation__user=user)
-        )
-        .distinct()
+    We use union() instead of Q objects because it creates
+    multiple simple queries in stead of a much more complex DB query
+    that can time out.
+
+    """
+
+    shelf_eds = Edition.objects.select_related("parent_work").filter(shelves__user=user)
+    rt_eds = Edition.objects.select_related("parent_work").filter(
+        readthrough__user=user
+    )
+    review_eds = Edition.objects.select_related("parent_work").filter(review__user=user)
+    list_eds = Edition.objects.select_related("parent_work").filter(list__user=user)
+    comment_eds = Edition.objects.select_related("parent_work").filter(
+        comment__user=user
+    )
+    quote_eds = Edition.objects.select_related("parent_work").filter(
+        quotation__user=user
     )
 
+    editions = shelf_eds.union(rt_eds, review_eds, list_eds, comment_eds, quote_eds)
+
     return editions

bookwyrm/models/bookwyrm_import_job.py

@@ -42,20 +42,23 @@ def start_import_task(**kwargs):
     try:
         archive_file.open("rb")
         with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar:
-            job.import_data = json.loads(tar.read("archive.json").decode("utf-8"))
+            json_filename = next(
+                filter(lambda n: n.startswith("archive"), tar.getnames())
+            )
+            job.import_data = json.loads(tar.read(json_filename).decode("utf-8"))
 
             if "include_user_profile" in job.required:
                 update_user_profile(job.user, tar, job.import_data)
             if "include_user_settings" in job.required:
                 update_user_settings(job.user, job.import_data)
             if "include_goals" in job.required:
-                update_goals(job.user, job.import_data.get("goals"))
+                update_goals(job.user, job.import_data.get("goals", []))
             if "include_saved_lists" in job.required:
-                upsert_saved_lists(job.user, job.import_data.get("saved_lists"))
+                upsert_saved_lists(job.user, job.import_data.get("saved_lists", []))
             if "include_follows" in job.required:
-                upsert_follows(job.user, job.import_data.get("follows"))
+                upsert_follows(job.user, job.import_data.get("follows", []))
             if "include_blocks" in job.required:
-                upsert_user_blocks(job.user, job.import_data.get("blocks"))
+                upsert_user_blocks(job.user, job.import_data.get("blocks", []))
 
             process_books(job, tar)
 
@@ -212,7 +215,7 @@ def upsert_statuses(user, cls, data, book_remote_id):
                 instance.save()  # save and broadcast
 
         else:
-            logger.info("User does not have permission to import statuses")
+            logger.warning("User does not have permission to import statuses")
 
 
 def upsert_lists(user, lists, book_id):

bookwyrm/models/connector.py

@@ -11,7 +11,7 @@ ConnectorFiles = models.TextChoices("ConnectorFiles", CONNECTORS)
 class Connector(BookWyrmModel):
     """book data source connectors"""
 
-    identifier = models.CharField(max_length=255, unique=True)
+    identifier = models.CharField(max_length=255, unique=True)  # domain
     priority = models.IntegerField(default=2)
     name = models.CharField(max_length=255, null=True, blank=True)
     connector_file = models.CharField(max_length=255, choices=ConnectorFiles.choices)

bookwyrm/models/federated_server.py

@@ -16,7 +16,7 @@ FederationStatus = [
 class FederatedServer(BookWyrmModel):
     """store which servers we federate with"""
 
-    server_name = models.CharField(max_length=255, unique=True)
+    server_name = models.CharField(max_length=255, unique=True)  # domain
     status = models.CharField(
         max_length=255, default="federated", choices=FederationStatus
     )
@@ -64,5 +64,4 @@ class FederatedServer(BookWyrmModel):
     def is_blocked(cls, url: str) -> bool:
         """look up if a domain is blocked"""
         url = urlparse(url)
-        domain = url.netloc
-        return cls.objects.filter(server_name=domain, status="blocked").exists()
+        return cls.objects.filter(server_name=url.hostname, status="blocked").exists()

bookwyrm/models/fields.py

@@ -193,8 +193,7 @@ class UsernameField(ActivitypubFieldMixin, models.CharField):
 
     def __init__(self, activitypub_field="preferredUsername", **kwargs):
         self.activitypub_field = activitypub_field
-        # I don't totally know why pylint is mad at this, but it makes it work
-        super(ActivitypubFieldMixin, self).__init__(  # pylint: disable=bad-super-call
+        super(ActivitypubFieldMixin, self).__init__(
             _("username"),
             max_length=150,
             unique=True,
@@ -234,7 +233,6 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, max_length=255, choices=PrivacyLevels, default="public")
 
-    # pylint: disable=invalid-name
     def set_field_from_activity(
         self, instance, data, overwrite=True, allow_external_connections=True
     ):
@@ -276,7 +274,6 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
         if hasattr(instance, "mention_users"):
             mentions = [u.remote_id for u in instance.mention_users.all()]
         # this is a link to the followers list
-        # pylint: disable=protected-access
         followers = instance.user.followers_url
         if instance.privacy == "public":
             activity["to"] = [self.public]
@@ -444,7 +441,7 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
         self.alt_field = alt_field
         super().__init__(*args, **kwargs)
 
-    # pylint: disable=arguments-differ,arguments-renamed,too-many-arguments
+    # pylint: disable=arguments-renamed,too-many-arguments
     def set_field_from_activity(
         self, instance, data, save=True, overwrite=True, allow_external_connections=True
     ):

bookwyrm/models/group.py

@@ -1,7 +1,7 @@
 """ do book related things with other users """
 from django.db import models, IntegrityError, transaction
 from django.db.models import Q
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from .base_model import BookWyrmModel
 from . import fields
 from .relationship import UserBlocks
@@ -17,7 +17,7 @@ class Group(BookWyrmModel):
 
     def get_remote_id(self):
         """don't want the user to be in there in this case"""
-        return f"https://{DOMAIN}/group/{self.id}"
+        return f"{BASE_URL}/group/{self.id}"
 
     @classmethod
     def followers_filter(cls, queryset, viewer):

bookwyrm/models/hashtag.py

@@ -2,18 +2,19 @@
 from bookwyrm import activitypub
 from .activitypub_mixin import ActivitypubMixin
 from .base_model import BookWyrmModel
-from .fields import CICharField
+from .fields import CharField
 
 
 class Hashtag(ActivitypubMixin, BookWyrmModel):
     "a hashtag which can be used in statuses"
 
-    name = CICharField(
+    name = CharField(
         max_length=256,
         blank=False,
         null=False,
         activitypub_field="name",
         deduplication_field=True,
+        db_collation="case_insensitive",
     )
 
     name_field = "name"

bookwyrm/models/import_job.py

@@ -4,6 +4,7 @@ import math
 import re
 import dateutil.parser
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
@@ -59,6 +60,7 @@ class ImportJob(models.Model):
     created_date = models.DateTimeField(default=timezone.now)
     updated_date = models.DateTimeField(default=timezone.now)
     include_reviews: bool = models.BooleanField(default=True)
+    create_shelves: bool = models.BooleanField(default=True)
     mappings = models.JSONField()
     source = models.CharField(max_length=100)
     privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels)
@@ -245,11 +247,26 @@ class ImportItem(models.Model):
         """the goodreads shelf field"""
         return self.normalized_data.get("shelf")
 
+    @property
+    def shelf_name(self):
+        """the goodreads shelf field"""
+        return self.normalized_data.get("shelf_name")
+
     @property
     def review(self):
         """a user-written review, to be imported with the book data"""
         return self.normalized_data.get("review_body")
 
+    @property
+    def review_name(self):
+        """a user-written review name, to be imported with the book data"""
+        return self.normalized_data.get("review_name")
+
+    @property
+    def review_published(self):
+        """date the review was published - included in BookWyrm export csv"""
+        return self.normalized_data.get("review_published", None)
+
     @property
     def rating(self):
         """x/5 star rating for a book"""
@@ -352,7 +369,7 @@ def import_item_task(item_id):
 
     try:
         item.resolve()
-    except Exception as err:  # pylint: disable=broad-except
+    except Exception as err:
         item.fail_reason = _("Error loading book")
         item.save()
         item.update_job()
@@ -368,7 +385,7 @@ def import_item_task(item_id):
     item.update_job()
 
 
-def handle_imported_book(item):
+def handle_imported_book(item):  # pylint: disable=too-many-branches
     """process a csv and then post about it"""
     job = item.job
     if job.complete:
@@ -385,13 +402,31 @@ def handle_imported_book(item):
         item.book = item.book.edition
 
     existing_shelf = ShelfBook.objects.filter(book=item.book, user=user).exists()
+    if job.create_shelves and item.shelf and not existing_shelf:
+        # shelve the book if it hasn't been shelved already
 
-    # shelve the book if it hasn't been shelved already
-    if item.shelf and not existing_shelf:
-        desired_shelf = Shelf.objects.get(identifier=item.shelf, user=user)
         shelved_date = item.date_added or timezone.now()
+        shelfname = getattr(item, "shelf_name", item.shelf)
+
+        try:
+            shelf = Shelf.objects.get(name=shelfname, user=user)
+        except ObjectDoesNotExist:
+            try:
+                shelf = Shelf.objects.get(identifier=item.shelf, user=user)
+            except ObjectDoesNotExist:
+
+                shelf = Shelf.objects.create(
+                    user=user,
+                    identifier=item.shelf,
+                    name=shelfname,
+                    privacy=job.privacy,
+                )
+
         ShelfBook(
-            book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date
+            book=item.book,
+            shelf=shelf,
+            user=user,
+            shelved_date=shelved_date,
         ).save(priority=IMPORT_TRIGGERED)
 
     for read in item.reads:
@@ -408,19 +443,25 @@ def handle_imported_book(item):
         read.save()
 
     if job.include_reviews and (item.rating or item.review) and not item.linked_review:
-        # we don't know the publication date of the review,
-        # but "now" is a bad guess
-        published_date_guess = item.date_read or item.date_added
+        # we don't necessarily know the publication date of the review,
+        # but "now" is a bad guess unless we have no choice
+
+        published_date_guess = (
+            item.review_published or item.date_read or item.date_added or timezone.now()
+        )
         if item.review:
+
             # pylint: disable=consider-using-f-string
             review_title = "Review of {!r} on {!r}".format(
                 item.book.title,
                 job.source,
             )
+            review_name = getattr(item, "review_name", review_title)
+
             review = Review.objects.filter(
                 user=user,
                 book=item.book,
-                name=review_title,
+                name=review_name,
                 rating=item.rating,
                 published_date=published_date_guess,
             ).first()
@@ -428,7 +469,7 @@ def handle_imported_book(item):
                 review = Review(
                     user=user,
                     book=item.book,
-                    name=review_title,
+                    name=review_name,
                     content=item.review,
                     rating=item.rating,
                     published_date=published_date_guess,

bookwyrm/models/job.py

@@ -135,8 +135,7 @@ class ParentJob(Job):
         )
         app.control.revoke(list(tasks))
 
-        for task in self.pending_child_jobs:
-            task.update(status=self.Status.STOPPED)
+        self.pending_child_jobs.update(status=self.Status.STOPPED)
 
     @property
     def has_completed(self):
@@ -248,7 +247,7 @@ class SubTask(app.Task):
     """
 
     def before_start(
-        self, task_id, args, kwargs
+        self, task_id, *args, **kwargs
     ):  # pylint: disable=no-self-use, unused-argument
         """Handler called before the task starts. Override.
 
@@ -272,7 +271,7 @@ class SubTask(app.Task):
         child_job.set_status(ChildJob.Status.ACTIVE)
 
     def on_success(
-        self, retval, task_id, args, kwargs
+        self, retval, task_id, *args, **kwargs
     ):  # pylint: disable=no-self-use, unused-argument
         """Run by the worker if the task executes successfully. Override.
 

bookwyrm/models/link.py

@@ -1,4 +1,5 @@
 """ outlink data """
+from typing import Optional, Iterable
 from urllib.parse import urlparse
 
 from django.core.exceptions import PermissionDenied
@@ -6,6 +7,7 @@ from django.db import models
 from django.utils.translation import gettext_lazy as _
 
 from bookwyrm import activitypub
+from bookwyrm.utils.db import add_update_fields
 from .activitypub_mixin import ActivitypubMixin
 from .base_model import BookWyrmModel
 from . import fields
@@ -34,17 +36,19 @@ class Link(ActivitypubMixin, BookWyrmModel):
         """link name via the associated domain"""
         return self.domain.name
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs):
         """create a link"""
         # get or create the associated domain
         if not self.domain:
-            domain = urlparse(self.url).netloc
+            domain = urlparse(self.url).hostname
             self.domain, _ = LinkDomain.objects.get_or_create(domain=domain)
+            update_fields = add_update_fields(update_fields, "domain")
 
         # this is never broadcast, the owning model broadcasts an update
         if "broadcast" in kwargs:
             del kwargs["broadcast"]
-        return super().save(*args, **kwargs)
+
+        super().save(*args, update_fields=update_fields, **kwargs)
 
 
 AvailabilityChoices = [
@@ -88,8 +92,10 @@ class LinkDomain(BookWyrmModel):
             return
         raise PermissionDenied()
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs):
         """set a default name"""
         if not self.name:
             self.name = self.domain
-        super().save(*args, **kwargs)
+            update_fields = add_update_fields(update_fields, "name")
+
+        super().save(*args, update_fields=update_fields, **kwargs)

bookwyrm/models/list.py

@@ -1,4 +1,5 @@
 """ make a list of books!! """
+from typing import Optional, Iterable
 import uuid
 
 from django.core.exceptions import PermissionDenied
@@ -7,7 +8,8 @@ from django.db.models import Q
 from django.utils import timezone
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
+from bookwyrm.utils.db import add_update_fields
 
 from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin
 from .base_model import BookWyrmModel
@@ -50,7 +52,7 @@ class List(OrderedCollectionMixin, BookWyrmModel):
 
     def get_remote_id(self):
         """don't want the user to be in there in this case"""
-        return f"https://{DOMAIN}/list/{self.id}"
+        return f"{BASE_URL}/list/{self.id}"
 
     @property
     def collection_queryset(self):
@@ -124,11 +126,13 @@ class List(OrderedCollectionMixin, BookWyrmModel):
             group=None, curation="closed"
         )
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs):
         """on save, update embed_key and avoid clash with existing code"""
         if not self.embed_key:
             self.embed_key = uuid.uuid4()
-        super().save(*args, **kwargs)
+            update_fields = add_update_fields(update_fields, "embed_key")
+
+        super().save(*args, update_fields=update_fields, **kwargs)
 
 
 class ListItem(CollectionItemMixin, BookWyrmModel):

bookwyrm/models/move.py

@@ -10,7 +10,7 @@ from .notification import Notification, NotificationType
 
 
 class Move(ActivityMixin, BookWyrmModel):
-    """migrating an activitypub user account"""
+    """migrating an activitypub object"""
 
     user = fields.ForeignKey(
         "User", on_delete=models.PROTECT, activitypub_field="actor"
@@ -48,24 +48,21 @@ class MoveUser(Move):
         """update user info and broadcast it"""
 
         # only allow if the source is listed in the target's alsoKnownAs
-        if self.user in self.target.also_known_as.all():
-            self.user.also_known_as.add(self.target.id)
-            self.user.update_active_date()
-            self.user.moved_to = self.target.remote_id
-            self.user.save(update_fields=["moved_to"])
-
-            if self.user.local:
-                kwargs[
-                    "broadcast"
-                ] = True  # Only broadcast if we are initiating the Move
-
-            super().save(*args, **kwargs)
-
-            for follower in self.user.followers.all():
-                if follower.local:
-                    Notification.notify(
-                        follower, self.user, notification_type=NotificationType.MOVE
-                    )
-
-        else:
+        if self.user not in self.target.also_known_as.all():
             raise PermissionDenied()
+
+        self.user.also_known_as.add(self.target.id)
+        self.user.update_active_date()
+        self.user.moved_to = self.target.remote_id
+        self.user.save(update_fields=["moved_to"])
+
+        if self.user.local:
+            kwargs["broadcast"] = True  # Only broadcast if we are initiating the Move
+
+        super().save(*args, **kwargs)
+
+        for follower in self.user.followers.all():
+            if follower.local:
+                Notification.notify(
+                    follower, self.user, notification_type=NotificationType.MOVE
+                )

bookwyrm/models/readthrough.py

@@ -1,9 +1,13 @@
 """ progress in a book """
+from typing import Optional, Iterable
+
 from django.core import validators
 from django.core.cache import cache
 from django.db import models
 from django.db.models import F, Q
 
+from bookwyrm.utils.db import add_update_fields
+
 from .base_model import BookWyrmModel
 
 
@@ -30,14 +34,17 @@ class ReadThrough(BookWyrmModel):
     stopped_date = models.DateTimeField(blank=True, null=True)
     is_active = models.BooleanField(default=True)
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs):
         """update user active time"""
-        cache.delete(f"latest_read_through-{self.user_id}-{self.book_id}")
-        self.user.update_active_date()
         # an active readthrough must have an unset finish date
         if self.finish_date or self.stopped_date:
             self.is_active = False
-        super().save(*args, **kwargs)
+            update_fields = add_update_fields(update_fields, "is_active")
+
+        super().save(*args, update_fields=update_fields, **kwargs)
+
+        cache.delete(f"latest_read_through-{self.user_id}-{self.book_id}")
+        self.user.update_active_date()
 
     def create_update(self):
         """add update to the readthrough"""

bookwyrm/models/relationship.py

@@ -38,14 +38,16 @@ class UserRelationship(BookWyrmModel):
 
     def save(self, *args, **kwargs):
         """clear the template cache"""
-        clear_cache(self.user_subject, self.user_object)
         super().save(*args, **kwargs)
 
+        clear_cache(self.user_subject, self.user_object)
+
     def delete(self, *args, **kwargs):
         """clear the template cache"""
-        clear_cache(self.user_subject, self.user_object)
         super().delete(*args, **kwargs)
 
+        clear_cache(self.user_subject, self.user_object)
+
     class Meta:
         """relationships should be unique"""
 
@@ -133,7 +135,7 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
     status = "follow_request"
     activity_serializer = activitypub.Follow
 
-    def save(self, *args, broadcast=True, **kwargs):  # pylint: disable=arguments-differ
+    def save(self, *args, broadcast=True, **kwargs):
         """make sure the follow or block relationship doesn't already exist"""
         # if there's a request for a follow that already exists, accept it
         # without changing the local database state

bookwyrm/models/report.py

@@ -3,7 +3,7 @@ from django.core.exceptions import PermissionDenied
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from .base_model import BookWyrmModel
 
 
@@ -46,7 +46,7 @@ class Report(BookWyrmModel):
         raise PermissionDenied()
 
     def get_remote_id(self):
-        return f"https://{DOMAIN}/settings/reports/{self.id}"
+        return f"{BASE_URL}/settings/reports/{self.id}"
 
     def comment(self, user, note):
         """comment on a report"""

bookwyrm/models/shelf.py

@@ -1,13 +1,15 @@
 """ puttin' books on shelves """
 import re
+from typing import Optional, Iterable
 from django.core.cache import cache
 from django.core.exceptions import PermissionDenied
 from django.db import models
 from django.utils import timezone
 
 from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import BASE_URL
 from bookwyrm.tasks import BROADCAST
+from bookwyrm.utils.db import add_update_fields
 from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin
 from .base_model import BookWyrmModel
 from . import fields
@@ -44,8 +46,9 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
         """set the identifier"""
         super().save(*args, priority=priority, **kwargs)
         if not self.identifier:
+            # this needs the auto increment ID from the save() above
             self.identifier = self.get_identifier()
-            super().save(*args, **kwargs, broadcast=False)
+            super().save(*args, **kwargs, broadcast=False, update_fields={"identifier"})
 
     def get_identifier(self):
         """custom-shelf-123 for the url"""
@@ -71,7 +74,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
     @property
     def local_path(self):
         """No slugs"""
-        return self.get_remote_id().replace(f"https://{DOMAIN}", "")
+        return self.get_remote_id().replace(BASE_URL, "")
 
     def raise_not_deletable(self, viewer):
         """don't let anyone delete a default shelf"""
@@ -100,10 +103,21 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel):
     activity_serializer = activitypub.ShelfItem
     collection_field = "shelf"
 
-    def save(self, *args, priority=BROADCAST, **kwargs):
+    def save(
+        self,
+        *args,
+        priority=BROADCAST,
+        update_fields: Optional[Iterable[str]] = None,
+        **kwargs,
+    ):
         if not self.user:
             self.user = self.shelf.user
-        if self.id and self.user.local:
+            update_fields = add_update_fields(update_fields, "user")
+
+        is_update = self.id is not None
+        super().save(*args, priority=priority, update_fields=update_fields, **kwargs)
+
+        if is_update and self.user.local:
             # remove all caches related to all editions of this book
             cache.delete_many(
                 [
@@ -111,7 +125,6 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel):
                     for book in self.book.parent_work.editions.all()
                 ]
             )
-        super().save(*args, priority=priority, **kwargs)
 
     def delete(self, *args, **kwargs):
         if self.id and self.user.local:

bookwyrm/models/site.py

@@ -1,5 +1,6 @@
 """ the particulars for this instance of BookWyrm """
 import datetime
+from typing import Optional, Iterable
 from urllib.parse import urljoin
 import uuid
 
@@ -12,9 +13,10 @@ from model_utils import FieldTracker
 
 from bookwyrm.connectors.abstract_connector import get_data
 from bookwyrm.preview_images import generate_site_preview_image_task
-from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL
+from bookwyrm.settings import BASE_URL, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL
 from bookwyrm.settings import RELEASE_API
 from bookwyrm.tasks import app, MISC
+from bookwyrm.utils.db import add_update_fields
 from .base_model import BookWyrmModel, new_access_code
 from .user import User
 from .fields import get_absolute_url
@@ -136,16 +138,19 @@ class SiteSettings(SiteModel):
             return get_absolute_url(uploaded)
         return urljoin(STATIC_FULL_URL, default_path)
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs):
         """if require_confirm_email is disabled, make sure no users are pending,
         if enabled, make sure invite_question_text is not empty"""
+        if not self.invite_question_text:
+            self.invite_question_text = "What is your favourite book?"
+            update_fields = add_update_fields(update_fields, "invite_question_text")
+
+        super().save(*args, update_fields=update_fields, **kwargs)
+
         if not self.require_confirm_email:
             User.objects.filter(is_active=False, deactivation_reason="pending").update(
                 is_active=True, deactivation_reason=None
             )
-        if not self.invite_question_text:
-            self.invite_question_text = "What is your favourite book?"
-        super().save(*args, **kwargs)
 
 
 class Theme(SiteModel):
@@ -188,7 +193,7 @@ class SiteInvite(models.Model):
     @property
     def link(self):
         """formats the invite link"""
-        return f"https://{DOMAIN}/invite/{self.code}"
+        return f"{BASE_URL}/invite/{self.code}"
 
 
 class InviteRequest(BookWyrmModel):
@@ -235,7 +240,7 @@ class PasswordReset(models.Model):
     @property
     def link(self):
         """formats the invite link"""
-        return f"https://{DOMAIN}/password-reset/{self.code}"
+        return f"{BASE_URL}/password-reset/{self.code}"
 
 
 # pylint: disable=unused-argument

bookwyrm/models/status.py

@@ -1,6 +1,6 @@
 """ models for storing different kinds of Activities """
 from dataclasses import MISSING
-from typing import Optional
+from typing import Optional, Iterable
 import re
 
 from django.apps import apps
@@ -20,6 +20,7 @@ from model_utils.managers import InheritanceManager
 from bookwyrm import activitypub
 from bookwyrm.preview_images import generate_edition_preview_image_task
 from bookwyrm.settings import ENABLE_PREVIEW_IMAGES
+from bookwyrm.utils.db import add_update_fields
 from .activitypub_mixin import ActivitypubMixin, ActivityMixin
 from .activitypub_mixin import OrderedCollectionPageMixin
 from .base_model import BookWyrmModel
@@ -80,19 +81,24 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         """default sorting"""
 
         ordering = ("-published_date",)
+        indexes = [
+            models.Index(fields=["remote_id"]),
+            models.Index(fields=["thread_id"]),
+        ]
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs):
         """save and notify"""
-        if self.reply_parent:
+        if self.thread_id is None and self.reply_parent:
             self.thread_id = self.reply_parent.thread_id or self.reply_parent_id
+            update_fields = add_update_fields(update_fields, "thread_id")
 
-        super().save(*args, **kwargs)
+        super().save(*args, update_fields=update_fields, **kwargs)
 
         if not self.reply_parent:
             self.thread_id = self.id
             super().save(broadcast=False, update_fields=["thread_id"])
 
-    def delete(self, *args, **kwargs):  # pylint: disable=unused-argument
+    def delete(self, *args, **kwargs):
         """ "delete" a status"""
         if hasattr(self, "boosted_status"):
             # okay but if it's a boost really delete it
@@ -207,7 +213,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
             **kwargs,
         ).serialize()
 
-    def to_activity_dataclass(self, pure=False):  # pylint: disable=arguments-differ
+    def to_activity_dataclass(self, pure=False):
         """return tombstone if the status is deleted"""
         if self.deleted:
             return activitypub.Tombstone(
@@ -388,10 +394,10 @@ class Quotation(BookStatus):
     def _format_position(self) -> Optional[str]:
         """serialize page position"""
         beg = self.position
-        end = self.endposition or 0
+        end = self.endposition
         if self.position_mode != "PG" or not beg:
             return None
-        return f"pp. {beg}-{end}" if end > beg else f"p. {beg}"
+        return f"pp. {beg}-{end}" if end else f"p. {beg}"
 
     @property
     def pure_content(self):
@@ -455,9 +461,10 @@ class Review(BookStatus):
 
     def save(self, *args, **kwargs):
         """clear rating caches"""
+        super().save(*args, **kwargs)
+
         if self.book.parent_work:
             cache.delete(f"book-rating-{self.book.parent_work.id}")
-        super().save(*args, **kwargs)
 
 
 class ReviewRating(Review):

bookwyrm/models/user.py

@@ -1,28 +1,31 @@
 """ database schema for user data """
+import datetime
 import re
+import zoneinfo
+from typing import Optional, Iterable
 from urllib.parse import urlparse
 from uuid import uuid4
 
 from django.apps import apps
 from django.contrib.auth.models import AbstractUser
-from django.contrib.postgres.fields import ArrayField, CICharField
+from django.contrib.postgres.fields import ArrayField as DjangoArrayField
 from django.core.exceptions import PermissionDenied, ObjectDoesNotExist
 from django.dispatch import receiver
 from django.db import models, transaction, IntegrityError
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from model_utils import FieldTracker
-import pytz
 
 from bookwyrm import activitypub
 from bookwyrm.connectors import get_data, ConnectorException
 from bookwyrm.models.shelf import Shelf
 from bookwyrm.models.status import Status
 from bookwyrm.preview_images import generate_user_preview_image_task
-from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, USE_HTTPS, LANGUAGES
+from bookwyrm.settings import BASE_URL, ENABLE_PREVIEW_IMAGES, LANGUAGES
 from bookwyrm.signatures import create_key_pair
 from bookwyrm.tasks import app, MISC
 from bookwyrm.utils import regex
+from bookwyrm.utils.db import add_update_fields
 from .activitypub_mixin import OrderedCollectionPageMixin, ActivitypubMixin
 from .base_model import BookWyrmModel, DeactivationReason, new_access_code
 from .federated_server import FederatedServer
@@ -42,12 +45,6 @@ def get_feed_filter_choices():
     return [f[0] for f in FeedFilterChoices]
 
 
-def site_link():
-    """helper for generating links to the site"""
-    protocol = "https" if USE_HTTPS else "http"
-    return f"{protocol}://{DOMAIN}"
-
-
 # pylint: disable=too-many-public-methods
 class User(OrderedCollectionPageMixin, AbstractUser):
     """a user who wants to read books"""
@@ -81,11 +78,12 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     summary = fields.HtmlField(null=True, blank=True)
     local = models.BooleanField(default=False)
     bookwyrm_user = fields.BooleanField(default=True)
-    localname = CICharField(
+    localname = models.CharField(
         max_length=255,
         null=True,
         unique=True,
         validators=[fields.validate_localname],
+        db_collation="case_insensitive",
     )
     # name is your display name, which you can change at will
     name = fields.CharField(max_length=100, null=True, blank=True)
@@ -143,7 +141,6 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     hide_follows = fields.BooleanField(default=False)
 
     # migration fields
-
     moved_to = fields.RemoteIdField(
         null=True, unique=False, activitypub_field="movedTo", deduplication_field=False
     )
@@ -160,9 +157,10 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     show_suggested_users = models.BooleanField(default=True)
     discoverable = fields.BooleanField(default=False)
     show_guided_tour = models.BooleanField(default=True)
+    show_ratings = models.BooleanField(default=True)
 
     # feed options
-    feed_status_types = ArrayField(
+    feed_status_types = DjangoArrayField(
         models.CharField(max_length=10, blank=False, choices=FeedFilterChoices),
         size=8,
         default=get_feed_filter_choices,
@@ -171,8 +169,8 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     summary_keys = models.JSONField(null=True)
 
     preferred_timezone = models.CharField(
-        choices=[(str(tz), str(tz)) for tz in pytz.all_timezones],
-        default=str(pytz.utc),
+        choices=[(str(tz), str(tz)) for tz in sorted(zoneinfo.available_timezones())],
+        default=str(datetime.timezone.utc),
         max_length=255,
     )
     preferred_language = models.CharField(
@@ -198,6 +196,14 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     hotp_secret = models.CharField(max_length=32, default=None, blank=True, null=True)
     hotp_count = models.IntegerField(default=0, blank=True, null=True)
 
+    class Meta(AbstractUser.Meta):
+        """indexes"""
+
+        indexes = [
+            models.Index(fields=["username"]),
+            models.Index(fields=["is_active", "local"]),
+        ]
+
     @property
     def active_follower_requests(self):
         """Follow requests from active users"""
@@ -206,8 +212,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     @property
     def confirmation_link(self):
         """helper for generating confirmation links"""
-        link = site_link()
-        return f"{link}/confirm-email/{self.confirmation_code}"
+        return f"{BASE_URL}/confirm-email/{self.confirmation_code}"
 
     @property
     def following_link(self):
@@ -326,6 +331,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
             "https://w3id.org/security/v1",
             {
                 "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+                "Hashtag": "as:Hashtag",
                 "schema": "http://schema.org#",
                 "PropertyValue": "schema:PropertyValue",
                 "value": "schema:value",
@@ -335,13 +341,14 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         ]
         return activity_object
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs):
         """populate fields for new local users"""
         created = not bool(self.id)
         if not self.local and not re.match(regex.FULL_USERNAME, self.username):
             # generate a username that uses the domain (webfinger format)
             actor_parts = urlparse(self.remote_id)
-            self.username = f"{self.username}@{actor_parts.netloc}"
+            self.username = f"{self.username}@{actor_parts.hostname}"
+            update_fields = add_update_fields(update_fields, "username")
 
         # this user already exists, no need to populate fields
         if not created:
@@ -350,26 +357,34 @@ class User(OrderedCollectionPageMixin, AbstractUser):
             elif not self.deactivation_date:
                 self.deactivation_date = timezone.now()
 
-            super().save(*args, **kwargs)
+            super().save(*args, update_fields=update_fields, **kwargs)
             return
 
         # this is a new remote user, we need to set their remote server field
         if not self.local:
-            super().save(*args, **kwargs)
+            super().save(*args, update_fields=update_fields, **kwargs)
             transaction.on_commit(lambda: set_remote_server(self.id))
             return
 
         with transaction.atomic():
             # populate fields for local users
-            link = site_link()
-            self.remote_id = f"{link}/user/{self.localname}"
+            self.remote_id = f"{BASE_URL}/user/{self.localname}"
             self.followers_url = f"{self.remote_id}/followers"
             self.inbox = f"{self.remote_id}/inbox"
-            self.shared_inbox = f"{link}/inbox"
+            self.shared_inbox = f"{BASE_URL}/inbox"
             self.outbox = f"{self.remote_id}/outbox"
 
+            update_fields = add_update_fields(
+                update_fields,
+                "remote_id",
+                "followers_url",
+                "inbox",
+                "shared_inbox",
+                "outbox",
+            )
+
             # an id needs to be set before we can proceed with related models
-            super().save(*args, **kwargs)
+            super().save(*args, update_fields=update_fields, **kwargs)
 
             # make users editors by default
             try:
@@ -394,7 +409,6 @@ class User(OrderedCollectionPageMixin, AbstractUser):
 
     def delete(self, *args, **kwargs):
         """We don't actually delete the database entry"""
-        # pylint: disable=attribute-defined-outside-init
         self.is_active = False
         self.allow_reactivation = False
         self.is_deleted = True
@@ -437,7 +451,6 @@ class User(OrderedCollectionPageMixin, AbstractUser):
 
     def deactivate(self):
         """Disable the user but allow them to reactivate"""
-        # pylint: disable=attribute-defined-outside-init
         self.is_active = False
         self.deactivation_reason = "self_deactivation"
         self.allow_reactivation = True
@@ -445,7 +458,6 @@ class User(OrderedCollectionPageMixin, AbstractUser):
 
     def reactivate(self):
         """Now you want to come back, huh?"""
-        # pylint: disable=attribute-defined-outside-init
         if not self.allow_reactivation:
             return
         self.is_active = True
@@ -509,18 +521,30 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
     activity_serializer = activitypub.PublicKey
     serialize_reverse_fields = [("owner", "owner", "id")]
 
+    class Meta:
+        """indexes"""
+
+        indexes = [
+            models.Index(fields=["remote_id"]),
+        ]
+
     def get_remote_id(self):
         # self.owner is set by the OneToOneField on User
         return f"{self.owner.remote_id}/#main-key"
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs):
         """create a key pair"""
         # no broadcasting happening here
         if "broadcast" in kwargs:
             del kwargs["broadcast"]
+
         if not self.public_key:
             self.private_key, self.public_key = create_key_pair()
-        return super().save(*args, **kwargs)
+            update_fields = add_update_fields(
+                update_fields, "private_key", "public_key"
+            )
+
+        super().save(*args, update_fields=update_fields, **kwargs)
 
 
 @app.task(queue=MISC)
@@ -543,7 +567,7 @@ def set_remote_server(user_id, allow_external_connections=False):
     user = User.objects.get(id=user_id)
     actor_parts = urlparse(user.remote_id)
     federated_server = get_or_create_remote_server(
-        actor_parts.netloc, allow_external_connections=allow_external_connections
+        actor_parts.hostname, allow_external_connections=allow_external_connections
     )
     # if we were unable to find the server, we need to create a new entry for it
     if not federated_server:

bookwyrm/preview_images.py

@@ -175,11 +175,13 @@ def generate_instance_layer(content_width):
     site = models.SiteSettings.objects.get()
 
     if site.logo_small:
-        logo_img = Image.open(site.logo_small)
+        with Image.open(site.logo_small) as logo_img:
+            logo_img.load()
     else:
         try:
             static_path = os.path.join(settings.STATIC_ROOT, "images/logo-small.png")
-            logo_img = Image.open(static_path)
+            with Image.open(static_path) as logo_img:
+                logo_img.load()
         except FileNotFoundError:
             logo_img = None
 
@@ -211,18 +213,9 @@ def generate_instance_layer(content_width):
 
 def generate_rating_layer(rating, content_width):
     """Places components for rating preview"""
-    try:
-        icon_star_full = Image.open(
-            os.path.join(settings.STATIC_ROOT, "images/icons/star-full.png")
-        )
-        icon_star_empty = Image.open(
-            os.path.join(settings.STATIC_ROOT, "images/icons/star-empty.png")
-        )
-        icon_star_half = Image.open(
-            os.path.join(settings.STATIC_ROOT, "images/icons/star-half.png")
-        )
-    except FileNotFoundError:
-        return None
+    path_star_full = os.path.join(settings.STATIC_ROOT, "images/icons/star-full.png")
+    path_star_empty = os.path.join(settings.STATIC_ROOT, "images/icons/star-empty.png")
+    path_star_half = os.path.join(settings.STATIC_ROOT, "images/icons/star-half.png")
 
     icon_size = 64
     icon_margin = 10
@@ -237,17 +230,23 @@ def generate_rating_layer(rating, content_width):
 
     position_x = 0
 
-    for _ in range(math.floor(rating)):
-        rating_layer_mask.alpha_composite(icon_star_full, (position_x, 0))
-        position_x = position_x + icon_size + icon_margin
+    try:
+        with Image.open(path_star_full) as icon_star_full:
+            for _ in range(math.floor(rating)):
+                rating_layer_mask.alpha_composite(icon_star_full, (position_x, 0))
+                position_x = position_x + icon_size + icon_margin
 
-    if math.floor(rating) != math.ceil(rating):
-        rating_layer_mask.alpha_composite(icon_star_half, (position_x, 0))
-        position_x = position_x + icon_size + icon_margin
+        if math.floor(rating) != math.ceil(rating):
+            with Image.open(path_star_half) as icon_star_half:
+                rating_layer_mask.alpha_composite(icon_star_half, (position_x, 0))
+                position_x = position_x + icon_size + icon_margin
 
-    for _ in range(5 - math.ceil(rating)):
-        rating_layer_mask.alpha_composite(icon_star_empty, (position_x, 0))
-        position_x = position_x + icon_size + icon_margin
+        with Image.open(path_star_empty) as icon_star_empty:
+            for _ in range(5 - math.ceil(rating)):
+                rating_layer_mask.alpha_composite(icon_star_empty, (position_x, 0))
+                position_x = position_x + icon_size + icon_margin
+    except FileNotFoundError:
+        return None
 
     rating_layer_mask = rating_layer_mask.getchannel("A")
     rating_layer_mask = ImageOps.invert(rating_layer_mask)
@@ -290,7 +289,8 @@ def generate_preview_image(
     texts = texts or {}
     # Cover
     try:
-        inner_img_layer = Image.open(picture)
+        with Image.open(picture) as inner_img_layer:
+            inner_img_layer.load()
         inner_img_layer.thumbnail(
             (inner_img_width, inner_img_height), Image.Resampling.LANCZOS
         )
@@ -420,7 +420,6 @@ def save_and_cleanup(image, instance=None):
     return True
 
 
-# pylint: disable=invalid-name
 @app.task(queue=IMAGES)
 def generate_site_preview_image_task():
     """generate preview_image for the website"""
@@ -445,7 +444,6 @@ def generate_site_preview_image_task():
     save_and_cleanup(image, instance=site)
 
 
-# pylint: disable=invalid-name
 @app.task(queue=IMAGES)
 def generate_edition_preview_image_task(book_id):
     """generate preview_image for a book"""

bookwyrm/settings.py

@@ -19,7 +19,6 @@ DOMAIN = env("DOMAIN")
 with open("VERSION", encoding="utf-8") as f:
     version = f.read()
     version = version.replace("\n", "")
-f.close()
 
 VERSION = version
 
@@ -102,6 +101,7 @@ INSTALLED_APPS = [
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "django.contrib.humanize",
+    "oauth2_provider",
     "file_resubmit",
     "sass_processor",
     "bookwyrm",
@@ -257,11 +257,8 @@ if env.bool("USE_DUMMY_CACHE", False):
 else:
     CACHES = {
         "default": {
-            "BACKEND": "django_redis.cache.RedisCache",
+            "BACKEND": "django.core.cache.backends.redis.RedisCache",
             "LOCATION": REDIS_ACTIVITY_URL,
-            "OPTIONS": {
-                "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            },
         },
         "file_resubmit": {
             "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
@@ -347,34 +344,37 @@ TIME_ZONE = "UTC"
 
 USE_I18N = True
 
-USE_L10N = True
-
 USE_TZ = True
 
-
-USER_AGENT = f"BookWyrm (BookWyrm/{VERSION}; +https://{DOMAIN}/)"
-
 # Imagekit generated thumbnails
 ENABLE_THUMBNAIL_GENERATION = env.bool("ENABLE_THUMBNAIL_GENERATION", False)
 IMAGEKIT_CACHEFILE_DIR = "thumbnails"
 IMAGEKIT_DEFAULT_CACHEFILE_STRATEGY = "bookwyrm.thumbnail_generation.Strategy"
 
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/3.2/howto/static-files/
-
 PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
 CSP_ADDITIONAL_HOSTS = env.list("CSP_ADDITIONAL_HOSTS", [])
 
-# Storage
-
 PROTOCOL = "http"
 if USE_HTTPS:
     PROTOCOL = "https"
     SESSION_COOKIE_SECURE = True
     CSRF_COOKIE_SECURE = True
 
+PORT = env.int("PORT", 443 if USE_HTTPS else 80)
+if (USE_HTTPS and PORT == 443) or (not USE_HTTPS and PORT == 80):
+    NETLOC = DOMAIN
+else:
+    NETLOC = f"{DOMAIN}:{PORT}"
+BASE_URL = f"{PROTOCOL}://{NETLOC}"
+CSRF_TRUSTED_ORIGINS = [BASE_URL]
+
+USER_AGENT = f"BookWyrm (BookWyrm/{VERSION}; +{BASE_URL})"
+
+# Storage
+
 USE_S3 = env.bool("USE_S3", False)
 USE_AZURE = env.bool("USE_AZURE", False)
+S3_SIGNED_URL_EXPIRY = env.int("S3_SIGNED_URL_EXPIRY", 900)
 
 if USE_S3:
     # AWS settings
@@ -386,44 +386,124 @@ if USE_S3:
     AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL", None)
     AWS_DEFAULT_ACL = "public-read"
     AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"}
+    AWS_S3_URL_PROTOCOL = env("AWS_S3_URL_PROTOCOL", f"{PROTOCOL}:")
+    # Storages
+    STORAGES = {
+        "default": {
+            "BACKEND": "storages.backends.s3.S3Storage",
+            "OPTIONS": {
+                "location": "images",
+                "default_acl": "public-read",
+                "file_overwrite": False,
+            },
+        },
+        "staticfiles": {
+            "BACKEND": "storages.backends.s3.S3Storage",
+            "OPTIONS": {
+                "location": "static",
+                "default_acl": "public-read",
+            },
+        },
+        "sass_processor": {
+            "BACKEND": "storages.backends.s3.S3Storage",
+            "OPTIONS": {
+                "location": "static",
+                "default_acl": "public-read",
+            },
+        },
+        "exports": {
+            "BACKEND": "storages.backends.s3.S3Storage",
+            "OPTIONS": {
+                "location": "images",
+                "default_acl": None,
+                "file_overwrite": False,
+            },
+        },
+    }
     # S3 Static settings
     STATIC_LOCATION = "static"
-    STATIC_URL = f"{PROTOCOL}://{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/"
-    STATICFILES_STORAGE = "bookwyrm.storage_backends.StaticStorage"
+    STATIC_URL = f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/"
+    STATIC_FULL_URL = STATIC_URL
     # S3 Media settings
     MEDIA_LOCATION = "images"
-    MEDIA_URL = f"{PROTOCOL}://{AWS_S3_CUSTOM_DOMAIN}/{MEDIA_LOCATION}/"
+    MEDIA_URL = f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}/{MEDIA_LOCATION}/"
     MEDIA_FULL_URL = MEDIA_URL
-    STATIC_FULL_URL = STATIC_URL
-    DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
-    CSP_DEFAULT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
-    CSP_SCRIPT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
+    # Content Security Policy
+    CSP_DEFAULT_SRC = [
+        "'self'",
+        f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}"
+        if AWS_S3_CUSTOM_DOMAIN
+        else None,
+    ] + CSP_ADDITIONAL_HOSTS
+    CSP_SCRIPT_SRC = [
+        "'self'",
+        f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}"
+        if AWS_S3_CUSTOM_DOMAIN
+        else None,
+    ] + CSP_ADDITIONAL_HOSTS
 elif USE_AZURE:
+    # Azure settings
     AZURE_ACCOUNT_NAME = env("AZURE_ACCOUNT_NAME")
     AZURE_ACCOUNT_KEY = env("AZURE_ACCOUNT_KEY")
     AZURE_CONTAINER = env("AZURE_CONTAINER")
     AZURE_CUSTOM_DOMAIN = env("AZURE_CUSTOM_DOMAIN")
+    # Storages
+    STORAGES = {
+        "default": {
+            "BACKEND": "storages.backends.azure_storage.AzureStorage",
+            "OPTIONS": {
+                "location": "images",
+                "overwrite_files": False,
+            },
+        },
+        "staticfiles": {
+            "BACKEND": "storages.backends.azure_storage.AzureStorage",
+            "OPTIONS": {
+                "location": "static",
+            },
+        },
+        "exports": {
+            "BACKEND": None,  # not implemented yet
+        },
+    }
     # Azure Static settings
     STATIC_LOCATION = "static"
     STATIC_URL = (
         f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{STATIC_LOCATION}/"
     )
-    STATICFILES_STORAGE = "bookwyrm.storage_backends.AzureStaticStorage"
+    STATIC_FULL_URL = STATIC_URL
     # Azure Media settings
     MEDIA_LOCATION = "images"
     MEDIA_URL = (
         f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{MEDIA_LOCATION}/"
     )
     MEDIA_FULL_URL = MEDIA_URL
-    STATIC_FULL_URL = STATIC_URL
-    DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.AzureImagesStorage"
+    # Content Security Policy
     CSP_DEFAULT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
     CSP_SCRIPT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
 else:
+    # Storages
+    STORAGES = {
+        "default": {
+            "BACKEND": "django.core.files.storage.FileSystemStorage",
+        },
+        "staticfiles": {
+            "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+        },
+        "exports": {
+            "BACKEND": "django.core.files.storage.FileSystemStorage",
+            "OPTIONS": {
+                "location": "exports",
+            },
+        },
+    }
+    # Static settings
     STATIC_URL = "/static/"
+    STATIC_FULL_URL = BASE_URL + STATIC_URL
+    # Media settings
     MEDIA_URL = "/images/"
-    MEDIA_FULL_URL = f"{PROTOCOL}://{DOMAIN}{MEDIA_URL}"
-    STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}"
+    MEDIA_FULL_URL = BASE_URL + MEDIA_URL
+    # Content Security Policy
     CSP_DEFAULT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS
     CSP_SCRIPT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS
 

bookwyrm/signatures.py

@@ -6,7 +6,7 @@ from base64 import b64encode, b64decode
 
 from Crypto import Random
 from Crypto.PublicKey import RSA
-from Crypto.Signature import pkcs1_15  # pylint: disable=no-name-in-module
+from Crypto.Signature import pkcs1_15
 from Crypto.Hash import SHA256
 
 MAX_SIGNATURE_AGE = 300
@@ -84,7 +84,6 @@ class Signature:
         self.headers = headers
         self.signature = signature
 
-    # pylint: disable=invalid-name
     @classmethod
     def parse(cls, request):
         """extract and parse a signature from an http request"""

bookwyrm/static/js/bookwyrm.js

@@ -14,6 +14,10 @@ let BookWyrm = new (class {
             .querySelectorAll("[data-controls]")
             .forEach((button) => button.addEventListener("click", this.toggleAction.bind(this)));
 
+        document
+            .querySelectorAll("[data-disappear]")
+            .forEach((button) => button.addEventListener("click", this.hideSelf.bind(this)));
+
         document
             .querySelectorAll(".interaction")
             .forEach((button) => button.addEventListener("submit", this.interact.bind(this)));
@@ -181,6 +185,18 @@ let BookWyrm = new (class {
         this.addRemoveClass(visible, "is-hidden", true);
     }
 
+    /**
+     * Hide the element you just clicked
+     *
+     * @param {Event} event
+     * @return {undefined}
+     */
+    hideSelf(event) {
+        let trigger = event.currentTarget;
+
+        this.addRemoveClass(trigger, "is-hidden", true);
+    }
+
     /**
      * Execute actions on targets based on triggers.
      *

bookwyrm/storage_backends.py

@@ -1,63 +0,0 @@
-"""Handles backends for storages"""
-import os
-from tempfile import SpooledTemporaryFile
-from storages.backends.s3boto3 import S3Boto3Storage
-from storages.backends.azure_storage import AzureStorage
-
-
-class StaticStorage(S3Boto3Storage):  # pylint: disable=abstract-method
-    """Storage class for Static contents"""
-
-    location = "static"
-    default_acl = "public-read"
-
-
-class ImagesStorage(S3Boto3Storage):  # pylint: disable=abstract-method
-    """Storage class for Image files"""
-
-    location = "images"
-    default_acl = "public-read"
-    file_overwrite = False
-
-    """
-    This is our custom version of S3Boto3Storage that fixes a bug in
-    boto3 where the passed in file is closed upon upload.
-    From:
-    https://github.com/matthewwithanm/django-imagekit/issues/391#issuecomment-275367006
-    https://github.com/boto/boto3/issues/929
-    https://github.com/matthewwithanm/django-imagekit/issues/391
-    """
-
-    def _save(self, name, content):
-        """
-        We create a clone of the content file as when this is passed to
-        boto3 it wrongly closes the file upon upload where as the storage
-        backend expects it to still be open
-        """
-        # Seek our content back to the start
-        content.seek(0, os.SEEK_SET)
-
-        # Create a temporary file that will write to disk after a specified
-        # size. This file will be automatically deleted when closed by
-        # boto3 or after exiting the `with` statement if the boto3 is fixed
-        with SpooledTemporaryFile() as content_autoclose:
-
-            # Write our original content into our copy that will be closed by boto3
-            content_autoclose.write(content.read())
-
-            # Upload the object which will auto close the
-            # content_autoclose instance
-            return super()._save(name, content_autoclose)
-
-
-class AzureStaticStorage(AzureStorage):  # pylint: disable=abstract-method
-    """Storage class for Static contents"""
-
-    location = "static"
-
-
-class AzureImagesStorage(AzureStorage):  # pylint: disable=abstract-method
-    """Storage class for Image files"""
-
-    location = "images"
-    overwrite_files = False

bookwyrm/suggested_users.py

@@ -34,7 +34,6 @@ class SuggestedUsers(RedisStore):
 
     def get_counts_from_rank(self, rank):  # pylint: disable=no-self-use
         """calculate mutuals count and shared books count from rank"""
-        # pylint: disable=c-extension-no-member
         return {
             "mutuals": math.floor(rank),
             # "shared_books": int(1 / (-1 * (rank % 1 - 1))) - 1,
@@ -128,7 +127,6 @@ def get_annotated_users(viewer, *args, **kwargs):
                 ),
                 distinct=True,
             ),
-            # pylint: disable=line-too-long
             # shared_books=Count(
             #     "shelfbook",
             #     filter=Q(
@@ -202,7 +200,7 @@ def update_suggestions_on_unfollow(sender, instance, **kwargs):
 
 
 @receiver(signals.post_save, sender=models.User)
-# pylint: disable=unused-argument, too-many-arguments
+# pylint: disable=unused-argument
 def update_user(sender, instance, created, update_fields=None, **kwargs):
     """an updated user, neat"""
     # a new user is found, create suggestions for them

bookwyrm/templates/book/file_links/verification_modal.html

@@ -21,9 +21,10 @@ Is that where you'd like to go?
 <div class="is-flex-grow-1">
     <a href="{% url 'report-link' link.id %}">{% trans "Report spam" %}</a>
 </div>
+{% endif %}
 
 <button type="button" class="button" data-modal-close>{% trans "Cancel" %}</button>
 <a href="{{ link.url }}" target="_blank" rel="nofollow noopener noreferrer" noreferrer" class="button is-primary">{% trans "Continue" %}</a>
-{% endif %}
+
 
 {% endblock %}

bookwyrm/templates/compose.html

@@ -2,10 +2,31 @@
 {% load i18n %}
 {% load utilities %}
 
-{% block title %}{% trans "Edit status" %}{% endblock %}
+{% block title %}
+    {% if draft.status_type == "Review" %}
+        {% trans "Edit review" %}
+    {% elif draft.status_type == "Quotation" %}
+        {% trans "Edit quote" %}
+    {% elif draft.status_type == "Comment" %}
+        {% trans "Edit comment" %}
+    {% else %}
+        {% trans "Edit status" %}
+    {% endif %}
+{% endblock %}
+
 {% block content %}
 <header class="block content">
-    <h1>{% trans "Edit status" %}</h1>
+    <h1>
+        {% if draft.status_type == "Review" %}
+            {% trans "Edit review" %}
+        {% elif draft.status_type == "Quotation" %}
+            {% trans "Edit quote" %}
+        {% elif draft.status_type == "Comment" %}
+            {% trans "Edit comment" %}
+        {% else %}
+            {% trans "Edit status" %}
+        {% endif %}
+    </h1>
 </header>
 
 {% with 0|uuid as uuid %}

bookwyrm/templates/email/html_layout.html

@@ -2,10 +2,10 @@
 <div style="font-family: BlinkMacSystemFont,-apple-system,'Segoe UI',Roboto,Oxygen,Ubuntu,Cantarell,'Fira Sans','Droid Sans','Helvetica Neue',Helvetica,Arial,sans-serif; border-radius: 6px; background-color: #efefef; max-width: 632px;">
     <div style="padding: 1rem; overflow: auto;">
         <div style="float: left; margin-right: 1rem;">
-            <a style="color: #3273dc;" href="https://{{ domain }}" style="text-decoration: none;"><img src="{{ logo }}" alt="logo" loading="lazy" decoding="async"></a>
+            <a style="color: #3273dc;" href="{{ base_url }}" style="text-decoration: none;"><img src="{{ logo }}" alt="logo" loading="lazy" decoding="async"></a>
         </div>
         <div>
-            <a style="color: black; text-decoration: none" href="https://{{ domain }}" style="text-decoration: none;"><strong>{{ site_name }}</strong><br>
+            <a style="color: black; text-decoration: none" href="{{ base_url }}" style="text-decoration: none;"><strong>{{ site_name }}</strong><br>
             {{ domain }}</a>
         </div>
     </div>
@@ -18,9 +18,9 @@
     </div>
 
     <div style="padding: 1rem; font-size: 0.8rem;">
-        <p style="margin: 0; color: #333;">{% blocktrans %}BookWyrm hosted on <a style="color: #3273dc;" href="https://{{ domain }}">{{ site_name }}</a>{% endblocktrans %}</p>
+        <p style="margin: 0; color: #333;">{% blocktrans %}BookWyrm hosted on <a style="color: #3273dc;" href="{{ base_url }}">{{ site_name }}</a>{% endblocktrans %}</p>
         {% if user %}
-        <p style="margin: 0; color: #333;"><a style="color: #3273dc;" href="https://{{ domain }}{% url 'prefs-profile' %}">{% trans "Email preference" %}</a></p>
+        <p style="margin: 0; color: #333;"><a style="color: #3273dc;" href="{{ base_url }}{% url 'prefs-profile' %}">{% trans "Email preference" %}</a></p>
         {% endif %}
     </div>
 </div>

bookwyrm/templates/email/invite/html_content.html

@@ -12,6 +12,6 @@
 <p>
     {% url 'code-of-conduct' as coc_path %}
     {% url 'about' as about_path %}
-    {% blocktrans %}Learn more <a href="https://{{ domain }}{{ about_path }}">about {{ site_name }}</a>.{% endblocktrans %}
+    {% blocktrans %}Learn more <a href="{{ base_url }}{{ about_path }}">about {{ site_name }}</a>.{% endblocktrans %}
 </p>
 {% endblock %}

bookwyrm/templates/email/invite/text_content.html

@@ -5,6 +5,6 @@
 
 {{ invite_link }}
 
-{% blocktrans %}Learn more about {{ site_name }}:{% endblocktrans %} https://{{ domain }}{% url 'about' %}
+{% blocktrans %}Learn more about {{ site_name }}:{% endblocktrans %} {{ base_url }}{% url 'about' %}
 
 {% endblock %}

bookwyrm/templates/import/import.html

@@ -69,6 +69,9 @@
                             <option value="Calibre" {% if current == 'Calibre' %}selected{% endif %}>
                                 {% trans "Calibre (CSV)" %}
                             </option>
+                            <option value="BookWyrm" {% if current == 'BookWyrm' %}selected{% endif %}>
+                                {% trans "BookWyrm (CSV)" %}
+                            </option>
                         </select>
                     </div>
 
@@ -93,9 +96,14 @@
                         <input type="checkbox" name="include_reviews" checked> {% trans "Include reviews" %}
                     </label>
                 </div>
+                <div class="field">
+                    <label class="label">
+                        <input type="checkbox" name="create_shelves" checked> {% trans "Create new shelves if they do not exist" %}
+                    </label>
+                </div>
                 <div class="field">
                     <label class="label" for="privacy_import">
-                        {% trans "Privacy setting for imported reviews:" %}
+                        {% trans "Privacy setting for imported reviews and shelves:" %}
                     </label>
                     {% include 'snippets/privacy_select.html' with no_label=True privacy_uuid="import" %}
                 </div>

bookwyrm/templates/opensearch.xml

@@ -10,6 +10,6 @@
     <Image width="16" height="16" type="image/x-icon">{{ image }}</Image>
     <Url
         type="text/html"
-        template="https://{{ DOMAIN }}{% url 'search' %}?q={searchTerms}"
+        template="{{ BASE_URL }}{% url 'search' %}?q={searchTerms}"
     />
 </OpenSearchDescription>

bookwyrm/templates/ostatus/remote_follow.html

@@ -3,8 +3,10 @@
 {% load utilities %}
 
 {% block heading %}
+{% block title %}
 {% blocktrans with username=user.localname sitename=site.name %}Follow {{ username }} on the fediverse{% endblocktrans %}
 {% endblock %}
+{% endblock %}
 
 {% block content %}
 <div class="block card">

bookwyrm/templates/ostatus/success.html

@@ -2,6 +2,10 @@
 {% load i18n %}
 {% load utilities %}
 
+{% block title %}
+{% blocktrans with display_name=user.display_name %}You are now following {{ display_name }}!{% endblocktrans %}
+{% endblock %}
+
 {% block content %}
 <div class="block card">
     <div class="card-content">

bookwyrm/templates/preferences/edit_user.html

@@ -69,6 +69,12 @@
                     {% trans "Show reading goal prompt in feed" %}
                 </label>
             </div>
+            <div class="field">
+                <label class="checkbox label" for="id_show_ratings">
+                    {{ form.show_ratings }}
+                    {% trans "Show ratings" %}
+                </label>
+            </div>
             <div class="field">
                 <label class="checkbox label" for="id_show_suggested_users">
                     {{ form.show_suggested_users }}

bookwyrm/templates/preferences/export-user.html

@@ -97,25 +97,25 @@
                 </td>
             </tr>
             {% endif %}
-            {% for job in jobs %}
+            {% for export in jobs %}
             <tr>
-                <td>{{ job.updated_date }}</td>
+                <td>{{ export.job.updated_date }}</td>
                 <td>
                     <span
-                        {% if job.status == "stopped" or job.status == "failed" %}
+                        {% if export.job.status == "stopped" or export.job.status == "failed" %}
                         class="tag is-danger"
-                        {% elif job.status == "pending" %}
+                        {% elif export.job.status == "pending" %}
                         class="tag is-warning"
-                        {% elif job.complete %}
+                        {% elif export.job.complete %}
                         class="tag"
                         {% else %}
                         class="tag is-success"
                         {% endif %}
                     >
-                        {% if job.status %}
-                        {{ job.status }}
-                            {{ job.status_display }}
-                        {% elif job.complete %}
+                        {% if export.job.status %}
+                        {{ export.job.status }}
+                            {{ export.job.status_display }}
+                        {% elif export.job.complete %}
                             {% trans "Complete" %}
                         {% else %}
                             {% trans "Active" %}
@@ -123,18 +123,20 @@
                     </span>
                 </td>
                 <td>
-                    <span>{{ job.export_data|get_file_size }}</span>
+                    {% if export.size %}
+                    <span>{{ export.size|get_file_size }}</span>
+                    {% endif %}
                 </td>
                 <td>
-                    {% if job.complete and not job.status == "stopped" and not job.status == "failed" %}
-                    <p>
-                        <a download="" href="/preferences/user-export/{{ job.task_id }}">
-                            <span class="icon icon-download" aria-hidden="true"></span>
-                            <span class="is-hidden-mobile">
-                                {% trans "Download your export" %}
-                            </span>
-                        </a>
-                    </p>
+                    {% if export.url %}
+                    <a href="{{ export.url }}">
+                        <span class="icon icon-download" aria-hidden="true"></span>
+                        <span class="is-hidden-mobile">
+                            {% trans "Download your export" %}
+                        </span>
+                    </a>
+                    {% elif export.unavailable %}
+                    {% trans "Archive is no longer available" %}
                     {% endif %}
                 </td>
             </tr>