diff --git a/.env.example b/.env.example index 1bf6d5406..c61ceba1e 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,11 @@ DEFAULT_LANGUAGE="English" ## Leave unset to allow all hosts # ALLOWED_HOSTS="localhost,127.0.0.1,[::1]" +# Specify when the site is served from a port that is not the default +# for the protocol (80 for HTTP or 443 for HTTPS). +# Probably only necessary in development. +# PORT=1333 + MEDIA_ROOT=images/ # Database configuration @@ -71,14 +76,20 @@ ENABLE_THUMBNAIL_GENERATION=true USE_S3=false AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= +# seconds for signed S3 urls to expire +# this is currently only used for user export files +S3_SIGNED_URL_EXPIRY=900 # Commented are example values if you use a non-AWS, S3-compatible service # AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME # non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME, -# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL +# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL. +# AWS_S3_URL_PROTOCOL must end in ":" and defaults to the same protocol as +# the BookWyrm instance ("http:" or "https:", based on USE_SSL). # AWS_STORAGE_BUCKET_NAME= # "example-bucket-name" # AWS_S3_CUSTOM_DOMAIN=None # "example-bucket-name.s3.fr-par.scw.cloud" +# AWS_S3_URL_PROTOCOL=None # "http:" # AWS_S3_REGION_NAME=None # "fr-par" # AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud" @@ -133,9 +144,9 @@ HTTP_X_FORWARDED_PROTO=false TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2 TWO_FACTOR_LOGIN_MAX_SECONDS=60 -# Additional hosts to allow in the Content-Security-Policy, "self" (should be DOMAIN) -# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default. -# Value should be a comma-separated list of host names. +# Additional hosts to allow in the Content-Security-Policy, "self" (should be +# DOMAIN with optionally ":" + PORT) and AWS_S3_CUSTOM_DOMAIN (if used) are +# added by default. Value should be a comma-separated list of host names. CSP_ADDITIONAL_HOSTS= # Time before being logged out (in seconds) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..570174248 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,68 @@ + +## Description + + + + + +- Related Issue # +- Closes # + +## What type of Pull Request is this? + + +- [ ] Bug Fix +- [ ] Enhancement +- [ ] Plumbing / Internals / Dependencies +- [ ] Refactor + +## Does this PR change settings or dependencies, or break something? + + +- [ ] This PR changes or adds default settings, configuration, or .env values +- [ ] This PR changes or adds dependencies +- [ ] This PR introduces other breaking changes + +### Details of breaking or configuration changes (if any of above checked) + + +## Documentation + + + + +- [ ] New or amended documentation will be required if this PR is merged +- [ ] I have created a matching pull request in the Documentation repository +- [ ] I intend to create a matching pull request in the Documentation repository after this PR is merged + + + +### Tests + + +- [ ] My changes do not need new tests +- [ ] All tests I have added are passing +- [ ] I have written tests but need help to make them pass +- [ ] I have not written tests and need help to write them diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 000000000..3a347bf51 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,26 @@ +changelog: + exclude: + labels: + - ignore-for-release + categories: + - title: ‼️ Breaking Changes & New Settings ⚙️ + labels: + - breaking-change + - config-change + - title: Updated Dependencies 🧸 + labels: + - dependencies + - title: New Features 🎉 + labels: + - enhancement + - title: Bug Fixes 🐛 + labels: + - fix + - bug + - title: Internals/Plumbing 👩‍🔧 + - plumbing + - tests + - deployment + - title: Other Changes + labels: + - "*" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 51316ef62..014745a52 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -40,7 +40,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -51,7 +51,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -65,4 +65,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/lint-frontend.yaml b/.github/workflows/lint-frontend.yaml index 21f11ebf3..b0322f371 100644 --- a/.github/workflows/lint-frontend.yaml +++ b/.github/workflows/lint-frontend.yaml @@ -22,7 +22,8 @@ jobs: - uses: actions/checkout@v4 - name: Install modules - run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint + # run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint + run: npm install eslint@^8.9.0 # See .stylelintignore for files that are not linted. # - name: Run stylelint diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 01241b467..68e3b7b65 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -48,7 +48,7 @@ jobs: - name: Set up .env run: cp .env.example .env - name: Check migrations up-to-date - run: python ./manage.py makemigrations --check + run: python ./manage.py makemigrations --check -v 3 - name: Run Tests run: pytest -n 3 diff --git a/.gitignore b/.gitignore index 755375b34..fd6cc7547 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ # BookWyrm .env /images/ +/exports/ /static/ bookwyrm/static/css/bookwyrm.css bookwyrm/static/css/themes/ @@ -37,3 +38,6 @@ nginx/default.conf #macOS **/.DS_Store + +# Docker +docker-compose.override.yml diff --git a/.pylintrc b/.pylintrc index 464638853..e89f7d536 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,7 +3,19 @@ ignore=migrations load-plugins=pylint.extensions.no_self_use [MESSAGES CONTROL] -disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801,C3001,import-error +disable = + cyclic-import, + duplicate-code, + fixme, + no-member, + raise-missing-from, + too-few-public-methods, + too-many-ancestors, + too-many-instance-attributes, + unnecessary-lambda-assignment, + unsubscriptable-object, +enable = + useless-suppression [FORMAT] max-line-length=88 diff --git a/README.md b/README.md index f8b2eb1f6..7e27d44e6 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ BookWyrm is a social network for tracking your reading, talking about books, wri ## Links [![Mastodon Follow](https://img.shields.io/mastodon/follow/000146121?domain=https%3A%2F%2Ftech.lgbt&style=social)](https://tech.lgbt/@bookwyrm) -[![Twitter Follow](https://img.shields.io/twitter/follow/BookWyrmSocial?style=social)](https://twitter.com/BookWyrmSocial) - [Project homepage](https://joinbookwyrm.com/) - [Support](https://patreon.com/bookwyrm) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index efc9d8da2..dc4b8f6ae 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -250,7 +250,10 @@ class ActivityObject: pass data = {k: v for (k, v) in data.items() if v is not None and k not in omit} if "@context" not in omit: - data["@context"] = "https://www.w3.org/ns/activitystreams" + data["@context"] = [ + "https://www.w3.org/ns/activitystreams", + {"Hashtag": "as:Hashtag"}, + ] return data @@ -400,11 +403,11 @@ def get_representative(): to sign outgoing HTTP GET requests""" return models.User.objects.get_or_create( username=f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}", - defaults=dict( - email="bookwyrm@localhost", - local=True, - localname=INSTANCE_ACTOR_USERNAME, - ), + defaults={ + "email": "bookwyrm@localhost", + "local": True, + "localname": INSTANCE_ACTOR_USERNAME, + }, )[0] diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py index a53222053..9a268f905 100644 --- a/bookwyrm/activitypub/book.py +++ b/bookwyrm/activitypub/book.py @@ -67,7 +67,6 @@ class Edition(Book): type: str = "Edition" -# pylint: disable=invalid-name @dataclass(init=False) class Work(Book): """work instance of a book object""" diff --git a/bookwyrm/activitypub/ordered_collection.py b/bookwyrm/activitypub/ordered_collection.py index 32e37c996..250490041 100644 --- a/bookwyrm/activitypub/ordered_collection.py +++ b/bookwyrm/activitypub/ordered_collection.py @@ -18,7 +18,6 @@ class OrderedCollection(ActivityObject): type: str = "OrderedCollection" -# pylint: disable=invalid-name @dataclass(init=False) class OrderedCollectionPrivate(OrderedCollection): """an ordered collection with privacy settings""" diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py index a365f4cc0..549f14c9c 100644 --- a/bookwyrm/activitypub/verbs.py +++ b/bookwyrm/activitypub/verbs.py @@ -22,7 +22,6 @@ class Verb(ActivityObject): self.object.to_model(allow_external_connections=allow_external_connections) -# pylint: disable=invalid-name @dataclass(init=False) class Create(Verb): """Create activity""" @@ -33,7 +32,6 @@ class Create(Verb): type: str = "Create" -# pylint: disable=invalid-name @dataclass(init=False) class Delete(Verb): """Create activity""" @@ -63,7 +61,6 @@ class Delete(Verb): # if we can't find it, we don't need to delete it because we don't have it -# pylint: disable=invalid-name @dataclass(init=False) class Update(Verb): """Update activity""" @@ -227,7 +224,6 @@ class Like(Verb): self.to_model(allow_external_connections=allow_external_connections) -# pylint: disable=invalid-name @dataclass(init=False) class Announce(Verb): """boosting a status""" diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 42f99e209..08fb757d5 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -32,7 +32,7 @@ class ActivityStream(RedisStore): stream_id = self.stream_id(user_id) return f"{stream_id}-unread-by-type" - def get_rank(self, obj): # pylint: disable=no-self-use + def get_rank(self, obj): """statuses are sorted by date published""" return obj.published_date.timestamp() @@ -139,14 +139,14 @@ class ActivityStream(RedisStore): | ( Q(following=status.user) & Q(following=status.reply_parent.user) ) # if the user is following both authors - ).distinct() + ) # only visible to the poster's followers and tagged users elif status.privacy == "followers": audience = audience.filter( Q(following=status.user) # if the user is following the author ) - return audience.distinct() + return audience.distinct("id") @tracer.start_as_current_span("ActivityStream.get_audience") def get_audience(self, status): @@ -156,7 +156,7 @@ class ActivityStream(RedisStore): status_author = models.User.objects.filter( is_active=True, local=True, id=status.user.id ).values_list("id", flat=True) - return list(set(list(audience) + list(status_author))) + return list(set(audience) | set(status_author)) def get_stores_for_users(self, user_ids): """convert a list of user ids into redis store ids""" @@ -183,15 +183,13 @@ class HomeStream(ActivityStream): def get_audience(self, status): trace.get_current_span().set_attribute("stream_id", self.key) audience = super()._get_audience(status) - if not audience: - return [] # if the user is following the author audience = audience.filter(following=status.user).values_list("id", flat=True) # if the user is the post's author status_author = models.User.objects.filter( is_active=True, local=True, id=status.user.id ).values_list("id", flat=True) - return list(set(list(audience) + list(status_author))) + return list(set(audience) | set(status_author)) def get_statuses_for_user(self, user): return models.Status.privacy_filter( @@ -239,9 +237,7 @@ class BooksStream(ActivityStream): ) audience = super()._get_audience(status) - if not audience: - return models.User.objects.none() - return audience.filter(shelfbook__book__parent_work=work).distinct() + return audience.filter(shelfbook__book__parent_work=work) def get_audience(self, status): # only show public statuses on the books feed, diff --git a/bookwyrm/apps.py b/bookwyrm/apps.py index 41b1a17a2..d5384bb7b 100644 --- a/bookwyrm/apps.py +++ b/bookwyrm/apps.py @@ -33,7 +33,6 @@ class BookwyrmConfig(AppConfig): name = "bookwyrm" verbose_name = "BookWyrm" - # pylint: disable=no-self-use def ready(self): """set up OTLP and preview image files, if desired""" if settings.OTEL_EXPORTER_OTLP_ENDPOINT or settings.OTEL_EXPORTER_CONSOLE: diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index cf48f4832..c11cdb8f1 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -36,7 +36,6 @@ def search( ... -# pylint: disable=arguments-differ def search( query: str, *, diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index 444a626ba..ad68af1dc 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -118,9 +118,11 @@ def get_connectors() -> Iterator[abstract_connector.AbstractConnector]: def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnector: """get the connector related to the object's server""" url = urlparse(remote_id) - identifier = url.netloc + identifier = url.hostname if not identifier: - raise ValueError("Invalid remote id") + raise ValueError(f"Invalid remote id: {remote_id}") + + base_url = f"{url.scheme}://{url.netloc}" try: connector_info = models.Connector.objects.get(identifier=identifier) @@ -128,10 +130,10 @@ def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnec connector_info = models.Connector.objects.create( identifier=identifier, connector_file="bookwyrm_connector", - base_url=f"https://{identifier}", - books_url=f"https://{identifier}/book", - covers_url=f"https://{identifier}/images/covers", - search_url=f"https://{identifier}/search?q=", + base_url=base_url, + books_url=f"{base_url}/book", + covers_url=f"{base_url}/images/covers", + search_url=f"{base_url}/search?q=", priority=2, ) @@ -143,7 +145,9 @@ def load_more_data(connector_id: str, book_id: str) -> None: """background the work of getting all 10,000 editions of LoTR""" connector_info = models.Connector.objects.get(id=connector_id) connector = load_connector(connector_info) - book = models.Book.objects.select_subclasses().get(id=book_id) + book = models.Book.objects.select_subclasses().get( # type: ignore[no-untyped-call] + id=book_id + ) connector.expand_book_data(book) @@ -154,7 +158,9 @@ def create_edition_task( """separate task for each of the 10,000 editions of LoTR""" connector_info = models.Connector.objects.get(id=connector_id) connector = load_connector(connector_info) - work = models.Work.objects.select_subclasses().get(id=work_id) + work = models.Work.objects.select_subclasses().get( # type: ignore[no-untyped-call] + id=work_id + ) connector.create_edition_from_data(work, data) @@ -188,8 +194,11 @@ def raise_not_valid_url(url: str) -> None: if not parsed.scheme in ["http", "https"]: raise ConnectorException("Invalid scheme: ", url) + if not parsed.hostname: + raise ConnectorException("Hostname missing: ", url) + try: - ipaddress.ip_address(parsed.netloc) + ipaddress.ip_address(parsed.hostname) raise ConnectorException("Provided url is an IP address: ", url) except ValueError: # it's not an IP address, which is good diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py index c08bcdee1..249f6b9ca 100644 --- a/bookwyrm/connectors/inventaire.py +++ b/bookwyrm/connectors/inventaire.py @@ -229,7 +229,7 @@ class Connector(AbstractConnector): data = get_data(url) except ConnectorException: return "" - return data.get("extract", "") + return str(data.get("extract", "")) def get_remote_id_from_model(self, obj: models.BookDataModel) -> str: """use get_remote_id to figure out the link from a model obj""" diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py index 0047bfce1..bec704a2c 100644 --- a/bookwyrm/context_processors.py +++ b/bookwyrm/context_processors.py @@ -2,7 +2,7 @@ from bookwyrm import models, settings -def site_settings(request): # pylint: disable=unused-argument +def site_settings(request): """include the custom info about the site""" request_protocol = "https://" if not request.is_secure(): diff --git a/bookwyrm/emailing.py b/bookwyrm/emailing.py index 5e08ebba1..ccc0aea61 100644 --- a/bookwyrm/emailing.py +++ b/bookwyrm/emailing.py @@ -4,7 +4,7 @@ from django.template.loader import get_template from bookwyrm import models, settings from bookwyrm.tasks import app, EMAIL -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import DOMAIN, BASE_URL def email_data(): @@ -14,6 +14,7 @@ def email_data(): "site_name": site.name, "logo": site.logo_small_url, "domain": DOMAIN, + "base_url": BASE_URL, "user": None, } diff --git a/bookwyrm/forms/custom_form.py b/bookwyrm/forms/custom_form.py index c604deea4..6b425d216 100644 --- a/bookwyrm/forms/custom_form.py +++ b/bookwyrm/forms/custom_form.py @@ -15,9 +15,9 @@ class StyledForm(ModelForm): css_classes["number"] = "input" css_classes["checkbox"] = "checkbox" css_classes["textarea"] = "textarea" - # pylint: disable=super-with-arguments super().__init__(*args, **kwargs) for visible in self.visible_fields(): + input_type = "" if hasattr(visible.field.widget, "input_type"): input_type = visible.field.widget.input_type if isinstance(visible.field.widget, Textarea): diff --git a/bookwyrm/forms/edit_user.py b/bookwyrm/forms/edit_user.py index 9024972c3..13ac285d8 100644 --- a/bookwyrm/forms/edit_user.py +++ b/bookwyrm/forms/edit_user.py @@ -18,6 +18,7 @@ class EditUserForm(CustomForm): "email", "summary", "show_goal", + "show_ratings", "show_suggested_users", "manually_approves_followers", "default_post_privacy", diff --git a/bookwyrm/forms/landing.py b/bookwyrm/forms/landing.py index 1da4fc4f1..831d1d539 100644 --- a/bookwyrm/forms/landing.py +++ b/bookwyrm/forms/landing.py @@ -34,7 +34,6 @@ class LoginForm(CustomForm): def add_invalid_password_error(self): """We don't want to be too specific about this""" - # pylint: disable=attribute-defined-outside-init self.non_field_errors = _("Username or password are incorrect") diff --git a/bookwyrm/forms/links.py b/bookwyrm/forms/links.py index 345c5c1d4..5156d2578 100644 --- a/bookwyrm/forms/links.py +++ b/bookwyrm/forms/links.py @@ -26,7 +26,7 @@ class FileLinkForm(CustomForm): url = cleaned_data.get("url") filetype = cleaned_data.get("filetype") book = cleaned_data.get("book") - domain = urlparse(url).netloc + domain = urlparse(url).hostname if models.LinkDomain.objects.filter(domain=domain).exists(): status = models.LinkDomain.objects.get(domain=domain).status if status == "blocked": diff --git a/bookwyrm/forms/widgets.py b/bookwyrm/forms/widgets.py index ee9345aa0..001fdbec4 100644 --- a/bookwyrm/forms/widgets.py +++ b/bookwyrm/forms/widgets.py @@ -5,8 +5,6 @@ from django import forms class ArrayWidget(forms.widgets.TextInput): """Inputs for postgres array fields""" - # pylint: disable=unused-argument - # pylint: disable=no-self-use def value_from_datadict(self, data, files, name): """get all values for this name""" return [i for i in data.getlist(name) if i] diff --git a/bookwyrm/importers/__init__.py b/bookwyrm/importers/__init__.py index 8e92872f2..3c895741b 100644 --- a/bookwyrm/importers/__init__.py +++ b/bookwyrm/importers/__init__.py @@ -1,7 +1,7 @@ """ import classes """ from .importer import Importer -from .bookwyrm_import import BookwyrmImporter +from .bookwyrm_import import BookwyrmImporter, BookwyrmBooksImporter from .calibre_import import CalibreImporter from .goodreads_import import GoodreadsImporter from .librarything_import import LibrarythingImporter diff --git a/bookwyrm/importers/bookwyrm_import.py b/bookwyrm/importers/bookwyrm_import.py index 206cd6219..8afc1abf7 100644 --- a/bookwyrm/importers/bookwyrm_import.py +++ b/bookwyrm/importers/bookwyrm_import.py @@ -3,6 +3,7 @@ from django.http import QueryDict from bookwyrm.models import User from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob +from . import Importer class BookwyrmImporter: @@ -22,3 +23,17 @@ class BookwyrmImporter: user=user, archive_file=archive_file, required=required ) return job + + +class BookwyrmBooksImporter(Importer): + """ + Handle reading a csv from BookWyrm. + Goodreads is the default importer, we basically just use the same structure + But BookWyrm has additional attributes in the csv + """ + + service = "BookWyrm" + row_mappings_guesses = Importer.row_mappings_guesses + [ + ("shelf_name", ["shelf_name"]), + ("review_published", ["review_published"]), + ] diff --git a/bookwyrm/importers/calibre_import.py b/bookwyrm/importers/calibre_import.py index 5c22a539d..542175dd7 100644 --- a/bookwyrm/importers/calibre_import.py +++ b/bookwyrm/importers/calibre_import.py @@ -14,15 +14,10 @@ class CalibreImporter(Importer): def __init__(self, *args: Any, **kwargs: Any): # Add timestamp to row_mappings_guesses for date_added to avoid # integrity error - row_mappings_guesses = [] - - for field, mapping in self.row_mappings_guesses: - if field in ("date_added",): - row_mappings_guesses.append((field, mapping + ["timestamp"])) - else: - row_mappings_guesses.append((field, mapping)) - - self.row_mappings_guesses = row_mappings_guesses + self.row_mappings_guesses = [ + (field, mapping + (["timestamp"] if field == "date_added" else [])) + for field, mapping in self.row_mappings_guesses + ] super().__init__(*args, **kwargs) def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]: diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 5b3192fa5..d2a11d7f2 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -18,17 +18,26 @@ class Importer: row_mappings_guesses = [ ("id", ["id", "book id"]), ("title", ["title"]), - ("authors", ["author", "authors", "primary author"]), - ("isbn_10", ["isbn10", "isbn", "isbn/uid"]), - ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]), + ("authors", ["author_text", "author", "authors", "primary author"]), + ("isbn_10", ["isbn_10", "isbn10", "isbn", "isbn/uid"]), + ("isbn_13", ["isbn_13", "isbn13", "isbn", "isbns", "isbn/uid"]), ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]), - ("review_name", ["review name"]), - ("review_body", ["my review", "review"]), + ("review_name", ["review_name", "review name"]), + ("review_body", ["review_content", "my review", "review"]), ("rating", ["my rating", "rating", "star rating"]), - ("date_added", ["date added", "entry date", "added"]), - ("date_started", ["date started", "started"]), - ("date_finished", ["date finished", "last date read", "date read", "finished"]), + ( + "date_added", + ["shelf_date", "date_added", "date added", "entry date", "added"], + ), + ("date_started", ["start_date", "date started", "started"]), + ( + "date_finished", + ["finish_date", "date finished", "last date read", "date read", "finished"], + ), ] + + # TODO: stopped + date_fields = ["date_added", "date_started", "date_finished"] shelf_mapping_guesses = { "to-read": ["to-read", "want to read"], @@ -36,9 +45,14 @@ class Importer: "reading": ["currently-reading", "reading", "currently reading"], } - # pylint: disable=too-many-locals + # pylint: disable=too-many-arguments def create_job( - self, user: User, csv_file: Iterable[str], include_reviews: bool, privacy: str + self, + user: User, + csv_file: Iterable[str], + include_reviews: bool, + privacy: str, + create_shelves: bool = True, ) -> ImportJob: """check over a csv and creates a database entry for the job""" csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter) @@ -55,6 +69,7 @@ class Importer: job = ImportJob.objects.create( user=user, include_reviews=include_reviews, + create_shelves=create_shelves, privacy=privacy, mappings=mappings, source=self.service, @@ -114,7 +129,7 @@ class Importer: shelf = [ s for (s, gs) in self.shelf_mapping_guesses.items() if shelf_name in gs ] - return shelf[0] if shelf else None + return shelf[0] if shelf else normalized_row.get("shelf") or None # pylint: disable=no-self-use def normalize_row( @@ -149,6 +164,7 @@ class Importer: job = ImportJob.objects.create( user=user, include_reviews=original_job.include_reviews, + create_shelves=original_job.create_shelves, privacy=original_job.privacy, source=original_job.source, # TODO: allow users to adjust mappings diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py index 145657ba0..24a2626bf 100644 --- a/bookwyrm/importers/librarything_import.py +++ b/bookwyrm/importers/librarything_import.py @@ -20,7 +20,7 @@ class LibrarythingImporter(Importer): def normalize_row( self, entry: dict[str, str], mappings: dict[str, Optional[str]] - ) -> dict[str, Optional[str]]: # pylint: disable=no-self-use + ) -> dict[str, Optional[str]]: """use the dataclass to create the formatted row of data""" normalized = { k: _remove_brackets(entry.get(v) if v else None) diff --git a/bookwyrm/lists_stream.py b/bookwyrm/lists_stream.py index 148b81a78..479eacb19 100644 --- a/bookwyrm/lists_stream.py +++ b/bookwyrm/lists_stream.py @@ -18,7 +18,7 @@ class ListsStream(RedisStore): return f"{user}-lists" return f"{user.id}-lists" - def get_rank(self, obj): # pylint: disable=no-self-use + def get_rank(self, obj): """lists are sorted by updated date""" return obj.updated_date.timestamp() diff --git a/bookwyrm/management/commands/deduplicate_book_data.py b/bookwyrm/management/commands/deduplicate_book_data.py index dde7d133c..c2d897ce3 100644 --- a/bookwyrm/management/commands/deduplicate_book_data.py +++ b/bookwyrm/management/commands/deduplicate_book_data.py @@ -1,13 +1,14 @@ """ PROCEED WITH CAUTION: uses deduplication fields to permanently merge book data objects """ + from django.core.management.base import BaseCommand from django.db.models import Count from bookwyrm import models -from bookwyrm.management.merge import merge_objects -def dedupe_model(model): +def dedupe_model(model, dry_run=False): """combine duplicate editions and update related models""" + print(f"deduplicating {model.__name__}:") fields = model._meta.get_fields() dedupe_fields = [ f for f in fields if hasattr(f, "deduplication_field") and f.deduplication_field @@ -16,30 +17,42 @@ def dedupe_model(model): dupes = ( model.objects.values(field.name) .annotate(Count(field.name)) - .filter(**{"%s__count__gt" % field.name: 1}) + .filter(**{f"{field.name}__count__gt": 1}) + .exclude(**{field.name: ""}) + .exclude(**{f"{field.name}__isnull": True}) ) for dupe in dupes: value = dupe[field.name] - if not value or value == "": - continue print("----------") - print(dupe) objs = model.objects.filter(**{field.name: value}).order_by("id") canonical = objs.first() - print("keeping", canonical.remote_id) + action = "would merge" if dry_run else "merging" + print( + f"{action} into {model.__name__} {canonical.remote_id} based on {field.name} {value}:" + ) for obj in objs[1:]: - print(obj.remote_id) - merge_objects(canonical, obj) + print(f"- {obj.remote_id}") + absorbed_fields = obj.merge_into(canonical, dry_run=dry_run) + print(f" absorbed fields: {absorbed_fields}") class Command(BaseCommand): """deduplicate allllll the book data models""" help = "merges duplicate book data" + + def add_arguments(self, parser): + """add the arguments for this command""" + parser.add_argument( + "--dry_run", + action="store_true", + help="don't actually merge, only print what would happen", + ) + # pylint: disable=no-self-use,unused-argument def handle(self, *args, **options): """run deduplications""" - dedupe_model(models.Edition) - dedupe_model(models.Work) - dedupe_model(models.Author) + dedupe_model(models.Edition, dry_run=options["dry_run"]) + dedupe_model(models.Work, dry_run=options["dry_run"]) + dedupe_model(models.Author, dry_run=options["dry_run"]) diff --git a/bookwyrm/management/merge.py b/bookwyrm/management/merge.py deleted file mode 100644 index f55229f18..000000000 --- a/bookwyrm/management/merge.py +++ /dev/null @@ -1,50 +0,0 @@ -from django.db.models import ManyToManyField - - -def update_related(canonical, obj): - """update all the models with fk to the object being removed""" - # move related models to canonical - related_models = [ - (r.remote_field.name, r.related_model) for r in canonical._meta.related_objects - ] - for (related_field, related_model) in related_models: - # Skip the ManyToMany fields that aren’t auto-created. These - # should have a corresponding OneToMany field in the model for - # the linking table anyway. If we update it through that model - # instead then we won’t lose the extra fields in the linking - # table. - related_field_obj = related_model._meta.get_field(related_field) - if isinstance(related_field_obj, ManyToManyField): - through = related_field_obj.remote_field.through - if not through._meta.auto_created: - continue - related_objs = related_model.objects.filter(**{related_field: obj}) - for related_obj in related_objs: - print("replacing in", related_model.__name__, related_field, related_obj.id) - try: - setattr(related_obj, related_field, canonical) - related_obj.save() - except TypeError: - getattr(related_obj, related_field).add(canonical) - getattr(related_obj, related_field).remove(obj) - - -def copy_data(canonical, obj): - """try to get the most data possible""" - for data_field in obj._meta.get_fields(): - if not hasattr(data_field, "activitypub_field"): - continue - data_value = getattr(obj, data_field.name) - if not data_value: - continue - if not getattr(canonical, data_field.name): - print("setting data field", data_field.name, data_value) - setattr(canonical, data_field.name, data_value) - canonical.save() - - -def merge_objects(canonical, obj): - copy_data(canonical, obj) - update_related(canonical, obj) - # remove the outdated entry - obj.delete() diff --git a/bookwyrm/management/merge_command.py b/bookwyrm/management/merge_command.py index 805dc73fa..66e60814a 100644 --- a/bookwyrm/management/merge_command.py +++ b/bookwyrm/management/merge_command.py @@ -1,4 +1,3 @@ -from bookwyrm.management.merge import merge_objects from django.core.management.base import BaseCommand @@ -9,6 +8,11 @@ class MergeCommand(BaseCommand): """add the arguments for this command""" parser.add_argument("--canonical", type=int, required=True) parser.add_argument("--other", type=int, required=True) + parser.add_argument( + "--dry_run", + action="store_true", + help="don't actually merge, only print what would happen", + ) # pylint: disable=no-self-use,unused-argument def handle(self, *args, **options): @@ -26,4 +30,8 @@ class MergeCommand(BaseCommand): print("other book doesn’t exist!") return - merge_objects(canonical, other) + absorbed_fields = other.merge_into(canonical, dry_run=options["dry_run"]) + + action = "would be" if options["dry_run"] else "has been" + print(f"{other.remote_id} {action} merged into {canonical.remote_id}") + print(f"absorbed fields: {absorbed_fields}") diff --git a/bookwyrm/middleware/timezone_middleware.py b/bookwyrm/middleware/timezone_middleware.py index 5033397a5..3cf084154 100644 --- a/bookwyrm/middleware/timezone_middleware.py +++ b/bookwyrm/middleware/timezone_middleware.py @@ -1,5 +1,5 @@ """ Makes the app aware of the users timezone """ -import pytz +import zoneinfo from django.utils import timezone @@ -12,9 +12,7 @@ class TimezoneMiddleware: def __call__(self, request): if request.user.is_authenticated: - timezone.activate(pytz.timezone(request.user.preferred_timezone)) + timezone.activate(zoneinfo.ZoneInfo(request.user.preferred_timezone)) else: - timezone.activate(pytz.utc) - response = self.get_response(request) - timezone.deactivate() - return response + timezone.deactivate() + return self.get_response(request) diff --git a/bookwyrm/migrations/0171_alter_user_preferred_timezone.py b/bookwyrm/migrations/0171_alter_user_preferred_timezone.py index 7dcd9546c..8d1dff553 100644 --- a/bookwyrm/migrations/0171_alter_user_preferred_timezone.py +++ b/bookwyrm/migrations/0171_alter_user_preferred_timezone.py @@ -10,6 +10,7 @@ class Migration(migrations.Migration): ] operations = [ + # The new timezones are "Factory" and "localtime" migrations.AlterField( model_name="user", name="preferred_timezone", diff --git a/bookwyrm/migrations/0189_importjob_create_shelves.py b/bookwyrm/migrations/0189_importjob_create_shelves.py new file mode 100644 index 000000000..a1b1fc512 --- /dev/null +++ b/bookwyrm/migrations/0189_importjob_create_shelves.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2023-11-25 05:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0188_theme_loads"), + ] + + operations = [ + migrations.AddField( + model_name="importjob", + name="create_shelves", + field=models.BooleanField(default=True), + ), + ] diff --git a/bookwyrm/migrations/0193_auto_20240128_0249.py b/bookwyrm/migrations/0193_auto_20240128_0249.py new file mode 100644 index 000000000..82e32ee48 --- /dev/null +++ b/bookwyrm/migrations/0193_auto_20240128_0249.py @@ -0,0 +1,92 @@ +# Generated by Django 3.2.23 on 2024-01-28 02:49 + +import django.core.serializers.json +from django.db import migrations, models +import django.db.models.deletion +from django.core.files.storage import storages + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0192_sitesettings_user_exports_enabled"), + ] + + operations = [ + migrations.AddField( + model_name="bookwyrmexportjob", + name="export_json", + field=models.JSONField( + encoder=django.core.serializers.json.DjangoJSONEncoder, null=True + ), + ), + migrations.AddField( + model_name="bookwyrmexportjob", + name="json_completed", + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name="bookwyrmexportjob", + name="export_data", + field=models.FileField( + null=True, + storage=storages["exports"], + upload_to="", + ), + ), + migrations.CreateModel( + name="AddFileToTar", + fields=[ + ( + "childjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.childjob", + ), + ), + ( + "parent_export_job", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="child_edition_export_jobs", + to="bookwyrm.bookwyrmexportjob", + ), + ), + ], + options={ + "abstract": False, + }, + bases=("bookwyrm.childjob",), + ), + migrations.CreateModel( + name="AddBookToUserExportJob", + fields=[ + ( + "childjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.childjob", + ), + ), + ( + "edition", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="bookwyrm.edition", + ), + ), + ], + options={ + "abstract": False, + }, + bases=("bookwyrm.childjob",), + ), + ] diff --git a/bookwyrm/migrations/0196_merge_20240318_1737.py b/bookwyrm/migrations/0196_merge_20240318_1737.py new file mode 100644 index 000000000..2d80b2e58 --- /dev/null +++ b/bookwyrm/migrations/0196_merge_20240318_1737.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.23 on 2024-03-18 17:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0193_auto_20240128_0249"), + ("bookwyrm", "0195_alter_user_preferred_language"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0197_merge_20240324_0235.py b/bookwyrm/migrations/0197_merge_20240324_0235.py new file mode 100644 index 000000000..a7c01a955 --- /dev/null +++ b/bookwyrm/migrations/0197_merge_20240324_0235.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.25 on 2024-03-24 02:35 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0196_merge_20240318_1737"), + ("bookwyrm", "0196_merge_pr3134_into_main"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0197_mergedauthor_mergedbook.py b/bookwyrm/migrations/0197_mergedauthor_mergedbook.py new file mode 100644 index 000000000..23ca38ab2 --- /dev/null +++ b/bookwyrm/migrations/0197_mergedauthor_mergedbook.py @@ -0,0 +1,48 @@ +# Generated by Django 3.2.24 on 2024-02-28 21:30 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0196_merge_pr3134_into_main"), + ] + + operations = [ + migrations.CreateModel( + name="MergedBook", + fields=[ + ("deleted_id", models.IntegerField(primary_key=True, serialize=False)), + ( + "merged_into", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="absorbed", + to="bookwyrm.book", + ), + ), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="MergedAuthor", + fields=[ + ("deleted_id", models.IntegerField(primary_key=True, serialize=False)), + ( + "merged_into", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="absorbed", + to="bookwyrm.author", + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/bookwyrm/migrations/0198_alter_bookwyrmexportjob_export_data.py b/bookwyrm/migrations/0198_alter_bookwyrmexportjob_export_data.py new file mode 100644 index 000000000..552584d2b --- /dev/null +++ b/bookwyrm/migrations/0198_alter_bookwyrmexportjob_export_data.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.25 on 2024-03-26 11:37 + +import bookwyrm.models.bookwyrm_export_job +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0197_merge_20240324_0235"), + ] + + operations = [ + migrations.AlterField( + model_name="bookwyrmexportjob", + name="export_data", + field=models.FileField( + null=True, + storage=bookwyrm.models.bookwyrm_export_job.select_exports_storage, + upload_to="", + ), + ), + ] diff --git a/bookwyrm/migrations/0199_alter_userblocks_user_object_and_more.py b/bookwyrm/migrations/0199_alter_userblocks_user_object_and_more.py new file mode 100644 index 000000000..bde1f25c1 --- /dev/null +++ b/bookwyrm/migrations/0199_alter_userblocks_user_object_and_more.py @@ -0,0 +1,70 @@ +# Generated by Django 4.2.11 on 2024-03-29 19:25 + +import bookwyrm.models.fields +from django.conf import settings +from django.db import migrations +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0198_book_search_vector_author_aliases"), + ] + + operations = [ + migrations.AlterField( + model_name="userblocks", + name="user_object", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_object", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="userblocks", + name="user_subject", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_subject", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="userfollowrequest", + name="user_object", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_object", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="userfollowrequest", + name="user_subject", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_subject", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="userfollows", + name="user_object", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_object", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="userfollows", + name="user_subject", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_subject", + to=settings.AUTH_USER_MODEL, + ), + ), + ] diff --git a/bookwyrm/migrations/0199_merge_20240326_1217.py b/bookwyrm/migrations/0199_merge_20240326_1217.py new file mode 100644 index 000000000..7794af54a --- /dev/null +++ b/bookwyrm/migrations/0199_merge_20240326_1217.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.25 on 2024-03-26 12:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0198_alter_bookwyrmexportjob_export_data"), + ("bookwyrm", "0198_book_search_vector_author_aliases"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0199_status_bookwyrm_st_remote__06aeba_idx.py b/bookwyrm/migrations/0199_status_bookwyrm_st_remote__06aeba_idx.py new file mode 100644 index 000000000..5d2513698 --- /dev/null +++ b/bookwyrm/migrations/0199_status_bookwyrm_st_remote__06aeba_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-04-02 19:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0198_book_search_vector_author_aliases"), + ] + + operations = [ + migrations.AddIndex( + model_name="status", + index=models.Index( + fields=["remote_id"], name="bookwyrm_st_remote__06aeba_idx" + ), + ), + ] diff --git a/bookwyrm/migrations/0200_alter_user_preferred_timezone.py b/bookwyrm/migrations/0200_alter_user_preferred_timezone.py new file mode 100644 index 000000000..1b21c0f94 --- /dev/null +++ b/bookwyrm/migrations/0200_alter_user_preferred_timezone.py @@ -0,0 +1,633 @@ +# Generated by Django 4.2.11 on 2024-04-01 20:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0199_alter_userblocks_user_object_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="preferred_timezone", + field=models.CharField( + choices=[ + ("Africa/Abidjan", "Africa/Abidjan"), + ("Africa/Accra", "Africa/Accra"), + ("Africa/Addis_Ababa", "Africa/Addis_Ababa"), + ("Africa/Algiers", "Africa/Algiers"), + ("Africa/Asmara", "Africa/Asmara"), + ("Africa/Asmera", "Africa/Asmera"), + ("Africa/Bamako", "Africa/Bamako"), + ("Africa/Bangui", "Africa/Bangui"), + ("Africa/Banjul", "Africa/Banjul"), + ("Africa/Bissau", "Africa/Bissau"), + ("Africa/Blantyre", "Africa/Blantyre"), + ("Africa/Brazzaville", "Africa/Brazzaville"), + ("Africa/Bujumbura", "Africa/Bujumbura"), + ("Africa/Cairo", "Africa/Cairo"), + ("Africa/Casablanca", "Africa/Casablanca"), + ("Africa/Ceuta", "Africa/Ceuta"), + ("Africa/Conakry", "Africa/Conakry"), + ("Africa/Dakar", "Africa/Dakar"), + ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"), + ("Africa/Djibouti", "Africa/Djibouti"), + ("Africa/Douala", "Africa/Douala"), + ("Africa/El_Aaiun", "Africa/El_Aaiun"), + ("Africa/Freetown", "Africa/Freetown"), + ("Africa/Gaborone", "Africa/Gaborone"), + ("Africa/Harare", "Africa/Harare"), + ("Africa/Johannesburg", "Africa/Johannesburg"), + ("Africa/Juba", "Africa/Juba"), + ("Africa/Kampala", "Africa/Kampala"), + ("Africa/Khartoum", "Africa/Khartoum"), + ("Africa/Kigali", "Africa/Kigali"), + ("Africa/Kinshasa", "Africa/Kinshasa"), + ("Africa/Lagos", "Africa/Lagos"), + ("Africa/Libreville", "Africa/Libreville"), + ("Africa/Lome", "Africa/Lome"), + ("Africa/Luanda", "Africa/Luanda"), + ("Africa/Lubumbashi", "Africa/Lubumbashi"), + ("Africa/Lusaka", "Africa/Lusaka"), + ("Africa/Malabo", "Africa/Malabo"), + ("Africa/Maputo", "Africa/Maputo"), + ("Africa/Maseru", "Africa/Maseru"), + ("Africa/Mbabane", "Africa/Mbabane"), + ("Africa/Mogadishu", "Africa/Mogadishu"), + ("Africa/Monrovia", "Africa/Monrovia"), + ("Africa/Nairobi", "Africa/Nairobi"), + ("Africa/Ndjamena", "Africa/Ndjamena"), + ("Africa/Niamey", "Africa/Niamey"), + ("Africa/Nouakchott", "Africa/Nouakchott"), + ("Africa/Ouagadougou", "Africa/Ouagadougou"), + ("Africa/Porto-Novo", "Africa/Porto-Novo"), + ("Africa/Sao_Tome", "Africa/Sao_Tome"), + ("Africa/Timbuktu", "Africa/Timbuktu"), + ("Africa/Tripoli", "Africa/Tripoli"), + ("Africa/Tunis", "Africa/Tunis"), + ("Africa/Windhoek", "Africa/Windhoek"), + ("America/Adak", "America/Adak"), + ("America/Anchorage", "America/Anchorage"), + ("America/Anguilla", "America/Anguilla"), + ("America/Antigua", "America/Antigua"), + ("America/Araguaina", "America/Araguaina"), + ( + "America/Argentina/Buenos_Aires", + "America/Argentina/Buenos_Aires", + ), + ("America/Argentina/Catamarca", "America/Argentina/Catamarca"), + ( + "America/Argentina/ComodRivadavia", + "America/Argentina/ComodRivadavia", + ), + ("America/Argentina/Cordoba", "America/Argentina/Cordoba"), + ("America/Argentina/Jujuy", "America/Argentina/Jujuy"), + ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"), + ("America/Argentina/Mendoza", "America/Argentina/Mendoza"), + ( + "America/Argentina/Rio_Gallegos", + "America/Argentina/Rio_Gallegos", + ), + ("America/Argentina/Salta", "America/Argentina/Salta"), + ("America/Argentina/San_Juan", "America/Argentina/San_Juan"), + ("America/Argentina/San_Luis", "America/Argentina/San_Luis"), + ("America/Argentina/Tucuman", "America/Argentina/Tucuman"), + ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"), + ("America/Aruba", "America/Aruba"), + ("America/Asuncion", "America/Asuncion"), + ("America/Atikokan", "America/Atikokan"), + ("America/Atka", "America/Atka"), + ("America/Bahia", "America/Bahia"), + ("America/Bahia_Banderas", "America/Bahia_Banderas"), + ("America/Barbados", "America/Barbados"), + ("America/Belem", "America/Belem"), + ("America/Belize", "America/Belize"), + ("America/Blanc-Sablon", "America/Blanc-Sablon"), + ("America/Boa_Vista", "America/Boa_Vista"), + ("America/Bogota", "America/Bogota"), + ("America/Boise", "America/Boise"), + ("America/Buenos_Aires", "America/Buenos_Aires"), + ("America/Cambridge_Bay", "America/Cambridge_Bay"), + ("America/Campo_Grande", "America/Campo_Grande"), + ("America/Cancun", "America/Cancun"), + ("America/Caracas", "America/Caracas"), + ("America/Catamarca", "America/Catamarca"), + ("America/Cayenne", "America/Cayenne"), + ("America/Cayman", "America/Cayman"), + ("America/Chicago", "America/Chicago"), + ("America/Chihuahua", "America/Chihuahua"), + ("America/Ciudad_Juarez", "America/Ciudad_Juarez"), + ("America/Coral_Harbour", "America/Coral_Harbour"), + ("America/Cordoba", "America/Cordoba"), + ("America/Costa_Rica", "America/Costa_Rica"), + ("America/Creston", "America/Creston"), + ("America/Cuiaba", "America/Cuiaba"), + ("America/Curacao", "America/Curacao"), + ("America/Danmarkshavn", "America/Danmarkshavn"), + ("America/Dawson", "America/Dawson"), + ("America/Dawson_Creek", "America/Dawson_Creek"), + ("America/Denver", "America/Denver"), + ("America/Detroit", "America/Detroit"), + ("America/Dominica", "America/Dominica"), + ("America/Edmonton", "America/Edmonton"), + ("America/Eirunepe", "America/Eirunepe"), + ("America/El_Salvador", "America/El_Salvador"), + ("America/Ensenada", "America/Ensenada"), + ("America/Fort_Nelson", "America/Fort_Nelson"), + ("America/Fort_Wayne", "America/Fort_Wayne"), + ("America/Fortaleza", "America/Fortaleza"), + ("America/Glace_Bay", "America/Glace_Bay"), + ("America/Godthab", "America/Godthab"), + ("America/Goose_Bay", "America/Goose_Bay"), + ("America/Grand_Turk", "America/Grand_Turk"), + ("America/Grenada", "America/Grenada"), + ("America/Guadeloupe", "America/Guadeloupe"), + ("America/Guatemala", "America/Guatemala"), + ("America/Guayaquil", "America/Guayaquil"), + ("America/Guyana", "America/Guyana"), + ("America/Halifax", "America/Halifax"), + ("America/Havana", "America/Havana"), + ("America/Hermosillo", "America/Hermosillo"), + ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"), + ("America/Indiana/Knox", "America/Indiana/Knox"), + ("America/Indiana/Marengo", "America/Indiana/Marengo"), + ("America/Indiana/Petersburg", "America/Indiana/Petersburg"), + ("America/Indiana/Tell_City", "America/Indiana/Tell_City"), + ("America/Indiana/Vevay", "America/Indiana/Vevay"), + ("America/Indiana/Vincennes", "America/Indiana/Vincennes"), + ("America/Indiana/Winamac", "America/Indiana/Winamac"), + ("America/Indianapolis", "America/Indianapolis"), + ("America/Inuvik", "America/Inuvik"), + ("America/Iqaluit", "America/Iqaluit"), + ("America/Jamaica", "America/Jamaica"), + ("America/Jujuy", "America/Jujuy"), + ("America/Juneau", "America/Juneau"), + ("America/Kentucky/Louisville", "America/Kentucky/Louisville"), + ("America/Kentucky/Monticello", "America/Kentucky/Monticello"), + ("America/Knox_IN", "America/Knox_IN"), + ("America/Kralendijk", "America/Kralendijk"), + ("America/La_Paz", "America/La_Paz"), + ("America/Lima", "America/Lima"), + ("America/Los_Angeles", "America/Los_Angeles"), + ("America/Louisville", "America/Louisville"), + ("America/Lower_Princes", "America/Lower_Princes"), + ("America/Maceio", "America/Maceio"), + ("America/Managua", "America/Managua"), + ("America/Manaus", "America/Manaus"), + ("America/Marigot", "America/Marigot"), + ("America/Martinique", "America/Martinique"), + ("America/Matamoros", "America/Matamoros"), + ("America/Mazatlan", "America/Mazatlan"), + ("America/Mendoza", "America/Mendoza"), + ("America/Menominee", "America/Menominee"), + ("America/Merida", "America/Merida"), + ("America/Metlakatla", "America/Metlakatla"), + ("America/Mexico_City", "America/Mexico_City"), + ("America/Miquelon", "America/Miquelon"), + ("America/Moncton", "America/Moncton"), + ("America/Monterrey", "America/Monterrey"), + ("America/Montevideo", "America/Montevideo"), + ("America/Montreal", "America/Montreal"), + ("America/Montserrat", "America/Montserrat"), + ("America/Nassau", "America/Nassau"), + ("America/New_York", "America/New_York"), + ("America/Nipigon", "America/Nipigon"), + ("America/Nome", "America/Nome"), + ("America/Noronha", "America/Noronha"), + ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"), + ("America/North_Dakota/Center", "America/North_Dakota/Center"), + ( + "America/North_Dakota/New_Salem", + "America/North_Dakota/New_Salem", + ), + ("America/Nuuk", "America/Nuuk"), + ("America/Ojinaga", "America/Ojinaga"), + ("America/Panama", "America/Panama"), + ("America/Pangnirtung", "America/Pangnirtung"), + ("America/Paramaribo", "America/Paramaribo"), + ("America/Phoenix", "America/Phoenix"), + ("America/Port-au-Prince", "America/Port-au-Prince"), + ("America/Port_of_Spain", "America/Port_of_Spain"), + ("America/Porto_Acre", "America/Porto_Acre"), + ("America/Porto_Velho", "America/Porto_Velho"), + ("America/Puerto_Rico", "America/Puerto_Rico"), + ("America/Punta_Arenas", "America/Punta_Arenas"), + ("America/Rainy_River", "America/Rainy_River"), + ("America/Rankin_Inlet", "America/Rankin_Inlet"), + ("America/Recife", "America/Recife"), + ("America/Regina", "America/Regina"), + ("America/Resolute", "America/Resolute"), + ("America/Rio_Branco", "America/Rio_Branco"), + ("America/Rosario", "America/Rosario"), + ("America/Santa_Isabel", "America/Santa_Isabel"), + ("America/Santarem", "America/Santarem"), + ("America/Santiago", "America/Santiago"), + ("America/Santo_Domingo", "America/Santo_Domingo"), + ("America/Sao_Paulo", "America/Sao_Paulo"), + ("America/Scoresbysund", "America/Scoresbysund"), + ("America/Shiprock", "America/Shiprock"), + ("America/Sitka", "America/Sitka"), + ("America/St_Barthelemy", "America/St_Barthelemy"), + ("America/St_Johns", "America/St_Johns"), + ("America/St_Kitts", "America/St_Kitts"), + ("America/St_Lucia", "America/St_Lucia"), + ("America/St_Thomas", "America/St_Thomas"), + ("America/St_Vincent", "America/St_Vincent"), + ("America/Swift_Current", "America/Swift_Current"), + ("America/Tegucigalpa", "America/Tegucigalpa"), + ("America/Thule", "America/Thule"), + ("America/Thunder_Bay", "America/Thunder_Bay"), + ("America/Tijuana", "America/Tijuana"), + ("America/Toronto", "America/Toronto"), + ("America/Tortola", "America/Tortola"), + ("America/Vancouver", "America/Vancouver"), + ("America/Virgin", "America/Virgin"), + ("America/Whitehorse", "America/Whitehorse"), + ("America/Winnipeg", "America/Winnipeg"), + ("America/Yakutat", "America/Yakutat"), + ("America/Yellowknife", "America/Yellowknife"), + ("Antarctica/Casey", "Antarctica/Casey"), + ("Antarctica/Davis", "Antarctica/Davis"), + ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"), + ("Antarctica/Macquarie", "Antarctica/Macquarie"), + ("Antarctica/Mawson", "Antarctica/Mawson"), + ("Antarctica/McMurdo", "Antarctica/McMurdo"), + ("Antarctica/Palmer", "Antarctica/Palmer"), + ("Antarctica/Rothera", "Antarctica/Rothera"), + ("Antarctica/South_Pole", "Antarctica/South_Pole"), + ("Antarctica/Syowa", "Antarctica/Syowa"), + ("Antarctica/Troll", "Antarctica/Troll"), + ("Antarctica/Vostok", "Antarctica/Vostok"), + ("Arctic/Longyearbyen", "Arctic/Longyearbyen"), + ("Asia/Aden", "Asia/Aden"), + ("Asia/Almaty", "Asia/Almaty"), + ("Asia/Amman", "Asia/Amman"), + ("Asia/Anadyr", "Asia/Anadyr"), + ("Asia/Aqtau", "Asia/Aqtau"), + ("Asia/Aqtobe", "Asia/Aqtobe"), + ("Asia/Ashgabat", "Asia/Ashgabat"), + ("Asia/Ashkhabad", "Asia/Ashkhabad"), + ("Asia/Atyrau", "Asia/Atyrau"), + ("Asia/Baghdad", "Asia/Baghdad"), + ("Asia/Bahrain", "Asia/Bahrain"), + ("Asia/Baku", "Asia/Baku"), + ("Asia/Bangkok", "Asia/Bangkok"), + ("Asia/Barnaul", "Asia/Barnaul"), + ("Asia/Beirut", "Asia/Beirut"), + ("Asia/Bishkek", "Asia/Bishkek"), + ("Asia/Brunei", "Asia/Brunei"), + ("Asia/Calcutta", "Asia/Calcutta"), + ("Asia/Chita", "Asia/Chita"), + ("Asia/Choibalsan", "Asia/Choibalsan"), + ("Asia/Chongqing", "Asia/Chongqing"), + ("Asia/Chungking", "Asia/Chungking"), + ("Asia/Colombo", "Asia/Colombo"), + ("Asia/Dacca", "Asia/Dacca"), + ("Asia/Damascus", "Asia/Damascus"), + ("Asia/Dhaka", "Asia/Dhaka"), + ("Asia/Dili", "Asia/Dili"), + ("Asia/Dubai", "Asia/Dubai"), + ("Asia/Dushanbe", "Asia/Dushanbe"), + ("Asia/Famagusta", "Asia/Famagusta"), + ("Asia/Gaza", "Asia/Gaza"), + ("Asia/Harbin", "Asia/Harbin"), + ("Asia/Hebron", "Asia/Hebron"), + ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"), + ("Asia/Hong_Kong", "Asia/Hong_Kong"), + ("Asia/Hovd", "Asia/Hovd"), + ("Asia/Irkutsk", "Asia/Irkutsk"), + ("Asia/Istanbul", "Asia/Istanbul"), + ("Asia/Jakarta", "Asia/Jakarta"), + ("Asia/Jayapura", "Asia/Jayapura"), + ("Asia/Jerusalem", "Asia/Jerusalem"), + ("Asia/Kabul", "Asia/Kabul"), + ("Asia/Kamchatka", "Asia/Kamchatka"), + ("Asia/Karachi", "Asia/Karachi"), + ("Asia/Kashgar", "Asia/Kashgar"), + ("Asia/Kathmandu", "Asia/Kathmandu"), + ("Asia/Katmandu", "Asia/Katmandu"), + ("Asia/Khandyga", "Asia/Khandyga"), + ("Asia/Kolkata", "Asia/Kolkata"), + ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"), + ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"), + ("Asia/Kuching", "Asia/Kuching"), + ("Asia/Kuwait", "Asia/Kuwait"), + ("Asia/Macao", "Asia/Macao"), + ("Asia/Macau", "Asia/Macau"), + ("Asia/Magadan", "Asia/Magadan"), + ("Asia/Makassar", "Asia/Makassar"), + ("Asia/Manila", "Asia/Manila"), + ("Asia/Muscat", "Asia/Muscat"), + ("Asia/Nicosia", "Asia/Nicosia"), + ("Asia/Novokuznetsk", "Asia/Novokuznetsk"), + ("Asia/Novosibirsk", "Asia/Novosibirsk"), + ("Asia/Omsk", "Asia/Omsk"), + ("Asia/Oral", "Asia/Oral"), + ("Asia/Phnom_Penh", "Asia/Phnom_Penh"), + ("Asia/Pontianak", "Asia/Pontianak"), + ("Asia/Pyongyang", "Asia/Pyongyang"), + ("Asia/Qatar", "Asia/Qatar"), + ("Asia/Qostanay", "Asia/Qostanay"), + ("Asia/Qyzylorda", "Asia/Qyzylorda"), + ("Asia/Rangoon", "Asia/Rangoon"), + ("Asia/Riyadh", "Asia/Riyadh"), + ("Asia/Saigon", "Asia/Saigon"), + ("Asia/Sakhalin", "Asia/Sakhalin"), + ("Asia/Samarkand", "Asia/Samarkand"), + ("Asia/Seoul", "Asia/Seoul"), + ("Asia/Shanghai", "Asia/Shanghai"), + ("Asia/Singapore", "Asia/Singapore"), + ("Asia/Srednekolymsk", "Asia/Srednekolymsk"), + ("Asia/Taipei", "Asia/Taipei"), + ("Asia/Tashkent", "Asia/Tashkent"), + ("Asia/Tbilisi", "Asia/Tbilisi"), + ("Asia/Tehran", "Asia/Tehran"), + ("Asia/Tel_Aviv", "Asia/Tel_Aviv"), + ("Asia/Thimbu", "Asia/Thimbu"), + ("Asia/Thimphu", "Asia/Thimphu"), + ("Asia/Tokyo", "Asia/Tokyo"), + ("Asia/Tomsk", "Asia/Tomsk"), + ("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"), + ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"), + ("Asia/Ulan_Bator", "Asia/Ulan_Bator"), + ("Asia/Urumqi", "Asia/Urumqi"), + ("Asia/Ust-Nera", "Asia/Ust-Nera"), + ("Asia/Vientiane", "Asia/Vientiane"), + ("Asia/Vladivostok", "Asia/Vladivostok"), + ("Asia/Yakutsk", "Asia/Yakutsk"), + ("Asia/Yangon", "Asia/Yangon"), + ("Asia/Yekaterinburg", "Asia/Yekaterinburg"), + ("Asia/Yerevan", "Asia/Yerevan"), + ("Atlantic/Azores", "Atlantic/Azores"), + ("Atlantic/Bermuda", "Atlantic/Bermuda"), + ("Atlantic/Canary", "Atlantic/Canary"), + ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"), + ("Atlantic/Faeroe", "Atlantic/Faeroe"), + ("Atlantic/Faroe", "Atlantic/Faroe"), + ("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"), + ("Atlantic/Madeira", "Atlantic/Madeira"), + ("Atlantic/Reykjavik", "Atlantic/Reykjavik"), + ("Atlantic/South_Georgia", "Atlantic/South_Georgia"), + ("Atlantic/St_Helena", "Atlantic/St_Helena"), + ("Atlantic/Stanley", "Atlantic/Stanley"), + ("Australia/ACT", "Australia/ACT"), + ("Australia/Adelaide", "Australia/Adelaide"), + ("Australia/Brisbane", "Australia/Brisbane"), + ("Australia/Broken_Hill", "Australia/Broken_Hill"), + ("Australia/Canberra", "Australia/Canberra"), + ("Australia/Currie", "Australia/Currie"), + ("Australia/Darwin", "Australia/Darwin"), + ("Australia/Eucla", "Australia/Eucla"), + ("Australia/Hobart", "Australia/Hobart"), + ("Australia/LHI", "Australia/LHI"), + ("Australia/Lindeman", "Australia/Lindeman"), + ("Australia/Lord_Howe", "Australia/Lord_Howe"), + ("Australia/Melbourne", "Australia/Melbourne"), + ("Australia/NSW", "Australia/NSW"), + ("Australia/North", "Australia/North"), + ("Australia/Perth", "Australia/Perth"), + ("Australia/Queensland", "Australia/Queensland"), + ("Australia/South", "Australia/South"), + ("Australia/Sydney", "Australia/Sydney"), + ("Australia/Tasmania", "Australia/Tasmania"), + ("Australia/Victoria", "Australia/Victoria"), + ("Australia/West", "Australia/West"), + ("Australia/Yancowinna", "Australia/Yancowinna"), + ("Brazil/Acre", "Brazil/Acre"), + ("Brazil/DeNoronha", "Brazil/DeNoronha"), + ("Brazil/East", "Brazil/East"), + ("Brazil/West", "Brazil/West"), + ("CET", "CET"), + ("CST6CDT", "CST6CDT"), + ("Canada/Atlantic", "Canada/Atlantic"), + ("Canada/Central", "Canada/Central"), + ("Canada/Eastern", "Canada/Eastern"), + ("Canada/Mountain", "Canada/Mountain"), + ("Canada/Newfoundland", "Canada/Newfoundland"), + ("Canada/Pacific", "Canada/Pacific"), + ("Canada/Saskatchewan", "Canada/Saskatchewan"), + ("Canada/Yukon", "Canada/Yukon"), + ("Chile/Continental", "Chile/Continental"), + ("Chile/EasterIsland", "Chile/EasterIsland"), + ("Cuba", "Cuba"), + ("EET", "EET"), + ("EST", "EST"), + ("EST5EDT", "EST5EDT"), + ("Egypt", "Egypt"), + ("Eire", "Eire"), + ("Etc/GMT", "Etc/GMT"), + ("Etc/GMT+0", "Etc/GMT+0"), + ("Etc/GMT+1", "Etc/GMT+1"), + ("Etc/GMT+10", "Etc/GMT+10"), + ("Etc/GMT+11", "Etc/GMT+11"), + ("Etc/GMT+12", "Etc/GMT+12"), + ("Etc/GMT+2", "Etc/GMT+2"), + ("Etc/GMT+3", "Etc/GMT+3"), + ("Etc/GMT+4", "Etc/GMT+4"), + ("Etc/GMT+5", "Etc/GMT+5"), + ("Etc/GMT+6", "Etc/GMT+6"), + ("Etc/GMT+7", "Etc/GMT+7"), + ("Etc/GMT+8", "Etc/GMT+8"), + ("Etc/GMT+9", "Etc/GMT+9"), + ("Etc/GMT-0", "Etc/GMT-0"), + ("Etc/GMT-1", "Etc/GMT-1"), + ("Etc/GMT-10", "Etc/GMT-10"), + ("Etc/GMT-11", "Etc/GMT-11"), + ("Etc/GMT-12", "Etc/GMT-12"), + ("Etc/GMT-13", "Etc/GMT-13"), + ("Etc/GMT-14", "Etc/GMT-14"), + ("Etc/GMT-2", "Etc/GMT-2"), + ("Etc/GMT-3", "Etc/GMT-3"), + ("Etc/GMT-4", "Etc/GMT-4"), + ("Etc/GMT-5", "Etc/GMT-5"), + ("Etc/GMT-6", "Etc/GMT-6"), + ("Etc/GMT-7", "Etc/GMT-7"), + ("Etc/GMT-8", "Etc/GMT-8"), + ("Etc/GMT-9", "Etc/GMT-9"), + ("Etc/GMT0", "Etc/GMT0"), + ("Etc/Greenwich", "Etc/Greenwich"), + ("Etc/UCT", "Etc/UCT"), + ("Etc/UTC", "Etc/UTC"), + ("Etc/Universal", "Etc/Universal"), + ("Etc/Zulu", "Etc/Zulu"), + ("Europe/Amsterdam", "Europe/Amsterdam"), + ("Europe/Andorra", "Europe/Andorra"), + ("Europe/Astrakhan", "Europe/Astrakhan"), + ("Europe/Athens", "Europe/Athens"), + ("Europe/Belfast", "Europe/Belfast"), + ("Europe/Belgrade", "Europe/Belgrade"), + ("Europe/Berlin", "Europe/Berlin"), + ("Europe/Bratislava", "Europe/Bratislava"), + ("Europe/Brussels", "Europe/Brussels"), + ("Europe/Bucharest", "Europe/Bucharest"), + ("Europe/Budapest", "Europe/Budapest"), + ("Europe/Busingen", "Europe/Busingen"), + ("Europe/Chisinau", "Europe/Chisinau"), + ("Europe/Copenhagen", "Europe/Copenhagen"), + ("Europe/Dublin", "Europe/Dublin"), + ("Europe/Gibraltar", "Europe/Gibraltar"), + ("Europe/Guernsey", "Europe/Guernsey"), + ("Europe/Helsinki", "Europe/Helsinki"), + ("Europe/Isle_of_Man", "Europe/Isle_of_Man"), + ("Europe/Istanbul", "Europe/Istanbul"), + ("Europe/Jersey", "Europe/Jersey"), + ("Europe/Kaliningrad", "Europe/Kaliningrad"), + ("Europe/Kiev", "Europe/Kiev"), + ("Europe/Kirov", "Europe/Kirov"), + ("Europe/Kyiv", "Europe/Kyiv"), + ("Europe/Lisbon", "Europe/Lisbon"), + ("Europe/Ljubljana", "Europe/Ljubljana"), + ("Europe/London", "Europe/London"), + ("Europe/Luxembourg", "Europe/Luxembourg"), + ("Europe/Madrid", "Europe/Madrid"), + ("Europe/Malta", "Europe/Malta"), + ("Europe/Mariehamn", "Europe/Mariehamn"), + ("Europe/Minsk", "Europe/Minsk"), + ("Europe/Monaco", "Europe/Monaco"), + ("Europe/Moscow", "Europe/Moscow"), + ("Europe/Nicosia", "Europe/Nicosia"), + ("Europe/Oslo", "Europe/Oslo"), + ("Europe/Paris", "Europe/Paris"), + ("Europe/Podgorica", "Europe/Podgorica"), + ("Europe/Prague", "Europe/Prague"), + ("Europe/Riga", "Europe/Riga"), + ("Europe/Rome", "Europe/Rome"), + ("Europe/Samara", "Europe/Samara"), + ("Europe/San_Marino", "Europe/San_Marino"), + ("Europe/Sarajevo", "Europe/Sarajevo"), + ("Europe/Saratov", "Europe/Saratov"), + ("Europe/Simferopol", "Europe/Simferopol"), + ("Europe/Skopje", "Europe/Skopje"), + ("Europe/Sofia", "Europe/Sofia"), + ("Europe/Stockholm", "Europe/Stockholm"), + ("Europe/Tallinn", "Europe/Tallinn"), + ("Europe/Tirane", "Europe/Tirane"), + ("Europe/Tiraspol", "Europe/Tiraspol"), + ("Europe/Ulyanovsk", "Europe/Ulyanovsk"), + ("Europe/Uzhgorod", "Europe/Uzhgorod"), + ("Europe/Vaduz", "Europe/Vaduz"), + ("Europe/Vatican", "Europe/Vatican"), + ("Europe/Vienna", "Europe/Vienna"), + ("Europe/Vilnius", "Europe/Vilnius"), + ("Europe/Volgograd", "Europe/Volgograd"), + ("Europe/Warsaw", "Europe/Warsaw"), + ("Europe/Zagreb", "Europe/Zagreb"), + ("Europe/Zaporozhye", "Europe/Zaporozhye"), + ("Europe/Zurich", "Europe/Zurich"), + ("Factory", "Factory"), + ("GB", "GB"), + ("GB-Eire", "GB-Eire"), + ("GMT", "GMT"), + ("GMT+0", "GMT+0"), + ("GMT-0", "GMT-0"), + ("GMT0", "GMT0"), + ("Greenwich", "Greenwich"), + ("HST", "HST"), + ("Hongkong", "Hongkong"), + ("Iceland", "Iceland"), + ("Indian/Antananarivo", "Indian/Antananarivo"), + ("Indian/Chagos", "Indian/Chagos"), + ("Indian/Christmas", "Indian/Christmas"), + ("Indian/Cocos", "Indian/Cocos"), + ("Indian/Comoro", "Indian/Comoro"), + ("Indian/Kerguelen", "Indian/Kerguelen"), + ("Indian/Mahe", "Indian/Mahe"), + ("Indian/Maldives", "Indian/Maldives"), + ("Indian/Mauritius", "Indian/Mauritius"), + ("Indian/Mayotte", "Indian/Mayotte"), + ("Indian/Reunion", "Indian/Reunion"), + ("Iran", "Iran"), + ("Israel", "Israel"), + ("Jamaica", "Jamaica"), + ("Japan", "Japan"), + ("Kwajalein", "Kwajalein"), + ("Libya", "Libya"), + ("MET", "MET"), + ("MST", "MST"), + ("MST7MDT", "MST7MDT"), + ("Mexico/BajaNorte", "Mexico/BajaNorte"), + ("Mexico/BajaSur", "Mexico/BajaSur"), + ("Mexico/General", "Mexico/General"), + ("NZ", "NZ"), + ("NZ-CHAT", "NZ-CHAT"), + ("Navajo", "Navajo"), + ("PRC", "PRC"), + ("PST8PDT", "PST8PDT"), + ("Pacific/Apia", "Pacific/Apia"), + ("Pacific/Auckland", "Pacific/Auckland"), + ("Pacific/Bougainville", "Pacific/Bougainville"), + ("Pacific/Chatham", "Pacific/Chatham"), + ("Pacific/Chuuk", "Pacific/Chuuk"), + ("Pacific/Easter", "Pacific/Easter"), + ("Pacific/Efate", "Pacific/Efate"), + ("Pacific/Enderbury", "Pacific/Enderbury"), + ("Pacific/Fakaofo", "Pacific/Fakaofo"), + ("Pacific/Fiji", "Pacific/Fiji"), + ("Pacific/Funafuti", "Pacific/Funafuti"), + ("Pacific/Galapagos", "Pacific/Galapagos"), + ("Pacific/Gambier", "Pacific/Gambier"), + ("Pacific/Guadalcanal", "Pacific/Guadalcanal"), + ("Pacific/Guam", "Pacific/Guam"), + ("Pacific/Honolulu", "Pacific/Honolulu"), + ("Pacific/Johnston", "Pacific/Johnston"), + ("Pacific/Kanton", "Pacific/Kanton"), + ("Pacific/Kiritimati", "Pacific/Kiritimati"), + ("Pacific/Kosrae", "Pacific/Kosrae"), + ("Pacific/Kwajalein", "Pacific/Kwajalein"), + ("Pacific/Majuro", "Pacific/Majuro"), + ("Pacific/Marquesas", "Pacific/Marquesas"), + ("Pacific/Midway", "Pacific/Midway"), + ("Pacific/Nauru", "Pacific/Nauru"), + ("Pacific/Niue", "Pacific/Niue"), + ("Pacific/Norfolk", "Pacific/Norfolk"), + ("Pacific/Noumea", "Pacific/Noumea"), + ("Pacific/Pago_Pago", "Pacific/Pago_Pago"), + ("Pacific/Palau", "Pacific/Palau"), + ("Pacific/Pitcairn", "Pacific/Pitcairn"), + ("Pacific/Pohnpei", "Pacific/Pohnpei"), + ("Pacific/Ponape", "Pacific/Ponape"), + ("Pacific/Port_Moresby", "Pacific/Port_Moresby"), + ("Pacific/Rarotonga", "Pacific/Rarotonga"), + ("Pacific/Saipan", "Pacific/Saipan"), + ("Pacific/Samoa", "Pacific/Samoa"), + ("Pacific/Tahiti", "Pacific/Tahiti"), + ("Pacific/Tarawa", "Pacific/Tarawa"), + ("Pacific/Tongatapu", "Pacific/Tongatapu"), + ("Pacific/Truk", "Pacific/Truk"), + ("Pacific/Wake", "Pacific/Wake"), + ("Pacific/Wallis", "Pacific/Wallis"), + ("Pacific/Yap", "Pacific/Yap"), + ("Poland", "Poland"), + ("Portugal", "Portugal"), + ("ROC", "ROC"), + ("ROK", "ROK"), + ("Singapore", "Singapore"), + ("Turkey", "Turkey"), + ("UCT", "UCT"), + ("US/Alaska", "US/Alaska"), + ("US/Aleutian", "US/Aleutian"), + ("US/Arizona", "US/Arizona"), + ("US/Central", "US/Central"), + ("US/East-Indiana", "US/East-Indiana"), + ("US/Eastern", "US/Eastern"), + ("US/Hawaii", "US/Hawaii"), + ("US/Indiana-Starke", "US/Indiana-Starke"), + ("US/Michigan", "US/Michigan"), + ("US/Mountain", "US/Mountain"), + ("US/Pacific", "US/Pacific"), + ("US/Samoa", "US/Samoa"), + ("UTC", "UTC"), + ("Universal", "Universal"), + ("W-SU", "W-SU"), + ("WET", "WET"), + ("Zulu", "Zulu"), + ("localtime", "localtime"), + ], + default="UTC", + max_length=255, + ), + ), + ] diff --git a/bookwyrm/migrations/0200_auto_20240327_1914.py b/bookwyrm/migrations/0200_auto_20240327_1914.py new file mode 100644 index 000000000..38180b3f9 --- /dev/null +++ b/bookwyrm/migrations/0200_auto_20240327_1914.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.25 on 2024-03-27 19:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0199_merge_20240326_1217"), + ] + + operations = [ + migrations.RemoveField( + model_name="addfiletotar", + name="childjob_ptr", + ), + migrations.RemoveField( + model_name="addfiletotar", + name="parent_export_job", + ), + migrations.DeleteModel( + name="AddBookToUserExportJob", + ), + migrations.DeleteModel( + name="AddFileToTar", + ), + ] diff --git a/bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py b/bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py new file mode 100644 index 000000000..daca654c7 --- /dev/null +++ b/bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-04-03 19:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0199_status_bookwyrm_st_remote__06aeba_idx"), + ] + + operations = [ + migrations.AddIndex( + model_name="status", + index=models.Index( + fields=["thread_id"], name="bookwyrm_st_thread__cf064f_idx" + ), + ), + ] diff --git a/bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py b/bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py new file mode 100644 index 000000000..4fe41ec35 --- /dev/null +++ b/bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.11 on 2024-04-01 21:09 + +import bookwyrm.models.fields +from django.db import migrations, models +from django.contrib.postgres.operations import CreateCollation + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0200_alter_user_preferred_timezone"), + ] + + operations = [ + CreateCollation( + "case_insensitive", + provider="icu", + locale="und-u-ks-level2", + deterministic=False, + ), + migrations.AlterField( + model_name="hashtag", + name="name", + field=bookwyrm.models.fields.CharField( + db_collation="case_insensitive", max_length=256 + ), + ), + migrations.AlterField( + model_name="user", + name="localname", + field=models.CharField( + db_collation="case_insensitive", + max_length=255, + null=True, + unique=True, + validators=[bookwyrm.models.fields.validate_localname], + ), + ), + ] diff --git a/bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py b/bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py new file mode 100644 index 000000000..e3d27a11b --- /dev/null +++ b/bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-04-03 19:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0200_status_bookwyrm_st_thread__cf064f_idx"), + ] + + operations = [ + migrations.AddIndex( + model_name="keypair", + index=models.Index( + fields=["remote_id"], name="bookwyrm_ke_remote__472927_idx" + ), + ), + ] diff --git a/bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py b/bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py new file mode 100644 index 000000000..d8666fe3f --- /dev/null +++ b/bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-04-03 19:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0201_keypair_bookwyrm_ke_remote__472927_idx"), + ] + + operations = [ + migrations.AddIndex( + model_name="user", + index=models.Index( + fields=["username"], name="bookwyrm_us_usernam_b2546d_idx" + ), + ), + ] diff --git a/bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py b/bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py new file mode 100644 index 000000000..b07f1c8a9 --- /dev/null +++ b/bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-04-03 19:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0202_user_bookwyrm_us_usernam_b2546d_idx"), + ] + + operations = [ + migrations.AddIndex( + model_name="user", + index=models.Index( + fields=["is_active", "local"], name="bookwyrm_us_is_acti_972dc4_idx" + ), + ), + ] diff --git a/bookwyrm/migrations/0204_merge_20240409_1042.py b/bookwyrm/migrations/0204_merge_20240409_1042.py new file mode 100644 index 000000000..5656ac586 --- /dev/null +++ b/bookwyrm/migrations/0204_merge_20240409_1042.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.25 on 2024-04-09 10:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0197_mergedauthor_mergedbook"), + ("bookwyrm", "0203_user_bookwyrm_us_is_acti_972dc4_idx"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0205_merge_20240410_2022.py b/bookwyrm/migrations/0205_merge_20240410_2022.py new file mode 100644 index 000000000..294f48487 --- /dev/null +++ b/bookwyrm/migrations/0205_merge_20240410_2022.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.11 on 2024-04-10 20:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0201_alter_hashtag_name_alter_user_localname"), + ("bookwyrm", "0204_merge_20240409_1042"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0205_merge_20240413_0232.py b/bookwyrm/migrations/0205_merge_20240413_0232.py new file mode 100644 index 000000000..9cca29c45 --- /dev/null +++ b/bookwyrm/migrations/0205_merge_20240413_0232.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.25 on 2024-04-13 02:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0200_auto_20240327_1914"), + ("bookwyrm", "0204_merge_20240409_1042"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0206_merge_20240415_1537.py b/bookwyrm/migrations/0206_merge_20240415_1537.py new file mode 100644 index 000000000..454e69880 --- /dev/null +++ b/bookwyrm/migrations/0206_merge_20240415_1537.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.11 on 2024-04-15 15:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0205_merge_20240410_2022"), + ("bookwyrm", "0205_merge_20240413_0232"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0207_merge_20240629_0626.py b/bookwyrm/migrations/0207_merge_20240629_0626.py new file mode 100644 index 000000000..b5a1a4556 --- /dev/null +++ b/bookwyrm/migrations/0207_merge_20240629_0626.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.11 on 2024-06-29 06:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0189_importjob_create_shelves"), + ("bookwyrm", "0206_merge_20240415_1537"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0207_sqlparse_update.py b/bookwyrm/migrations/0207_sqlparse_update.py new file mode 100644 index 000000000..95c46eba2 --- /dev/null +++ b/bookwyrm/migrations/0207_sqlparse_update.py @@ -0,0 +1,51 @@ +# Generated by Django 4.2.11 on 2024-07-27 18:18 + +from django.db import migrations, models +import pgtrigger.compiler +import pgtrigger.migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0206_merge_20240415_1537"), + ] + + operations = [ + pgtrigger.migrations.RemoveTrigger( + model_name="author", + name="reset_book_search_vector_on_author_edit", + ), + pgtrigger.migrations.RemoveTrigger( + model_name="book", + name="update_search_vector_on_book_edit", + ), + pgtrigger.migrations.AddTrigger( + model_name="author", + trigger=pgtrigger.compiler.Trigger( + name="reset_book_search_vector_on_author_edit", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;", + hash="4eeb17d1c9c53f543615bcae1234bd0260adefcc", + operation='UPDATE OF "name", "aliases"', + pgid="pgtrigger_reset_book_search_vector_on_author_edit_a50c7", + table="bookwyrm_author", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="book", + trigger=pgtrigger.compiler.Trigger( + name="update_search_vector_on_book_edit", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="WITH author_names AS (SELECT array_to_string(bookwyrm_author.name || bookwyrm_author.aliases, ' ') AS name_and_aliases FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id) SELECT setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(name_and_aliases), ' '), '')), 'C') FROM author_names) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D') INTO new.search_vector;RETURN NEW;", + hash="676d929ce95beff671544b6add09cf9360b6f299", + operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"', + pgid="pgtrigger_update_search_vector_on_book_edit_bec58", + table="bookwyrm_book", + when="BEFORE", + ), + ), + ), + ] diff --git a/bookwyrm/migrations/0208_merge_0207_merge_20240629_0626_0207_sqlparse_update.py b/bookwyrm/migrations/0208_merge_0207_merge_20240629_0626_0207_sqlparse_update.py new file mode 100644 index 000000000..24ef28e04 --- /dev/null +++ b/bookwyrm/migrations/0208_merge_0207_merge_20240629_0626_0207_sqlparse_update.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.11 on 2024-07-28 11:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0207_merge_20240629_0626"), + ("bookwyrm", "0207_sqlparse_update"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0209_user_show_ratings.py b/bookwyrm/migrations/0209_user_show_ratings.py new file mode 100644 index 000000000..94e074407 --- /dev/null +++ b/bookwyrm/migrations/0209_user_show_ratings.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.15 on 2024-08-24 01:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0208_merge_0207_merge_20240629_0626_0207_sqlparse_update"), + ] + + operations = [ + migrations.AddField( + model_name="user", + name="show_ratings", + field=models.BooleanField(default=True), + ), + ] diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index db737b8bc..54ad11511 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) PropertyField = namedtuple("PropertyField", ("set_activity_from_field")) -# pylint: disable=invalid-name + def set_activity_from_property_field(activity, obj, field): """assign a model property value to the activity json""" activity[field[1]] = getattr(obj, field[0]) @@ -169,7 +169,7 @@ class ActivitypubMixin: # filter users first by whether they're using the desired software # this lets us send book updates only to other bw servers if software: - queryset = queryset.filter(bookwyrm_user=(software == "bookwyrm")) + queryset = queryset.filter(bookwyrm_user=software == "bookwyrm") # if there's a user, we only want to send to the user's followers if user: queryset = queryset.filter(following=user) @@ -206,14 +206,10 @@ class ObjectMixin(ActivitypubMixin): created: Optional[bool] = None, software: Any = None, priority: str = BROADCAST, + broadcast: bool = True, **kwargs: Any, ) -> None: """broadcast created/updated/deleted objects as appropriate""" - broadcast = kwargs.get("broadcast", True) - # this bonus kwarg would cause an error in the base save method - if "broadcast" in kwargs: - del kwargs["broadcast"] - created = created or not bool(self.id) # first off, we want to save normally no matter what super().save(*args, **kwargs) diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 9dc3962ad..20c4e9e00 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -1,22 +1,25 @@ """ database schema for info about authors """ + import re -from typing import Tuple, Any +from typing import Any from django.db import models from django.contrib.postgres.indexes import GinIndex import pgtrigger from bookwyrm import activitypub -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import BASE_URL from bookwyrm.utils.db import format_trigger -from .book import BookDataModel +from .book import BookDataModel, MergedAuthor from . import fields class Author(BookDataModel): """basic biographic info""" + merged_model = MergedAuthor + wikipedia_link = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) @@ -42,12 +45,12 @@ class Author(BookDataModel): ) bio = fields.HtmlField(null=True, blank=True) - def save(self, *args: Tuple[Any, ...], **kwargs: dict[str, Any]) -> None: + def save(self, *args: Any, **kwargs: Any) -> None: """normalize isni format""" - if self.isni: + if self.isni is not None: self.isni = re.sub(r"\s", "", self.isni) - return super().save(*args, **kwargs) + super().save(*args, **kwargs) @property def isni_link(self): @@ -67,7 +70,7 @@ class Author(BookDataModel): def get_remote_id(self): """editions and works both use "book" instead of model_name""" - return f"https://{DOMAIN}/author/{self.id}" + return f"{BASE_URL}/author/{self.id}" class Meta: """sets up indexes and triggers""" diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 2d39e2a6f..ca13d9553 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -10,7 +10,7 @@ from django.http import Http404 from django.utils.translation import gettext_lazy as _ from django.utils.text import slugify -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import BASE_URL from .fields import RemoteIdField @@ -38,7 +38,7 @@ class BookWyrmModel(models.Model): def get_remote_id(self): """generate the url that resolves to the local object, without a slug""" - base_path = f"https://{DOMAIN}" + base_path = BASE_URL if hasattr(self, "user"): base_path = f"{base_path}{self.user.local_path}" @@ -53,7 +53,7 @@ class BookWyrmModel(models.Model): @property def local_path(self): """how to link to this object in the local app, with a slug""" - local = self.get_remote_id().replace(f"https://{DOMAIN}", "") + local = self.get_remote_id().replace(BASE_URL, "") name = None if hasattr(self, "name_field"): diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 5dba6532f..368276523 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -1,13 +1,15 @@ """ database schema for books and shelves """ + from itertools import chain import re -from typing import Any +from typing import Any, Dict, Optional, Iterable +from typing_extensions import Self from django.contrib.postgres.search import SearchVectorField from django.contrib.postgres.indexes import GinIndex from django.core.cache import cache from django.db import models, transaction -from django.db.models import Prefetch +from django.db.models import Prefetch, ManyToManyField from django.dispatch import receiver from django.utils.translation import gettext_lazy as _ from model_utils import FieldTracker @@ -19,13 +21,13 @@ from bookwyrm import activitypub from bookwyrm.isbn.isbn import hyphenator_singleton as hyphenator from bookwyrm.preview_images import generate_edition_preview_image_task from bookwyrm.settings import ( - DOMAIN, + BASE_URL, DEFAULT_LANGUAGE, LANGUAGE_ARTICLES, ENABLE_PREVIEW_IMAGES, ENABLE_THUMBNAIL_GENERATION, ) -from bookwyrm.utils.db import format_trigger +from bookwyrm.utils.db import format_trigger, add_update_fields from .activitypub_mixin import OrderedCollectionPageMixin, ObjectMixin from .base_model import BookWyrmModel @@ -94,24 +96,133 @@ class BookDataModel(ObjectMixin, BookWyrmModel): abstract = True - def save(self, *args: Any, **kwargs: Any) -> None: + def save( + self, *args: Any, update_fields: Optional[Iterable[str]] = None, **kwargs: Any + ) -> None: """ensure that the remote_id is within this instance""" if self.id: self.remote_id = self.get_remote_id() + update_fields = add_update_fields(update_fields, "remote_id") else: self.origin_id = self.remote_id self.remote_id = None - return super().save(*args, **kwargs) + update_fields = add_update_fields(update_fields, "origin_id", "remote_id") + + super().save(*args, update_fields=update_fields, **kwargs) # pylint: disable=arguments-differ def broadcast(self, activity, sender, software="bookwyrm", **kwargs): """only send book data updates to other bookwyrm instances""" super().broadcast(activity, sender, software=software, **kwargs) + def merge_into(self, canonical: Self, dry_run=False) -> Dict[str, Any]: + """merge this entity into another entity""" + if canonical.id == self.id: + raise ValueError(f"Cannot merge {self} into itself") + + absorbed_fields = canonical.absorb_data_from(self, dry_run=dry_run) + + if dry_run: + return absorbed_fields + + canonical.save() + + self.merged_model.objects.create(deleted_id=self.id, merged_into=canonical) + + # move related models to canonical + related_models = [ + (r.remote_field.name, r.related_model) for r in self._meta.related_objects + ] + for related_field, related_model in related_models: + # Skip the ManyToMany fields that aren’t auto-created. These + # should have a corresponding OneToMany field in the model for + # the linking table anyway. If we update it through that model + # instead then we won’t lose the extra fields in the linking + # table. + # pylint: disable=protected-access + related_field_obj = related_model._meta.get_field(related_field) + if isinstance(related_field_obj, ManyToManyField): + through = related_field_obj.remote_field.through + if not through._meta.auto_created: + continue + related_objs = related_model.objects.filter(**{related_field: self}) + for related_obj in related_objs: + try: + setattr(related_obj, related_field, canonical) + related_obj.save() + except TypeError: + getattr(related_obj, related_field).add(canonical) + getattr(related_obj, related_field).remove(self) + + self.delete() + return absorbed_fields + + def absorb_data_from(self, other: Self, dry_run=False) -> Dict[str, Any]: + """fill empty fields with values from another entity""" + absorbed_fields = {} + for data_field in self._meta.get_fields(): + if not hasattr(data_field, "activitypub_field"): + continue + canonical_value = getattr(self, data_field.name) + other_value = getattr(other, data_field.name) + if not other_value: + continue + if isinstance(data_field, fields.ArrayField): + if new_values := list(set(other_value) - set(canonical_value)): + # append at the end (in no particular order) + if not dry_run: + setattr(self, data_field.name, canonical_value + new_values) + absorbed_fields[data_field.name] = new_values + elif isinstance(data_field, fields.PartialDateField): + if ( + (not canonical_value) + or (other_value.has_day and not canonical_value.has_day) + or (other_value.has_month and not canonical_value.has_month) + ): + if not dry_run: + setattr(self, data_field.name, other_value) + absorbed_fields[data_field.name] = other_value + else: + if not canonical_value: + if not dry_run: + setattr(self, data_field.name, other_value) + absorbed_fields[data_field.name] = other_value + return absorbed_fields + + +class MergedBookDataModel(models.Model): + """a BookDataModel instance that has been merged into another instance. kept + to be able to redirect old URLs""" + + deleted_id = models.IntegerField(primary_key=True) + + class Meta: + """abstract just like BookDataModel""" + + abstract = True + + +class MergedBook(MergedBookDataModel): + """an Book that has been merged into another one""" + + merged_into = models.ForeignKey( + "Book", on_delete=models.PROTECT, related_name="absorbed" + ) + + +class MergedAuthor(MergedBookDataModel): + """an Author that has been merged into another one""" + + merged_into = models.ForeignKey( + "Author", on_delete=models.PROTECT, related_name="absorbed" + ) + class Book(BookDataModel): """a generic book, which can mean either an edition or a work""" + merged_model = MergedBook + connector = models.ForeignKey("Connector", on_delete=models.PROTECT, null=True) # book/work metadata @@ -192,9 +303,13 @@ class Book(BookDataModel): """properties of this edition, as a string""" items = [ self.physical_format if hasattr(self, "physical_format") else None, - f"{self.languages[0]} language" - if self.languages and self.languages[0] and self.languages[0] != "English" - else None, + ( + f"{self.languages[0]} language" + if self.languages + and self.languages[0] + and self.languages[0] != "English" + else None + ), str(self.published_date.year) if self.published_date else None, ", ".join(self.publishers) if hasattr(self, "publishers") else None, ] @@ -212,11 +327,11 @@ class Book(BookDataModel): if not isinstance(self, (Edition, Work)): raise ValueError("Books should be added as Editions or Works") - return super().save(*args, **kwargs) + super().save(*args, **kwargs) def get_remote_id(self): """editions and works both use "book" instead of model_name""" - return f"https://{DOMAIN}/book/{self.id}" + return f"{BASE_URL}/book/{self.id}" def guess_sort_title(self): """Get a best-guess sort title for the current book""" @@ -289,10 +404,11 @@ class Work(OrderedCollectionPageMixin, Book): def save(self, *args, **kwargs): """set some fields on the edition object""" + super().save(*args, **kwargs) + # set rank for edition in self.editions.all(): edition.save() - return super().save(*args, **kwargs) @property def default_edition(self): @@ -398,33 +514,48 @@ class Edition(Book): # max rank is 9 return rank - def save(self, *args: Any, **kwargs: Any) -> None: + def save( + self, *args: Any, update_fields: Optional[Iterable[str]] = None, **kwargs: Any + ) -> None: """set some fields on the edition object""" # calculate isbn 10/13 - if self.isbn_13 and self.isbn_13[:3] == "978" and not self.isbn_10: + if ( + self.isbn_10 is None + and self.isbn_13 is not None + and self.isbn_13[:3] == "978" + ): self.isbn_10 = isbn_13_to_10(self.isbn_13) - if self.isbn_10 and not self.isbn_13: + update_fields = add_update_fields(update_fields, "isbn_10") + if self.isbn_13 is None and self.isbn_10 is not None: self.isbn_13 = isbn_10_to_13(self.isbn_10) + update_fields = add_update_fields(update_fields, "isbn_13") # normalize isbn format - if self.isbn_10: + if self.isbn_10 is not None: self.isbn_10 = normalize_isbn(self.isbn_10) - if self.isbn_13: + if self.isbn_13 is not None: self.isbn_13 = normalize_isbn(self.isbn_13) # set rank - self.edition_rank = self.get_rank() - - # clear author cache - if self.id: - for author_id in self.authors.values_list("id", flat=True): - cache.delete(f"author-books-{author_id}") + if (new := self.get_rank()) != self.edition_rank: + self.edition_rank = new + update_fields = add_update_fields(update_fields, "edition_rank") # Create sort title by removing articles from title if self.sort_title in [None, ""]: self.sort_title = self.guess_sort_title() + update_fields = add_update_fields(update_fields, "sort_title") - return super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) + + # clear author cache + if self.id: + cache.delete_many( + [ + f"author-books-{author_id}" + for author_id in self.authors.values_list("id", flat=True) + ] + ) @transaction.atomic def repair(self): diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index 2f32cbd29..a42562f30 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -1,229 +1,341 @@ """Export user account to tar.gz file for import into another Bookwyrm instance""" -import dataclasses import logging -from uuid import uuid4 +import os -from django.db.models import FileField -from django.db.models import Q +from boto3.session import Session as BotoSession +from s3_tar import S3Tar + +from django.db.models import BooleanField, FileField, JSONField from django.core.serializers.json import DjangoJSONEncoder from django.core.files.base import ContentFile +from django.core.files.storage import storages -from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, List, ListItem +from bookwyrm import settings + +from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, ListItem from bookwyrm.models import Review, Comment, Quotation from bookwyrm.models import Edition from bookwyrm.models import UserFollows, User, UserBlocks -from bookwyrm.models.job import ParentJob, ParentTask +from bookwyrm.models.job import ParentJob from bookwyrm.tasks import app, IMPORTS from bookwyrm.utils.tar import BookwyrmTarFile logger = logging.getLogger(__name__) +class BookwyrmAwsSession(BotoSession): + """a boto session that always uses settings.AWS_S3_ENDPOINT_URL""" + + def client(self, *args, **kwargs): + kwargs["endpoint_url"] = settings.AWS_S3_ENDPOINT_URL + return super().client("s3", *args, **kwargs) + + +def select_exports_storage(): + """callable to allow for dependency on runtime configuration""" + return storages["exports"] + + class BookwyrmExportJob(ParentJob): """entry for a specific request to export a bookwyrm user""" - export_data = FileField(null=True) + export_data = FileField(null=True, storage=select_exports_storage) + export_json = JSONField(null=True, encoder=DjangoJSONEncoder) + json_completed = BooleanField(default=False) def start_job(self): - """Start the job""" - start_export_task.delay(job_id=self.id, no_children=True) + """schedule the first task""" - return self + task = create_export_json_task.delay(job_id=self.id) + self.task_id = task.id + self.save(update_fields=["task_id"]) -@app.task(queue=IMPORTS, base=ParentTask) -def start_export_task(**kwargs): - """trigger the child tasks for each row""" - job = BookwyrmExportJob.objects.get(id=kwargs["job_id"]) +@app.task(queue=IMPORTS) +def create_export_json_task(job_id): + """create the JSON data for the export""" + + job = BookwyrmExportJob.objects.get(id=job_id) # don't start the job if it was stopped from the UI if job.complete: return + try: - # This is where ChildJobs get made - job.export_data = ContentFile(b"", str(uuid4())) - json_data = json_export(job.user) - tar_export(json_data, job.user, job.export_data) - job.save(update_fields=["export_data"]) + job.set_status("active") + + # generate JSON structure + job.export_json = export_json(job.user) + job.save(update_fields=["export_json"]) + + # create archive in separate task + create_archive_task.delay(job_id=job.id) except Exception as err: # pylint: disable=broad-except - logger.exception("User Export Job %s Failed with error: %s", job.id, err) + logger.exception( + "create_export_json_task for %s failed with error: %s", job, err + ) job.set_status("failed") - job.set_status("complete") + +def archive_file_location(file, directory="") -> str: + """get the relative location of a file inside the archive""" + return os.path.join(directory, file.name) -def tar_export(json_data: str, user, file): - """wrap the export information in a tar file""" - file.open("wb") - with BookwyrmTarFile.open(mode="w:gz", fileobj=file) as tar: - tar.write_bytes(json_data.encode("utf-8")) +def add_file_to_s3_tar(s3_tar: S3Tar, storage, file, directory=""): + """ + add file to S3Tar inside directory, keeping any directories under its + storage location + """ + s3_tar.add_file( + os.path.join(storage.location, file.name), + folder=os.path.dirname(archive_file_location(file, directory=directory)), + ) - # Add avatar image if present - if getattr(user, "avatar", False): - tar.add_image(user.avatar, filename="avatar") +@app.task(queue=IMPORTS) +def create_archive_task(job_id): + """create the archive containing the JSON file and additional files""" + + job = BookwyrmExportJob.objects.get(id=job_id) + + # don't start the job if it was stopped from the UI + if job.complete: + return + + try: + export_task_id = str(job.task_id) + archive_filename = f"{export_task_id}.tar.gz" + export_json_bytes = DjangoJSONEncoder().encode(job.export_json).encode("utf-8") + + user = job.user editions = get_books_for_user(user) - for book in editions: - if getattr(book, "cover", False): - tar.add_image(book.cover) - file.close() + if settings.USE_S3: + # Storage for writing temporary files + exports_storage = storages["exports"] + + # Handle for creating the final archive + s3_tar = S3Tar( + exports_storage.bucket_name, + os.path.join(exports_storage.location, archive_filename), + session=BookwyrmAwsSession(), + ) + + # Save JSON file to a temporary location + export_json_tmp_file = os.path.join(export_task_id, "archive.json") + exports_storage.save( + export_json_tmp_file, + ContentFile(export_json_bytes), + ) + s3_tar.add_file( + os.path.join(exports_storage.location, export_json_tmp_file) + ) + + # Add images to TAR + images_storage = storages["default"] + + if user.avatar: + add_file_to_s3_tar(s3_tar, images_storage, user.avatar) + + for edition in editions: + if edition.cover: + add_file_to_s3_tar( + s3_tar, images_storage, edition.cover, directory="images" + ) + + # Create archive and store file name + s3_tar.tar() + job.export_data = archive_filename + job.save(update_fields=["export_data"]) + + # Delete temporary files + exports_storage.delete(export_json_tmp_file) + + else: + job.export_data = archive_filename + with job.export_data.open("wb") as tar_file: + with BookwyrmTarFile.open(mode="w:gz", fileobj=tar_file) as tar: + # save json file + tar.write_bytes(export_json_bytes) + + # Add avatar image if present + if user.avatar: + tar.add_image(user.avatar) + + for edition in editions: + if edition.cover: + tar.add_image(edition.cover, directory="images") + job.save(update_fields=["export_data"]) + + job.set_status("completed") + + except Exception as err: # pylint: disable=broad-except + logger.exception("create_archive_task for %s failed with error: %s", job, err) + job.set_status("failed") -def json_export( - user, -): # pylint: disable=too-many-locals, too-many-statements, too-many-branches - """Generate an export for a user""" +def export_json(user: User): + """create export JSON""" + data = export_user(user) # in the root of the JSON structure + data["settings"] = export_settings(user) + data["goals"] = export_goals(user) + data["books"] = export_books(user) + data["saved_lists"] = export_saved_lists(user) + data["follows"] = export_follows(user) + data["blocks"] = export_blocks(user) + return data - # User as AP object - exported_user = user.to_activity() - # I don't love this but it prevents a JSON encoding error - # when there is no user image - if exported_user.get("icon") in (None, dataclasses.MISSING): - exported_user["icon"] = {} + +def export_user(user: User): + """export user data""" + data = user.to_activity() + if user.avatar: + data["icon"]["url"] = archive_file_location(user.avatar) else: - # change the URL to be relative to the JSON file - file_type = exported_user["icon"]["url"].rsplit(".", maxsplit=1)[-1] - filename = f"avatar.{file_type}" - exported_user["icon"]["url"] = filename + data["icon"] = {} + return data - # Additional settings - can't be serialized as AP + +def export_settings(user: User): + """Additional settings - can't be serialized as AP""" vals = [ "show_goal", "preferred_timezone", "default_post_privacy", "show_suggested_users", ] - exported_user["settings"] = {} - for k in vals: - exported_user["settings"][k] = getattr(user, k) + return {k: getattr(user, k) for k in vals} - # Reading goals - can't be serialized as AP - reading_goals = AnnualGoal.objects.filter(user=user).distinct() - exported_user["goals"] = [] - for goal in reading_goals: - exported_user["goals"].append( - {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy} - ) - # Reading history - can't be serialized as AP - readthroughs = ReadThrough.objects.filter(user=user).distinct().values() - readthroughs = list(readthroughs) +def export_saved_lists(user: User): + """add user saved lists to export JSON""" + return [l.remote_id for l in user.saved_lists.all()] - # Books - editions = get_books_for_user(user) - exported_user["books"] = [] - for edition in editions: - book = {} - book["work"] = edition.parent_work.to_activity() - book["edition"] = edition.to_activity() - - if book["edition"].get("cover"): - # change the URL to be relative to the JSON file - filename = book["edition"]["cover"]["url"].rsplit("/", maxsplit=1)[-1] - book["edition"]["cover"]["url"] = f"covers/{filename}" - - # authors - book["authors"] = [] - for author in edition.authors.all(): - book["authors"].append(author.to_activity()) - - # Shelves this book is on - # Every ShelfItem is this book so we don't other serializing - book["shelves"] = [] - shelf_books = ( - ShelfBook.objects.select_related("shelf") - .filter(user=user, book=edition) - .distinct() - ) - - for shelfbook in shelf_books: - book["shelves"].append(shelfbook.shelf.to_activity()) - - # Lists and ListItems - # ListItems include "notes" and "approved" so we need them - # even though we know it's this book - book["lists"] = [] - list_items = ListItem.objects.filter(book=edition, user=user).distinct() - - for item in list_items: - list_info = item.book_list.to_activity() - list_info[ - "privacy" - ] = item.book_list.privacy # this isn't serialized so we add it - list_info["list_item"] = item.to_activity() - book["lists"].append(list_info) - - # Statuses - # Can't use select_subclasses here because - # we need to filter on the "book" value, - # which is not available on an ordinary Status - for status in ["comments", "quotations", "reviews"]: - book[status] = [] - - comments = Comment.objects.filter(user=user, book=edition).all() - for status in comments: - obj = status.to_activity() - obj["progress"] = status.progress - obj["progress_mode"] = status.progress_mode - book["comments"].append(obj) - - quotes = Quotation.objects.filter(user=user, book=edition).all() - for status in quotes: - obj = status.to_activity() - obj["position"] = status.position - obj["endposition"] = status.endposition - obj["position_mode"] = status.position_mode - book["quotations"].append(obj) - - reviews = Review.objects.filter(user=user, book=edition).all() - for status in reviews: - obj = status.to_activity() - book["reviews"].append(obj) - - # readthroughs can't be serialized to activity - book_readthroughs = ( - ReadThrough.objects.filter(user=user, book=edition).distinct().values() - ) - book["readthroughs"] = list(book_readthroughs) - - # append everything - exported_user["books"].append(book) - - # saved book lists - just the remote id - saved_lists = List.objects.filter(id__in=user.saved_lists.all()).distinct() - exported_user["saved_lists"] = [l.remote_id for l in saved_lists] - - # follows - just the remote id +def export_follows(user: User): + """add user follows to export JSON""" follows = UserFollows.objects.filter(user_subject=user).distinct() following = User.objects.filter(userfollows_user_object__in=follows).distinct() - exported_user["follows"] = [f.remote_id for f in following] + return [f.remote_id for f in following] - # blocks - just the remote id + +def export_blocks(user: User): + """add user blocks to export JSON""" blocks = UserBlocks.objects.filter(user_subject=user).distinct() blocking = User.objects.filter(userblocks_user_object__in=blocks).distinct() + return [b.remote_id for b in blocking] - exported_user["blocks"] = [b.remote_id for b in blocking] - return DjangoJSONEncoder().encode(exported_user) +def export_goals(user: User): + """add user reading goals to export JSON""" + reading_goals = AnnualGoal.objects.filter(user=user).distinct() + return [ + {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy} + for goal in reading_goals + ] + + +def export_books(user: User): + """add books to export JSON""" + editions = get_books_for_user(user) + return [export_book(user, edition) for edition in editions] + + +def export_book(user: User, edition: Edition): + """add book to export JSON""" + data = {} + data["work"] = edition.parent_work.to_activity() + data["edition"] = edition.to_activity() + + if edition.cover: + data["edition"]["cover"]["url"] = archive_file_location( + edition.cover, directory="images" + ) + + # authors + data["authors"] = [author.to_activity() for author in edition.authors.all()] + + # Shelves this book is on + # Every ShelfItem is this book so we don't other serializing + shelf_books = ( + ShelfBook.objects.select_related("shelf") + .filter(user=user, book=edition) + .distinct() + ) + data["shelves"] = [shelfbook.shelf.to_activity() for shelfbook in shelf_books] + + # Lists and ListItems + # ListItems include "notes" and "approved" so we need them + # even though we know it's this book + list_items = ListItem.objects.filter(book=edition, user=user).distinct() + + data["lists"] = [] + for item in list_items: + list_info = item.book_list.to_activity() + list_info[ + "privacy" + ] = item.book_list.privacy # this isn't serialized so we add it + list_info["list_item"] = item.to_activity() + data["lists"].append(list_info) + + # Statuses + # Can't use select_subclasses here because + # we need to filter on the "book" value, + # which is not available on an ordinary Status + for status in ["comments", "quotations", "reviews"]: + data[status] = [] + + comments = Comment.objects.filter(user=user, book=edition).all() + for status in comments: + obj = status.to_activity() + obj["progress"] = status.progress + obj["progress_mode"] = status.progress_mode + data["comments"].append(obj) + + quotes = Quotation.objects.filter(user=user, book=edition).all() + for status in quotes: + obj = status.to_activity() + obj["position"] = status.position + obj["endposition"] = status.endposition + obj["position_mode"] = status.position_mode + data["quotations"].append(obj) + + reviews = Review.objects.filter(user=user, book=edition).all() + data["reviews"] = [status.to_activity() for status in reviews] + + # readthroughs can't be serialized to activity + book_readthroughs = ( + ReadThrough.objects.filter(user=user, book=edition).distinct().values() + ) + data["readthroughs"] = list(book_readthroughs) + return data def get_books_for_user(user): - """Get all the books and editions related to a user""" + """ + Get all the books and editions related to a user. - editions = ( - Edition.objects.select_related("parent_work") - .filter( - Q(shelves__user=user) - | Q(readthrough__user=user) - | Q(review__user=user) - | Q(list__user=user) - | Q(comment__user=user) - | Q(quotation__user=user) - ) - .distinct() + We use union() instead of Q objects because it creates + multiple simple queries in stead of a much more complex DB query + that can time out. + + """ + + shelf_eds = Edition.objects.select_related("parent_work").filter(shelves__user=user) + rt_eds = Edition.objects.select_related("parent_work").filter( + readthrough__user=user + ) + review_eds = Edition.objects.select_related("parent_work").filter(review__user=user) + list_eds = Edition.objects.select_related("parent_work").filter(list__user=user) + comment_eds = Edition.objects.select_related("parent_work").filter( + comment__user=user + ) + quote_eds = Edition.objects.select_related("parent_work").filter( + quotation__user=user ) + editions = shelf_eds.union(rt_eds, review_eds, list_eds, comment_eds, quote_eds) + return editions diff --git a/bookwyrm/models/bookwyrm_import_job.py b/bookwyrm/models/bookwyrm_import_job.py index 9a11fd932..5229430eb 100644 --- a/bookwyrm/models/bookwyrm_import_job.py +++ b/bookwyrm/models/bookwyrm_import_job.py @@ -42,20 +42,23 @@ def start_import_task(**kwargs): try: archive_file.open("rb") with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: - job.import_data = json.loads(tar.read("archive.json").decode("utf-8")) + json_filename = next( + filter(lambda n: n.startswith("archive"), tar.getnames()) + ) + job.import_data = json.loads(tar.read(json_filename).decode("utf-8")) if "include_user_profile" in job.required: update_user_profile(job.user, tar, job.import_data) if "include_user_settings" in job.required: update_user_settings(job.user, job.import_data) if "include_goals" in job.required: - update_goals(job.user, job.import_data.get("goals")) + update_goals(job.user, job.import_data.get("goals", [])) if "include_saved_lists" in job.required: - upsert_saved_lists(job.user, job.import_data.get("saved_lists")) + upsert_saved_lists(job.user, job.import_data.get("saved_lists", [])) if "include_follows" in job.required: - upsert_follows(job.user, job.import_data.get("follows")) + upsert_follows(job.user, job.import_data.get("follows", [])) if "include_blocks" in job.required: - upsert_user_blocks(job.user, job.import_data.get("blocks")) + upsert_user_blocks(job.user, job.import_data.get("blocks", [])) process_books(job, tar) @@ -212,7 +215,7 @@ def upsert_statuses(user, cls, data, book_remote_id): instance.save() # save and broadcast else: - logger.info("User does not have permission to import statuses") + logger.warning("User does not have permission to import statuses") def upsert_lists(user, lists, book_id): diff --git a/bookwyrm/models/connector.py b/bookwyrm/models/connector.py index 99e73ab37..f4b5be04c 100644 --- a/bookwyrm/models/connector.py +++ b/bookwyrm/models/connector.py @@ -11,7 +11,7 @@ ConnectorFiles = models.TextChoices("ConnectorFiles", CONNECTORS) class Connector(BookWyrmModel): """book data source connectors""" - identifier = models.CharField(max_length=255, unique=True) + identifier = models.CharField(max_length=255, unique=True) # domain priority = models.IntegerField(default=2) name = models.CharField(max_length=255, null=True, blank=True) connector_file = models.CharField(max_length=255, choices=ConnectorFiles.choices) diff --git a/bookwyrm/models/federated_server.py b/bookwyrm/models/federated_server.py index e1081ed45..5e08fc11d 100644 --- a/bookwyrm/models/federated_server.py +++ b/bookwyrm/models/federated_server.py @@ -16,7 +16,7 @@ FederationStatus = [ class FederatedServer(BookWyrmModel): """store which servers we federate with""" - server_name = models.CharField(max_length=255, unique=True) + server_name = models.CharField(max_length=255, unique=True) # domain status = models.CharField( max_length=255, default="federated", choices=FederationStatus ) @@ -64,5 +64,4 @@ class FederatedServer(BookWyrmModel): def is_blocked(cls, url: str) -> bool: """look up if a domain is blocked""" url = urlparse(url) - domain = url.netloc - return cls.objects.filter(server_name=domain, status="blocked").exists() + return cls.objects.filter(server_name=url.hostname, status="blocked").exists() diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 6643bdc19..75d3acf76 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -193,8 +193,7 @@ class UsernameField(ActivitypubFieldMixin, models.CharField): def __init__(self, activitypub_field="preferredUsername", **kwargs): self.activitypub_field = activitypub_field - # I don't totally know why pylint is mad at this, but it makes it work - super(ActivitypubFieldMixin, self).__init__( # pylint: disable=bad-super-call + super(ActivitypubFieldMixin, self).__init__( _("username"), max_length=150, unique=True, @@ -234,7 +233,6 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField): def __init__(self, *args, **kwargs): super().__init__(*args, max_length=255, choices=PrivacyLevels, default="public") - # pylint: disable=invalid-name def set_field_from_activity( self, instance, data, overwrite=True, allow_external_connections=True ): @@ -276,7 +274,6 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField): if hasattr(instance, "mention_users"): mentions = [u.remote_id for u in instance.mention_users.all()] # this is a link to the followers list - # pylint: disable=protected-access followers = instance.user.followers_url if instance.privacy == "public": activity["to"] = [self.public] @@ -444,7 +441,7 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): self.alt_field = alt_field super().__init__(*args, **kwargs) - # pylint: disable=arguments-differ,arguments-renamed,too-many-arguments + # pylint: disable=arguments-renamed,too-many-arguments def set_field_from_activity( self, instance, data, save=True, overwrite=True, allow_external_connections=True ): diff --git a/bookwyrm/models/group.py b/bookwyrm/models/group.py index d02b56ab1..40a32b5dc 100644 --- a/bookwyrm/models/group.py +++ b/bookwyrm/models/group.py @@ -1,7 +1,7 @@ """ do book related things with other users """ from django.db import models, IntegrityError, transaction from django.db.models import Q -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import BASE_URL from .base_model import BookWyrmModel from . import fields from .relationship import UserBlocks @@ -17,7 +17,7 @@ class Group(BookWyrmModel): def get_remote_id(self): """don't want the user to be in there in this case""" - return f"https://{DOMAIN}/group/{self.id}" + return f"{BASE_URL}/group/{self.id}" @classmethod def followers_filter(cls, queryset, viewer): diff --git a/bookwyrm/models/hashtag.py b/bookwyrm/models/hashtag.py index 7894a3528..5126f012d 100644 --- a/bookwyrm/models/hashtag.py +++ b/bookwyrm/models/hashtag.py @@ -2,18 +2,19 @@ from bookwyrm import activitypub from .activitypub_mixin import ActivitypubMixin from .base_model import BookWyrmModel -from .fields import CICharField +from .fields import CharField class Hashtag(ActivitypubMixin, BookWyrmModel): "a hashtag which can be used in statuses" - name = CICharField( + name = CharField( max_length=256, blank=False, null=False, activitypub_field="name", deduplication_field=True, + db_collation="case_insensitive", ) name_field = "name" diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index f5d86ad2e..5a6ba3f51 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -4,6 +4,7 @@ import math import re import dateutil.parser +from django.core.exceptions import ObjectDoesNotExist from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -59,6 +60,7 @@ class ImportJob(models.Model): created_date = models.DateTimeField(default=timezone.now) updated_date = models.DateTimeField(default=timezone.now) include_reviews: bool = models.BooleanField(default=True) + create_shelves: bool = models.BooleanField(default=True) mappings = models.JSONField() source = models.CharField(max_length=100) privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels) @@ -245,11 +247,26 @@ class ImportItem(models.Model): """the goodreads shelf field""" return self.normalized_data.get("shelf") + @property + def shelf_name(self): + """the goodreads shelf field""" + return self.normalized_data.get("shelf_name") + @property def review(self): """a user-written review, to be imported with the book data""" return self.normalized_data.get("review_body") + @property + def review_name(self): + """a user-written review name, to be imported with the book data""" + return self.normalized_data.get("review_name") + + @property + def review_published(self): + """date the review was published - included in BookWyrm export csv""" + return self.normalized_data.get("review_published", None) + @property def rating(self): """x/5 star rating for a book""" @@ -352,7 +369,7 @@ def import_item_task(item_id): try: item.resolve() - except Exception as err: # pylint: disable=broad-except + except Exception as err: item.fail_reason = _("Error loading book") item.save() item.update_job() @@ -368,7 +385,7 @@ def import_item_task(item_id): item.update_job() -def handle_imported_book(item): +def handle_imported_book(item): # pylint: disable=too-many-branches """process a csv and then post about it""" job = item.job if job.complete: @@ -385,13 +402,31 @@ def handle_imported_book(item): item.book = item.book.edition existing_shelf = ShelfBook.objects.filter(book=item.book, user=user).exists() + if job.create_shelves and item.shelf and not existing_shelf: + # shelve the book if it hasn't been shelved already - # shelve the book if it hasn't been shelved already - if item.shelf and not existing_shelf: - desired_shelf = Shelf.objects.get(identifier=item.shelf, user=user) shelved_date = item.date_added or timezone.now() + shelfname = getattr(item, "shelf_name", item.shelf) + + try: + shelf = Shelf.objects.get(name=shelfname, user=user) + except ObjectDoesNotExist: + try: + shelf = Shelf.objects.get(identifier=item.shelf, user=user) + except ObjectDoesNotExist: + + shelf = Shelf.objects.create( + user=user, + identifier=item.shelf, + name=shelfname, + privacy=job.privacy, + ) + ShelfBook( - book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date + book=item.book, + shelf=shelf, + user=user, + shelved_date=shelved_date, ).save(priority=IMPORT_TRIGGERED) for read in item.reads: @@ -408,19 +443,25 @@ def handle_imported_book(item): read.save() if job.include_reviews and (item.rating or item.review) and not item.linked_review: - # we don't know the publication date of the review, - # but "now" is a bad guess - published_date_guess = item.date_read or item.date_added + # we don't necessarily know the publication date of the review, + # but "now" is a bad guess unless we have no choice + + published_date_guess = ( + item.review_published or item.date_read or item.date_added or timezone.now() + ) if item.review: + # pylint: disable=consider-using-f-string review_title = "Review of {!r} on {!r}".format( item.book.title, job.source, ) + review_name = getattr(item, "review_name", review_title) + review = Review.objects.filter( user=user, book=item.book, - name=review_title, + name=review_name, rating=item.rating, published_date=published_date_guess, ).first() @@ -428,7 +469,7 @@ def handle_imported_book(item): review = Review( user=user, book=item.book, - name=review_title, + name=review_name, content=item.review, rating=item.rating, published_date=published_date_guess, diff --git a/bookwyrm/models/job.py b/bookwyrm/models/job.py index 4f5cb2093..5a2653571 100644 --- a/bookwyrm/models/job.py +++ b/bookwyrm/models/job.py @@ -135,8 +135,7 @@ class ParentJob(Job): ) app.control.revoke(list(tasks)) - for task in self.pending_child_jobs: - task.update(status=self.Status.STOPPED) + self.pending_child_jobs.update(status=self.Status.STOPPED) @property def has_completed(self): @@ -248,7 +247,7 @@ class SubTask(app.Task): """ def before_start( - self, task_id, args, kwargs + self, task_id, *args, **kwargs ): # pylint: disable=no-self-use, unused-argument """Handler called before the task starts. Override. @@ -272,7 +271,7 @@ class SubTask(app.Task): child_job.set_status(ChildJob.Status.ACTIVE) def on_success( - self, retval, task_id, args, kwargs + self, retval, task_id, *args, **kwargs ): # pylint: disable=no-self-use, unused-argument """Run by the worker if the task executes successfully. Override. diff --git a/bookwyrm/models/link.py b/bookwyrm/models/link.py index d334a9d29..4519f0c81 100644 --- a/bookwyrm/models/link.py +++ b/bookwyrm/models/link.py @@ -1,4 +1,5 @@ """ outlink data """ +from typing import Optional, Iterable from urllib.parse import urlparse from django.core.exceptions import PermissionDenied @@ -6,6 +7,7 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from bookwyrm import activitypub +from bookwyrm.utils.db import add_update_fields from .activitypub_mixin import ActivitypubMixin from .base_model import BookWyrmModel from . import fields @@ -34,17 +36,19 @@ class Link(ActivitypubMixin, BookWyrmModel): """link name via the associated domain""" return self.domain.name - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """create a link""" # get or create the associated domain if not self.domain: - domain = urlparse(self.url).netloc + domain = urlparse(self.url).hostname self.domain, _ = LinkDomain.objects.get_or_create(domain=domain) + update_fields = add_update_fields(update_fields, "domain") # this is never broadcast, the owning model broadcasts an update if "broadcast" in kwargs: del kwargs["broadcast"] - return super().save(*args, **kwargs) + + super().save(*args, update_fields=update_fields, **kwargs) AvailabilityChoices = [ @@ -88,8 +92,10 @@ class LinkDomain(BookWyrmModel): return raise PermissionDenied() - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """set a default name""" if not self.name: self.name = self.domain - super().save(*args, **kwargs) + update_fields = add_update_fields(update_fields, "name") + + super().save(*args, update_fields=update_fields, **kwargs) diff --git a/bookwyrm/models/list.py b/bookwyrm/models/list.py index 63dd5b23f..df7e8162c 100644 --- a/bookwyrm/models/list.py +++ b/bookwyrm/models/list.py @@ -1,4 +1,5 @@ """ make a list of books!! """ +from typing import Optional, Iterable import uuid from django.core.exceptions import PermissionDenied @@ -7,7 +8,8 @@ from django.db.models import Q from django.utils import timezone from bookwyrm import activitypub -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import BASE_URL +from bookwyrm.utils.db import add_update_fields from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin from .base_model import BookWyrmModel @@ -50,7 +52,7 @@ class List(OrderedCollectionMixin, BookWyrmModel): def get_remote_id(self): """don't want the user to be in there in this case""" - return f"https://{DOMAIN}/list/{self.id}" + return f"{BASE_URL}/list/{self.id}" @property def collection_queryset(self): @@ -124,11 +126,13 @@ class List(OrderedCollectionMixin, BookWyrmModel): group=None, curation="closed" ) - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """on save, update embed_key and avoid clash with existing code""" if not self.embed_key: self.embed_key = uuid.uuid4() - super().save(*args, **kwargs) + update_fields = add_update_fields(update_fields, "embed_key") + + super().save(*args, update_fields=update_fields, **kwargs) class ListItem(CollectionItemMixin, BookWyrmModel): diff --git a/bookwyrm/models/move.py b/bookwyrm/models/move.py index d6d8ef78f..5038058b7 100644 --- a/bookwyrm/models/move.py +++ b/bookwyrm/models/move.py @@ -10,7 +10,7 @@ from .notification import Notification, NotificationType class Move(ActivityMixin, BookWyrmModel): - """migrating an activitypub user account""" + """migrating an activitypub object""" user = fields.ForeignKey( "User", on_delete=models.PROTECT, activitypub_field="actor" @@ -48,24 +48,21 @@ class MoveUser(Move): """update user info and broadcast it""" # only allow if the source is listed in the target's alsoKnownAs - if self.user in self.target.also_known_as.all(): - self.user.also_known_as.add(self.target.id) - self.user.update_active_date() - self.user.moved_to = self.target.remote_id - self.user.save(update_fields=["moved_to"]) - - if self.user.local: - kwargs[ - "broadcast" - ] = True # Only broadcast if we are initiating the Move - - super().save(*args, **kwargs) - - for follower in self.user.followers.all(): - if follower.local: - Notification.notify( - follower, self.user, notification_type=NotificationType.MOVE - ) - - else: + if self.user not in self.target.also_known_as.all(): raise PermissionDenied() + + self.user.also_known_as.add(self.target.id) + self.user.update_active_date() + self.user.moved_to = self.target.remote_id + self.user.save(update_fields=["moved_to"]) + + if self.user.local: + kwargs["broadcast"] = True # Only broadcast if we are initiating the Move + + super().save(*args, **kwargs) + + for follower in self.user.followers.all(): + if follower.local: + Notification.notify( + follower, self.user, notification_type=NotificationType.MOVE + ) diff --git a/bookwyrm/models/readthrough.py b/bookwyrm/models/readthrough.py index 4911c715b..7700b4a87 100644 --- a/bookwyrm/models/readthrough.py +++ b/bookwyrm/models/readthrough.py @@ -1,9 +1,13 @@ """ progress in a book """ +from typing import Optional, Iterable + from django.core import validators from django.core.cache import cache from django.db import models from django.db.models import F, Q +from bookwyrm.utils.db import add_update_fields + from .base_model import BookWyrmModel @@ -30,14 +34,17 @@ class ReadThrough(BookWyrmModel): stopped_date = models.DateTimeField(blank=True, null=True) is_active = models.BooleanField(default=True) - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """update user active time""" - cache.delete(f"latest_read_through-{self.user_id}-{self.book_id}") - self.user.update_active_date() # an active readthrough must have an unset finish date if self.finish_date or self.stopped_date: self.is_active = False - super().save(*args, **kwargs) + update_fields = add_update_fields(update_fields, "is_active") + + super().save(*args, update_fields=update_fields, **kwargs) + + cache.delete(f"latest_read_through-{self.user_id}-{self.book_id}") + self.user.update_active_date() def create_update(self): """add update to the readthrough""" diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index 3386a02dc..ed630fbe5 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -38,14 +38,16 @@ class UserRelationship(BookWyrmModel): def save(self, *args, **kwargs): """clear the template cache""" - clear_cache(self.user_subject, self.user_object) super().save(*args, **kwargs) + clear_cache(self.user_subject, self.user_object) + def delete(self, *args, **kwargs): """clear the template cache""" - clear_cache(self.user_subject, self.user_object) super().delete(*args, **kwargs) + clear_cache(self.user_subject, self.user_object) + class Meta: """relationships should be unique""" @@ -133,7 +135,7 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship): status = "follow_request" activity_serializer = activitypub.Follow - def save(self, *args, broadcast=True, **kwargs): # pylint: disable=arguments-differ + def save(self, *args, broadcast=True, **kwargs): """make sure the follow or block relationship doesn't already exist""" # if there's a request for a follow that already exists, accept it # without changing the local database state diff --git a/bookwyrm/models/report.py b/bookwyrm/models/report.py index 74a9bbe41..64ade3a40 100644 --- a/bookwyrm/models/report.py +++ b/bookwyrm/models/report.py @@ -3,7 +3,7 @@ from django.core.exceptions import PermissionDenied from django.db import models from django.utils.translation import gettext_lazy as _ -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import BASE_URL from .base_model import BookWyrmModel @@ -46,7 +46,7 @@ class Report(BookWyrmModel): raise PermissionDenied() def get_remote_id(self): - return f"https://{DOMAIN}/settings/reports/{self.id}" + return f"{BASE_URL}/settings/reports/{self.id}" def comment(self, user, note): """comment on a report""" diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py index 3d92f8d43..0b9ef2b09 100644 --- a/bookwyrm/models/shelf.py +++ b/bookwyrm/models/shelf.py @@ -1,13 +1,15 @@ """ puttin' books on shelves """ import re +from typing import Optional, Iterable from django.core.cache import cache from django.core.exceptions import PermissionDenied from django.db import models from django.utils import timezone from bookwyrm import activitypub -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import BASE_URL from bookwyrm.tasks import BROADCAST +from bookwyrm.utils.db import add_update_fields from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin from .base_model import BookWyrmModel from . import fields @@ -44,8 +46,9 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): """set the identifier""" super().save(*args, priority=priority, **kwargs) if not self.identifier: + # this needs the auto increment ID from the save() above self.identifier = self.get_identifier() - super().save(*args, **kwargs, broadcast=False) + super().save(*args, **kwargs, broadcast=False, update_fields={"identifier"}) def get_identifier(self): """custom-shelf-123 for the url""" @@ -71,7 +74,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): @property def local_path(self): """No slugs""" - return self.get_remote_id().replace(f"https://{DOMAIN}", "") + return self.get_remote_id().replace(BASE_URL, "") def raise_not_deletable(self, viewer): """don't let anyone delete a default shelf""" @@ -100,10 +103,21 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel): activity_serializer = activitypub.ShelfItem collection_field = "shelf" - def save(self, *args, priority=BROADCAST, **kwargs): + def save( + self, + *args, + priority=BROADCAST, + update_fields: Optional[Iterable[str]] = None, + **kwargs, + ): if not self.user: self.user = self.shelf.user - if self.id and self.user.local: + update_fields = add_update_fields(update_fields, "user") + + is_update = self.id is not None + super().save(*args, priority=priority, update_fields=update_fields, **kwargs) + + if is_update and self.user.local: # remove all caches related to all editions of this book cache.delete_many( [ @@ -111,7 +125,6 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel): for book in self.book.parent_work.editions.all() ] ) - super().save(*args, priority=priority, **kwargs) def delete(self, *args, **kwargs): if self.id and self.user.local: diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index 201a499e5..6c2a73422 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -1,5 +1,6 @@ """ the particulars for this instance of BookWyrm """ import datetime +from typing import Optional, Iterable from urllib.parse import urljoin import uuid @@ -12,9 +13,10 @@ from model_utils import FieldTracker from bookwyrm.connectors.abstract_connector import get_data from bookwyrm.preview_images import generate_site_preview_image_task -from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL +from bookwyrm.settings import BASE_URL, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL from bookwyrm.settings import RELEASE_API from bookwyrm.tasks import app, MISC +from bookwyrm.utils.db import add_update_fields from .base_model import BookWyrmModel, new_access_code from .user import User from .fields import get_absolute_url @@ -136,16 +138,19 @@ class SiteSettings(SiteModel): return get_absolute_url(uploaded) return urljoin(STATIC_FULL_URL, default_path) - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """if require_confirm_email is disabled, make sure no users are pending, if enabled, make sure invite_question_text is not empty""" + if not self.invite_question_text: + self.invite_question_text = "What is your favourite book?" + update_fields = add_update_fields(update_fields, "invite_question_text") + + super().save(*args, update_fields=update_fields, **kwargs) + if not self.require_confirm_email: User.objects.filter(is_active=False, deactivation_reason="pending").update( is_active=True, deactivation_reason=None ) - if not self.invite_question_text: - self.invite_question_text = "What is your favourite book?" - super().save(*args, **kwargs) class Theme(SiteModel): @@ -188,7 +193,7 @@ class SiteInvite(models.Model): @property def link(self): """formats the invite link""" - return f"https://{DOMAIN}/invite/{self.code}" + return f"{BASE_URL}/invite/{self.code}" class InviteRequest(BookWyrmModel): @@ -235,7 +240,7 @@ class PasswordReset(models.Model): @property def link(self): """formats the invite link""" - return f"https://{DOMAIN}/password-reset/{self.code}" + return f"{BASE_URL}/password-reset/{self.code}" # pylint: disable=unused-argument diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index f6235dab6..2b357ebd2 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -1,6 +1,6 @@ """ models for storing different kinds of Activities """ from dataclasses import MISSING -from typing import Optional +from typing import Optional, Iterable import re from django.apps import apps @@ -20,6 +20,7 @@ from model_utils.managers import InheritanceManager from bookwyrm import activitypub from bookwyrm.preview_images import generate_edition_preview_image_task from bookwyrm.settings import ENABLE_PREVIEW_IMAGES +from bookwyrm.utils.db import add_update_fields from .activitypub_mixin import ActivitypubMixin, ActivityMixin from .activitypub_mixin import OrderedCollectionPageMixin from .base_model import BookWyrmModel @@ -80,19 +81,24 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): """default sorting""" ordering = ("-published_date",) + indexes = [ + models.Index(fields=["remote_id"]), + models.Index(fields=["thread_id"]), + ] - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """save and notify""" - if self.reply_parent: + if self.thread_id is None and self.reply_parent: self.thread_id = self.reply_parent.thread_id or self.reply_parent_id + update_fields = add_update_fields(update_fields, "thread_id") - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) if not self.reply_parent: self.thread_id = self.id super().save(broadcast=False, update_fields=["thread_id"]) - def delete(self, *args, **kwargs): # pylint: disable=unused-argument + def delete(self, *args, **kwargs): """ "delete" a status""" if hasattr(self, "boosted_status"): # okay but if it's a boost really delete it @@ -207,7 +213,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): **kwargs, ).serialize() - def to_activity_dataclass(self, pure=False): # pylint: disable=arguments-differ + def to_activity_dataclass(self, pure=False): """return tombstone if the status is deleted""" if self.deleted: return activitypub.Tombstone( @@ -388,10 +394,10 @@ class Quotation(BookStatus): def _format_position(self) -> Optional[str]: """serialize page position""" beg = self.position - end = self.endposition or 0 + end = self.endposition if self.position_mode != "PG" or not beg: return None - return f"pp. {beg}-{end}" if end > beg else f"p. {beg}" + return f"pp. {beg}-{end}" if end else f"p. {beg}" @property def pure_content(self): @@ -455,9 +461,10 @@ class Review(BookStatus): def save(self, *args, **kwargs): """clear rating caches""" + super().save(*args, **kwargs) + if self.book.parent_work: cache.delete(f"book-rating-{self.book.parent_work.id}") - super().save(*args, **kwargs) class ReviewRating(Review): diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 89fd39b73..93465720e 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -1,28 +1,31 @@ """ database schema for user data """ +import datetime import re +import zoneinfo +from typing import Optional, Iterable from urllib.parse import urlparse from uuid import uuid4 from django.apps import apps from django.contrib.auth.models import AbstractUser -from django.contrib.postgres.fields import ArrayField, CICharField +from django.contrib.postgres.fields import ArrayField as DjangoArrayField from django.core.exceptions import PermissionDenied, ObjectDoesNotExist from django.dispatch import receiver from django.db import models, transaction, IntegrityError from django.utils import timezone from django.utils.translation import gettext_lazy as _ from model_utils import FieldTracker -import pytz from bookwyrm import activitypub from bookwyrm.connectors import get_data, ConnectorException from bookwyrm.models.shelf import Shelf from bookwyrm.models.status import Status from bookwyrm.preview_images import generate_user_preview_image_task -from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, USE_HTTPS, LANGUAGES +from bookwyrm.settings import BASE_URL, ENABLE_PREVIEW_IMAGES, LANGUAGES from bookwyrm.signatures import create_key_pair from bookwyrm.tasks import app, MISC from bookwyrm.utils import regex +from bookwyrm.utils.db import add_update_fields from .activitypub_mixin import OrderedCollectionPageMixin, ActivitypubMixin from .base_model import BookWyrmModel, DeactivationReason, new_access_code from .federated_server import FederatedServer @@ -42,12 +45,6 @@ def get_feed_filter_choices(): return [f[0] for f in FeedFilterChoices] -def site_link(): - """helper for generating links to the site""" - protocol = "https" if USE_HTTPS else "http" - return f"{protocol}://{DOMAIN}" - - # pylint: disable=too-many-public-methods class User(OrderedCollectionPageMixin, AbstractUser): """a user who wants to read books""" @@ -81,11 +78,12 @@ class User(OrderedCollectionPageMixin, AbstractUser): summary = fields.HtmlField(null=True, blank=True) local = models.BooleanField(default=False) bookwyrm_user = fields.BooleanField(default=True) - localname = CICharField( + localname = models.CharField( max_length=255, null=True, unique=True, validators=[fields.validate_localname], + db_collation="case_insensitive", ) # name is your display name, which you can change at will name = fields.CharField(max_length=100, null=True, blank=True) @@ -143,7 +141,6 @@ class User(OrderedCollectionPageMixin, AbstractUser): hide_follows = fields.BooleanField(default=False) # migration fields - moved_to = fields.RemoteIdField( null=True, unique=False, activitypub_field="movedTo", deduplication_field=False ) @@ -160,9 +157,10 @@ class User(OrderedCollectionPageMixin, AbstractUser): show_suggested_users = models.BooleanField(default=True) discoverable = fields.BooleanField(default=False) show_guided_tour = models.BooleanField(default=True) + show_ratings = models.BooleanField(default=True) # feed options - feed_status_types = ArrayField( + feed_status_types = DjangoArrayField( models.CharField(max_length=10, blank=False, choices=FeedFilterChoices), size=8, default=get_feed_filter_choices, @@ -171,8 +169,8 @@ class User(OrderedCollectionPageMixin, AbstractUser): summary_keys = models.JSONField(null=True) preferred_timezone = models.CharField( - choices=[(str(tz), str(tz)) for tz in pytz.all_timezones], - default=str(pytz.utc), + choices=[(str(tz), str(tz)) for tz in sorted(zoneinfo.available_timezones())], + default=str(datetime.timezone.utc), max_length=255, ) preferred_language = models.CharField( @@ -198,6 +196,14 @@ class User(OrderedCollectionPageMixin, AbstractUser): hotp_secret = models.CharField(max_length=32, default=None, blank=True, null=True) hotp_count = models.IntegerField(default=0, blank=True, null=True) + class Meta(AbstractUser.Meta): + """indexes""" + + indexes = [ + models.Index(fields=["username"]), + models.Index(fields=["is_active", "local"]), + ] + @property def active_follower_requests(self): """Follow requests from active users""" @@ -206,8 +212,7 @@ class User(OrderedCollectionPageMixin, AbstractUser): @property def confirmation_link(self): """helper for generating confirmation links""" - link = site_link() - return f"{link}/confirm-email/{self.confirmation_code}" + return f"{BASE_URL}/confirm-email/{self.confirmation_code}" @property def following_link(self): @@ -326,6 +331,7 @@ class User(OrderedCollectionPageMixin, AbstractUser): "https://w3id.org/security/v1", { "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "Hashtag": "as:Hashtag", "schema": "http://schema.org#", "PropertyValue": "schema:PropertyValue", "value": "schema:value", @@ -335,13 +341,14 @@ class User(OrderedCollectionPageMixin, AbstractUser): ] return activity_object - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """populate fields for new local users""" created = not bool(self.id) if not self.local and not re.match(regex.FULL_USERNAME, self.username): # generate a username that uses the domain (webfinger format) actor_parts = urlparse(self.remote_id) - self.username = f"{self.username}@{actor_parts.netloc}" + self.username = f"{self.username}@{actor_parts.hostname}" + update_fields = add_update_fields(update_fields, "username") # this user already exists, no need to populate fields if not created: @@ -350,26 +357,34 @@ class User(OrderedCollectionPageMixin, AbstractUser): elif not self.deactivation_date: self.deactivation_date = timezone.now() - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) return # this is a new remote user, we need to set their remote server field if not self.local: - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) transaction.on_commit(lambda: set_remote_server(self.id)) return with transaction.atomic(): # populate fields for local users - link = site_link() - self.remote_id = f"{link}/user/{self.localname}" + self.remote_id = f"{BASE_URL}/user/{self.localname}" self.followers_url = f"{self.remote_id}/followers" self.inbox = f"{self.remote_id}/inbox" - self.shared_inbox = f"{link}/inbox" + self.shared_inbox = f"{BASE_URL}/inbox" self.outbox = f"{self.remote_id}/outbox" + update_fields = add_update_fields( + update_fields, + "remote_id", + "followers_url", + "inbox", + "shared_inbox", + "outbox", + ) + # an id needs to be set before we can proceed with related models - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) # make users editors by default try: @@ -394,7 +409,6 @@ class User(OrderedCollectionPageMixin, AbstractUser): def delete(self, *args, **kwargs): """We don't actually delete the database entry""" - # pylint: disable=attribute-defined-outside-init self.is_active = False self.allow_reactivation = False self.is_deleted = True @@ -437,7 +451,6 @@ class User(OrderedCollectionPageMixin, AbstractUser): def deactivate(self): """Disable the user but allow them to reactivate""" - # pylint: disable=attribute-defined-outside-init self.is_active = False self.deactivation_reason = "self_deactivation" self.allow_reactivation = True @@ -445,7 +458,6 @@ class User(OrderedCollectionPageMixin, AbstractUser): def reactivate(self): """Now you want to come back, huh?""" - # pylint: disable=attribute-defined-outside-init if not self.allow_reactivation: return self.is_active = True @@ -509,18 +521,30 @@ class KeyPair(ActivitypubMixin, BookWyrmModel): activity_serializer = activitypub.PublicKey serialize_reverse_fields = [("owner", "owner", "id")] + class Meta: + """indexes""" + + indexes = [ + models.Index(fields=["remote_id"]), + ] + def get_remote_id(self): # self.owner is set by the OneToOneField on User return f"{self.owner.remote_id}/#main-key" - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """create a key pair""" # no broadcasting happening here if "broadcast" in kwargs: del kwargs["broadcast"] + if not self.public_key: self.private_key, self.public_key = create_key_pair() - return super().save(*args, **kwargs) + update_fields = add_update_fields( + update_fields, "private_key", "public_key" + ) + + super().save(*args, update_fields=update_fields, **kwargs) @app.task(queue=MISC) @@ -543,7 +567,7 @@ def set_remote_server(user_id, allow_external_connections=False): user = User.objects.get(id=user_id) actor_parts = urlparse(user.remote_id) federated_server = get_or_create_remote_server( - actor_parts.netloc, allow_external_connections=allow_external_connections + actor_parts.hostname, allow_external_connections=allow_external_connections ) # if we were unable to find the server, we need to create a new entry for it if not federated_server: diff --git a/bookwyrm/preview_images.py b/bookwyrm/preview_images.py index 995f25bfd..66d2e2d4b 100644 --- a/bookwyrm/preview_images.py +++ b/bookwyrm/preview_images.py @@ -175,11 +175,13 @@ def generate_instance_layer(content_width): site = models.SiteSettings.objects.get() if site.logo_small: - logo_img = Image.open(site.logo_small) + with Image.open(site.logo_small) as logo_img: + logo_img.load() else: try: static_path = os.path.join(settings.STATIC_ROOT, "images/logo-small.png") - logo_img = Image.open(static_path) + with Image.open(static_path) as logo_img: + logo_img.load() except FileNotFoundError: logo_img = None @@ -211,18 +213,9 @@ def generate_instance_layer(content_width): def generate_rating_layer(rating, content_width): """Places components for rating preview""" - try: - icon_star_full = Image.open( - os.path.join(settings.STATIC_ROOT, "images/icons/star-full.png") - ) - icon_star_empty = Image.open( - os.path.join(settings.STATIC_ROOT, "images/icons/star-empty.png") - ) - icon_star_half = Image.open( - os.path.join(settings.STATIC_ROOT, "images/icons/star-half.png") - ) - except FileNotFoundError: - return None + path_star_full = os.path.join(settings.STATIC_ROOT, "images/icons/star-full.png") + path_star_empty = os.path.join(settings.STATIC_ROOT, "images/icons/star-empty.png") + path_star_half = os.path.join(settings.STATIC_ROOT, "images/icons/star-half.png") icon_size = 64 icon_margin = 10 @@ -237,17 +230,23 @@ def generate_rating_layer(rating, content_width): position_x = 0 - for _ in range(math.floor(rating)): - rating_layer_mask.alpha_composite(icon_star_full, (position_x, 0)) - position_x = position_x + icon_size + icon_margin + try: + with Image.open(path_star_full) as icon_star_full: + for _ in range(math.floor(rating)): + rating_layer_mask.alpha_composite(icon_star_full, (position_x, 0)) + position_x = position_x + icon_size + icon_margin - if math.floor(rating) != math.ceil(rating): - rating_layer_mask.alpha_composite(icon_star_half, (position_x, 0)) - position_x = position_x + icon_size + icon_margin + if math.floor(rating) != math.ceil(rating): + with Image.open(path_star_half) as icon_star_half: + rating_layer_mask.alpha_composite(icon_star_half, (position_x, 0)) + position_x = position_x + icon_size + icon_margin - for _ in range(5 - math.ceil(rating)): - rating_layer_mask.alpha_composite(icon_star_empty, (position_x, 0)) - position_x = position_x + icon_size + icon_margin + with Image.open(path_star_empty) as icon_star_empty: + for _ in range(5 - math.ceil(rating)): + rating_layer_mask.alpha_composite(icon_star_empty, (position_x, 0)) + position_x = position_x + icon_size + icon_margin + except FileNotFoundError: + return None rating_layer_mask = rating_layer_mask.getchannel("A") rating_layer_mask = ImageOps.invert(rating_layer_mask) @@ -290,7 +289,8 @@ def generate_preview_image( texts = texts or {} # Cover try: - inner_img_layer = Image.open(picture) + with Image.open(picture) as inner_img_layer: + inner_img_layer.load() inner_img_layer.thumbnail( (inner_img_width, inner_img_height), Image.Resampling.LANCZOS ) @@ -420,7 +420,6 @@ def save_and_cleanup(image, instance=None): return True -# pylint: disable=invalid-name @app.task(queue=IMAGES) def generate_site_preview_image_task(): """generate preview_image for the website""" @@ -445,7 +444,6 @@ def generate_site_preview_image_task(): save_and_cleanup(image, instance=site) -# pylint: disable=invalid-name @app.task(queue=IMAGES) def generate_edition_preview_image_task(book_id): """generate preview_image for a book""" diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 77bec0d8e..6da6f4bae 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -19,7 +19,6 @@ DOMAIN = env("DOMAIN") with open("VERSION", encoding="utf-8") as f: version = f.read() version = version.replace("\n", "") -f.close() VERSION = version @@ -102,6 +101,7 @@ INSTALLED_APPS = [ "django.contrib.messages", "django.contrib.staticfiles", "django.contrib.humanize", + "oauth2_provider", "file_resubmit", "sass_processor", "bookwyrm", @@ -257,11 +257,8 @@ if env.bool("USE_DUMMY_CACHE", False): else: CACHES = { "default": { - "BACKEND": "django_redis.cache.RedisCache", + "BACKEND": "django.core.cache.backends.redis.RedisCache", "LOCATION": REDIS_ACTIVITY_URL, - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - }, }, "file_resubmit": { "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", @@ -347,34 +344,37 @@ TIME_ZONE = "UTC" USE_I18N = True -USE_L10N = True - USE_TZ = True - -USER_AGENT = f"BookWyrm (BookWyrm/{VERSION}; +https://{DOMAIN}/)" - # Imagekit generated thumbnails ENABLE_THUMBNAIL_GENERATION = env.bool("ENABLE_THUMBNAIL_GENERATION", False) IMAGEKIT_CACHEFILE_DIR = "thumbnails" IMAGEKIT_DEFAULT_CACHEFILE_STRATEGY = "bookwyrm.thumbnail_generation.Strategy" -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.2/howto/static-files/ - PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) CSP_ADDITIONAL_HOSTS = env.list("CSP_ADDITIONAL_HOSTS", []) -# Storage - PROTOCOL = "http" if USE_HTTPS: PROTOCOL = "https" SESSION_COOKIE_SECURE = True CSRF_COOKIE_SECURE = True +PORT = env.int("PORT", 443 if USE_HTTPS else 80) +if (USE_HTTPS and PORT == 443) or (not USE_HTTPS and PORT == 80): + NETLOC = DOMAIN +else: + NETLOC = f"{DOMAIN}:{PORT}" +BASE_URL = f"{PROTOCOL}://{NETLOC}" +CSRF_TRUSTED_ORIGINS = [BASE_URL] + +USER_AGENT = f"BookWyrm (BookWyrm/{VERSION}; +{BASE_URL})" + +# Storage + USE_S3 = env.bool("USE_S3", False) USE_AZURE = env.bool("USE_AZURE", False) +S3_SIGNED_URL_EXPIRY = env.int("S3_SIGNED_URL_EXPIRY", 900) if USE_S3: # AWS settings @@ -386,44 +386,124 @@ if USE_S3: AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL", None) AWS_DEFAULT_ACL = "public-read" AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} + AWS_S3_URL_PROTOCOL = env("AWS_S3_URL_PROTOCOL", f"{PROTOCOL}:") + # Storages + STORAGES = { + "default": { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + "location": "images", + "default_acl": "public-read", + "file_overwrite": False, + }, + }, + "staticfiles": { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + "location": "static", + "default_acl": "public-read", + }, + }, + "sass_processor": { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + "location": "static", + "default_acl": "public-read", + }, + }, + "exports": { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + "location": "images", + "default_acl": None, + "file_overwrite": False, + }, + }, + } # S3 Static settings STATIC_LOCATION = "static" - STATIC_URL = f"{PROTOCOL}://{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/" - STATICFILES_STORAGE = "bookwyrm.storage_backends.StaticStorage" + STATIC_URL = f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/" + STATIC_FULL_URL = STATIC_URL # S3 Media settings MEDIA_LOCATION = "images" - MEDIA_URL = f"{PROTOCOL}://{AWS_S3_CUSTOM_DOMAIN}/{MEDIA_LOCATION}/" + MEDIA_URL = f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}/{MEDIA_LOCATION}/" MEDIA_FULL_URL = MEDIA_URL - STATIC_FULL_URL = STATIC_URL - DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage" - CSP_DEFAULT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS - CSP_SCRIPT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS + # Content Security Policy + CSP_DEFAULT_SRC = [ + "'self'", + f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}" + if AWS_S3_CUSTOM_DOMAIN + else None, + ] + CSP_ADDITIONAL_HOSTS + CSP_SCRIPT_SRC = [ + "'self'", + f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}" + if AWS_S3_CUSTOM_DOMAIN + else None, + ] + CSP_ADDITIONAL_HOSTS elif USE_AZURE: + # Azure settings AZURE_ACCOUNT_NAME = env("AZURE_ACCOUNT_NAME") AZURE_ACCOUNT_KEY = env("AZURE_ACCOUNT_KEY") AZURE_CONTAINER = env("AZURE_CONTAINER") AZURE_CUSTOM_DOMAIN = env("AZURE_CUSTOM_DOMAIN") + # Storages + STORAGES = { + "default": { + "BACKEND": "storages.backends.azure_storage.AzureStorage", + "OPTIONS": { + "location": "images", + "overwrite_files": False, + }, + }, + "staticfiles": { + "BACKEND": "storages.backends.azure_storage.AzureStorage", + "OPTIONS": { + "location": "static", + }, + }, + "exports": { + "BACKEND": None, # not implemented yet + }, + } # Azure Static settings STATIC_LOCATION = "static" STATIC_URL = ( f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{STATIC_LOCATION}/" ) - STATICFILES_STORAGE = "bookwyrm.storage_backends.AzureStaticStorage" + STATIC_FULL_URL = STATIC_URL # Azure Media settings MEDIA_LOCATION = "images" MEDIA_URL = ( f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{MEDIA_LOCATION}/" ) MEDIA_FULL_URL = MEDIA_URL - STATIC_FULL_URL = STATIC_URL - DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.AzureImagesStorage" + # Content Security Policy CSP_DEFAULT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS CSP_SCRIPT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS else: + # Storages + STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, + "exports": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + "OPTIONS": { + "location": "exports", + }, + }, + } + # Static settings STATIC_URL = "/static/" + STATIC_FULL_URL = BASE_URL + STATIC_URL + # Media settings MEDIA_URL = "/images/" - MEDIA_FULL_URL = f"{PROTOCOL}://{DOMAIN}{MEDIA_URL}" - STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}" + MEDIA_FULL_URL = BASE_URL + MEDIA_URL + # Content Security Policy CSP_DEFAULT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS CSP_SCRIPT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py index 08780b731..f59367b51 100644 --- a/bookwyrm/signatures.py +++ b/bookwyrm/signatures.py @@ -6,7 +6,7 @@ from base64 import b64encode, b64decode from Crypto import Random from Crypto.PublicKey import RSA -from Crypto.Signature import pkcs1_15 # pylint: disable=no-name-in-module +from Crypto.Signature import pkcs1_15 from Crypto.Hash import SHA256 MAX_SIGNATURE_AGE = 300 @@ -84,7 +84,6 @@ class Signature: self.headers = headers self.signature = signature - # pylint: disable=invalid-name @classmethod def parse(cls, request): """extract and parse a signature from an http request""" diff --git a/bookwyrm/static/js/bookwyrm.js b/bookwyrm/static/js/bookwyrm.js index a2351a98c..5a5e5f68e 100644 --- a/bookwyrm/static/js/bookwyrm.js +++ b/bookwyrm/static/js/bookwyrm.js @@ -14,6 +14,10 @@ let BookWyrm = new (class { .querySelectorAll("[data-controls]") .forEach((button) => button.addEventListener("click", this.toggleAction.bind(this))); + document + .querySelectorAll("[data-disappear]") + .forEach((button) => button.addEventListener("click", this.hideSelf.bind(this))); + document .querySelectorAll(".interaction") .forEach((button) => button.addEventListener("submit", this.interact.bind(this))); @@ -181,6 +185,18 @@ let BookWyrm = new (class { this.addRemoveClass(visible, "is-hidden", true); } + /** + * Hide the element you just clicked + * + * @param {Event} event + * @return {undefined} + */ + hideSelf(event) { + let trigger = event.currentTarget; + + this.addRemoveClass(trigger, "is-hidden", true); + } + /** * Execute actions on targets based on triggers. * diff --git a/bookwyrm/storage_backends.py b/bookwyrm/storage_backends.py deleted file mode 100644 index 6dd9f522c..000000000 --- a/bookwyrm/storage_backends.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Handles backends for storages""" -import os -from tempfile import SpooledTemporaryFile -from storages.backends.s3boto3 import S3Boto3Storage -from storages.backends.azure_storage import AzureStorage - - -class StaticStorage(S3Boto3Storage): # pylint: disable=abstract-method - """Storage class for Static contents""" - - location = "static" - default_acl = "public-read" - - -class ImagesStorage(S3Boto3Storage): # pylint: disable=abstract-method - """Storage class for Image files""" - - location = "images" - default_acl = "public-read" - file_overwrite = False - - """ - This is our custom version of S3Boto3Storage that fixes a bug in - boto3 where the passed in file is closed upon upload. - From: - https://github.com/matthewwithanm/django-imagekit/issues/391#issuecomment-275367006 - https://github.com/boto/boto3/issues/929 - https://github.com/matthewwithanm/django-imagekit/issues/391 - """ - - def _save(self, name, content): - """ - We create a clone of the content file as when this is passed to - boto3 it wrongly closes the file upon upload where as the storage - backend expects it to still be open - """ - # Seek our content back to the start - content.seek(0, os.SEEK_SET) - - # Create a temporary file that will write to disk after a specified - # size. This file will be automatically deleted when closed by - # boto3 or after exiting the `with` statement if the boto3 is fixed - with SpooledTemporaryFile() as content_autoclose: - - # Write our original content into our copy that will be closed by boto3 - content_autoclose.write(content.read()) - - # Upload the object which will auto close the - # content_autoclose instance - return super()._save(name, content_autoclose) - - -class AzureStaticStorage(AzureStorage): # pylint: disable=abstract-method - """Storage class for Static contents""" - - location = "static" - - -class AzureImagesStorage(AzureStorage): # pylint: disable=abstract-method - """Storage class for Image files""" - - location = "images" - overwrite_files = False diff --git a/bookwyrm/suggested_users.py b/bookwyrm/suggested_users.py index a13ee97fd..3e1cf17bd 100644 --- a/bookwyrm/suggested_users.py +++ b/bookwyrm/suggested_users.py @@ -34,7 +34,6 @@ class SuggestedUsers(RedisStore): def get_counts_from_rank(self, rank): # pylint: disable=no-self-use """calculate mutuals count and shared books count from rank""" - # pylint: disable=c-extension-no-member return { "mutuals": math.floor(rank), # "shared_books": int(1 / (-1 * (rank % 1 - 1))) - 1, @@ -128,7 +127,6 @@ def get_annotated_users(viewer, *args, **kwargs): ), distinct=True, ), - # pylint: disable=line-too-long # shared_books=Count( # "shelfbook", # filter=Q( @@ -202,7 +200,7 @@ def update_suggestions_on_unfollow(sender, instance, **kwargs): @receiver(signals.post_save, sender=models.User) -# pylint: disable=unused-argument, too-many-arguments +# pylint: disable=unused-argument def update_user(sender, instance, created, update_fields=None, **kwargs): """an updated user, neat""" # a new user is found, create suggestions for them diff --git a/bookwyrm/templates/book/file_links/verification_modal.html b/bookwyrm/templates/book/file_links/verification_modal.html index 7a9e41ad2..4f2f96a4a 100644 --- a/bookwyrm/templates/book/file_links/verification_modal.html +++ b/bookwyrm/templates/book/file_links/verification_modal.html @@ -21,9 +21,10 @@ Is that where you'd like to go?
{% trans "Report spam" %}
+{% endif %} {% trans "Continue" %} -{% endif %} + {% endblock %} diff --git a/bookwyrm/templates/compose.html b/bookwyrm/templates/compose.html index ca0d8296a..112088648 100644 --- a/bookwyrm/templates/compose.html +++ b/bookwyrm/templates/compose.html @@ -2,10 +2,31 @@ {% load i18n %} {% load utilities %} -{% block title %}{% trans "Edit status" %}{% endblock %} +{% block title %} + {% if draft.status_type == "Review" %} + {% trans "Edit review" %} + {% elif draft.status_type == "Quotation" %} + {% trans "Edit quote" %} + {% elif draft.status_type == "Comment" %} + {% trans "Edit comment" %} + {% else %} + {% trans "Edit status" %} + {% endif %} +{% endblock %} + {% block content %}
-

{% trans "Edit status" %}

+

+ {% if draft.status_type == "Review" %} + {% trans "Edit review" %} + {% elif draft.status_type == "Quotation" %} + {% trans "Edit quote" %} + {% elif draft.status_type == "Comment" %} + {% trans "Edit comment" %} + {% else %} + {% trans "Edit status" %} + {% endif %} +

{% with 0|uuid as uuid %} diff --git a/bookwyrm/templates/email/html_layout.html b/bookwyrm/templates/email/html_layout.html index b9f88732f..467d6d6e5 100644 --- a/bookwyrm/templates/email/html_layout.html +++ b/bookwyrm/templates/email/html_layout.html @@ -2,10 +2,10 @@
- logo + logo
- {{ site_name }}
+ {{ site_name }}
{{ domain }}
@@ -18,9 +18,9 @@
-

{% blocktrans %}BookWyrm hosted on {{ site_name }}{% endblocktrans %}

+

{% blocktrans %}BookWyrm hosted on {{ site_name }}{% endblocktrans %}

{% if user %} -

{% trans "Email preference" %}

+

{% trans "Email preference" %}

{% endif %}
diff --git a/bookwyrm/templates/email/invite/html_content.html b/bookwyrm/templates/email/invite/html_content.html index adc993b7b..9d2cda364 100644 --- a/bookwyrm/templates/email/invite/html_content.html +++ b/bookwyrm/templates/email/invite/html_content.html @@ -12,6 +12,6 @@

{% url 'code-of-conduct' as coc_path %} {% url 'about' as about_path %} - {% blocktrans %}Learn more about {{ site_name }}.{% endblocktrans %} + {% blocktrans %}Learn more about {{ site_name }}.{% endblocktrans %}

{% endblock %} diff --git a/bookwyrm/templates/email/invite/text_content.html b/bookwyrm/templates/email/invite/text_content.html index 26dcd1720..05fe91456 100644 --- a/bookwyrm/templates/email/invite/text_content.html +++ b/bookwyrm/templates/email/invite/text_content.html @@ -5,6 +5,6 @@ {{ invite_link }} -{% blocktrans %}Learn more about {{ site_name }}:{% endblocktrans %} https://{{ domain }}{% url 'about' %} +{% blocktrans %}Learn more about {{ site_name }}:{% endblocktrans %} {{ base_url }}{% url 'about' %} {% endblock %} diff --git a/bookwyrm/templates/import/import.html b/bookwyrm/templates/import/import.html index 01014fa94..57a141b7e 100644 --- a/bookwyrm/templates/import/import.html +++ b/bookwyrm/templates/import/import.html @@ -69,6 +69,9 @@ + @@ -93,9 +96,14 @@ {% trans "Include reviews" %} +
+ +
{% include 'snippets/privacy_select.html' with no_label=True privacy_uuid="import" %}
diff --git a/bookwyrm/templates/opensearch.xml b/bookwyrm/templates/opensearch.xml index fd5c8f231..980ca5604 100644 --- a/bookwyrm/templates/opensearch.xml +++ b/bookwyrm/templates/opensearch.xml @@ -10,6 +10,6 @@ {{ image }} diff --git a/bookwyrm/templates/ostatus/remote_follow.html b/bookwyrm/templates/ostatus/remote_follow.html index ca47c529b..c54e6bfa7 100644 --- a/bookwyrm/templates/ostatus/remote_follow.html +++ b/bookwyrm/templates/ostatus/remote_follow.html @@ -3,8 +3,10 @@ {% load utilities %} {% block heading %} +{% block title %} {% blocktrans with username=user.localname sitename=site.name %}Follow {{ username }} on the fediverse{% endblocktrans %} {% endblock %} +{% endblock %} {% block content %}
diff --git a/bookwyrm/templates/ostatus/success.html b/bookwyrm/templates/ostatus/success.html index 66577e83f..c2b8edd75 100644 --- a/bookwyrm/templates/ostatus/success.html +++ b/bookwyrm/templates/ostatus/success.html @@ -2,6 +2,10 @@ {% load i18n %} {% load utilities %} +{% block title %} +{% blocktrans with display_name=user.display_name %}You are now following {{ display_name }}!{% endblocktrans %} +{% endblock %} + {% block content %}
diff --git a/bookwyrm/templates/preferences/edit_user.html b/bookwyrm/templates/preferences/edit_user.html index f2b14babf..c10bc1ee4 100644 --- a/bookwyrm/templates/preferences/edit_user.html +++ b/bookwyrm/templates/preferences/edit_user.html @@ -69,6 +69,12 @@ {% trans "Show reading goal prompt in feed" %}
+
+ +