From 2f4010b93bfcd8cfa0dfb24505369739d9ad0b65 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 20:23:25 +0100 Subject: [PATCH 01/56] Upgrade Django to 4.2 - https://docs.djangoproject.com/en/5.0/releases/4.0/ - https://docs.djangoproject.com/en/5.0/releases/4.1/ - https://docs.djangoproject.com/en/5.0/releases/4.2/ --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index df00f5806..1a974b167 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ boto3==1.26.57 bw-file-resubmit==0.6.0rc2 celery==5.3.1 colorthief==0.2.1 -Django==3.2.25 +Django==4.2.11 django-celery-beat==2.5.0 django-compressor==4.4 django-csp==3.7 From 45bd67cb04fe2cc00c50fa2671eb05645e2260f4 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 20:26:55 +0100 Subject: [PATCH 02/56] Add migration resulting from Django 4.2 upgrade --- ...9_alter_userblocks_user_object_and_more.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 bookwyrm/migrations/0199_alter_userblocks_user_object_and_more.py diff --git a/bookwyrm/migrations/0199_alter_userblocks_user_object_and_more.py b/bookwyrm/migrations/0199_alter_userblocks_user_object_and_more.py new file mode 100644 index 000000000..bde1f25c1 --- /dev/null +++ b/bookwyrm/migrations/0199_alter_userblocks_user_object_and_more.py @@ -0,0 +1,70 @@ +# Generated by Django 4.2.11 on 2024-03-29 19:25 + +import bookwyrm.models.fields +from django.conf import settings +from django.db import migrations +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0198_book_search_vector_author_aliases"), + ] + + operations = [ + migrations.AlterField( + model_name="userblocks", + name="user_object", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_object", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="userblocks", + name="user_subject", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_subject", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="userfollowrequest", + name="user_object", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_object", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="userfollowrequest", + name="user_subject", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_subject", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="userfollows", + name="user_object", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_object", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="userfollows", + name="user_subject", + field=bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="%(class)s_user_subject", + to=settings.AUTH_USER_MODEL, + ), + ), + ] From 3349817a0b1bf52cb594816d901577d0646080af Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 20:37:01 +0100 Subject: [PATCH 03/56] settings.USE_L10N is deprecated --- bookwyrm/settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 438c41d2f..e90e07eb8 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -347,8 +347,6 @@ TIME_ZONE = "UTC" USE_I18N = True -USE_L10N = True - USE_TZ = True # Imagekit generated thumbnails From 47fdad9c8795062e96efa48a715582e6ade57979 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 20:44:01 +0100 Subject: [PATCH 04/56] Use new STORAGES setting --- bookwyrm/settings.py | 48 +++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index e90e07eb8..0ad403ce5 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -389,18 +389,26 @@ if USE_S3: AWS_DEFAULT_ACL = "public-read" AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} AWS_S3_URL_PROTOCOL = env("AWS_S3_URL_PROTOCOL", f"{PROTOCOL}:") + # Storages + STORAGES = { + "default": { + "BACKEND": "bookwyrm.storage_backends.ImagesStorage", + }, + "staticfiles": { + "BACKEND": "bookwyrm.storage_backends.StaticStorage", + }, + "exports": { + "BACKEND": "bookwyrm.storage_backends.ExportsS3Storage", + }, + } # S3 Static settings STATIC_LOCATION = "static" STATIC_URL = f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/" STATIC_FULL_URL = STATIC_URL - STATICFILES_STORAGE = "bookwyrm.storage_backends.StaticStorage" # S3 Media settings MEDIA_LOCATION = "images" MEDIA_URL = f"{AWS_S3_URL_PROTOCOL}//{AWS_S3_CUSTOM_DOMAIN}/{MEDIA_LOCATION}/" MEDIA_FULL_URL = MEDIA_URL - DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage" - # S3 Exports settings - EXPORTS_STORAGE = "bookwyrm.storage_backends.ExportsS3Storage" # Content Security Policy CSP_DEFAULT_SRC = [ "'self'", @@ -420,36 +428,52 @@ elif USE_AZURE: AZURE_ACCOUNT_KEY = env("AZURE_ACCOUNT_KEY") AZURE_CONTAINER = env("AZURE_CONTAINER") AZURE_CUSTOM_DOMAIN = env("AZURE_CUSTOM_DOMAIN") + # Storages + STORAGES = { + "default": { + "BACKEND": "bookwyrm.storage_backends.AzureImagesStorage", + }, + "staticfiles": { + "BACKEND": "bookwyrm.storage_backends.AzureStaticStorage", + }, + "exports": { + "BACKEND": None, # not implemented yet + }, + } # Azure Static settings STATIC_LOCATION = "static" STATIC_URL = ( f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{STATIC_LOCATION}/" ) STATIC_FULL_URL = STATIC_URL - STATICFILES_STORAGE = "bookwyrm.storage_backends.AzureStaticStorage" # Azure Media settings MEDIA_LOCATION = "images" MEDIA_URL = ( f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{MEDIA_LOCATION}/" ) MEDIA_FULL_URL = MEDIA_URL - DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.AzureImagesStorage" - # Azure Exports settings - EXPORTS_STORAGE = None # not implemented yet # Content Security Policy CSP_DEFAULT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS CSP_SCRIPT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS else: + # Storages + STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, + "exports": { + "BACKEND": "bookwyrm.storage_backends.ExportsFileStorage", + }, + } # Static settings STATIC_URL = "/static/" STATIC_FULL_URL = BASE_URL + STATIC_URL - STATICFILES_STORAGE = "django.contrib.staticfiles.storage.StaticFilesStorage" # Media settings MEDIA_URL = "/images/" MEDIA_FULL_URL = BASE_URL + MEDIA_URL - DEFAULT_FILE_STORAGE = "django.core.files.storage.FileSystemStorage" - # Exports settings - EXPORTS_STORAGE = "bookwyrm.storage_backends.ExportsFileStorage" # Content Security Policy CSP_DEFAULT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS CSP_SCRIPT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS From 0d621b68e0ff2685e3ef0711d46156a71000918f Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 21:12:59 +0100 Subject: [PATCH 05/56] Reorder operations in save() overrides Accessing many-to-many relations before saving is no longer allowed. Reorder all operations consistently: 1. Validations 2. Modify own fields 3. Perform save by calling super().save() 4. Modify related objects and clear caches Especially clearing caches should be done after actually saving, otherwise the old data can be re-added immediately by another request before the new data is written. --- bookwyrm/models/author.py | 2 +- bookwyrm/models/book.py | 19 ++++++++++--------- bookwyrm/models/readthrough.py | 6 ++++-- bookwyrm/models/relationship.py | 6 ++++-- bookwyrm/models/shelf.py | 8 ++++++-- bookwyrm/models/site.py | 8 +++++--- bookwyrm/models/status.py | 3 ++- 7 files changed, 32 insertions(+), 20 deletions(-) diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index abe78dafb..8ea1858fd 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -50,7 +50,7 @@ class Author(BookDataModel): if self.isni: self.isni = re.sub(r"\s", "", self.isni) - return super().save(*args, **kwargs) + super().save(*args, **kwargs) @property def isni_link(self): diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 8e957b717..6fc447228 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -103,7 +103,7 @@ class BookDataModel(ObjectMixin, BookWyrmModel): else: self.origin_id = self.remote_id self.remote_id = None - return super().save(*args, **kwargs) + super().save(*args, **kwargs) # pylint: disable=arguments-differ def broadcast(self, activity, sender, software="bookwyrm", **kwargs): @@ -323,7 +323,7 @@ class Book(BookDataModel): if not isinstance(self, (Edition, Work)): raise ValueError("Books should be added as Editions or Works") - return super().save(*args, **kwargs) + super().save(*args, **kwargs) def get_remote_id(self): """editions and works both use "book" instead of model_name""" @@ -400,10 +400,11 @@ class Work(OrderedCollectionPageMixin, Book): def save(self, *args, **kwargs): """set some fields on the edition object""" + super().save(*args, **kwargs) + # set rank for edition in self.editions.all(): edition.save() - return super().save(*args, **kwargs) @property def default_edition(self): @@ -526,16 +527,16 @@ class Edition(Book): # set rank self.edition_rank = self.get_rank() - # clear author cache - if self.id: - for author_id in self.authors.values_list("id", flat=True): - cache.delete(f"author-books-{author_id}") - # Create sort title by removing articles from title if self.sort_title in [None, ""]: self.sort_title = self.guess_sort_title() - return super().save(*args, **kwargs) + super().save(*args, **kwargs) + + # clear author cache + if self.id: + for author_id in self.authors.values_list("id", flat=True): + cache.delete(f"author-books-{author_id}") @transaction.atomic def repair(self): diff --git a/bookwyrm/models/readthrough.py b/bookwyrm/models/readthrough.py index 4911c715b..910b2a7a9 100644 --- a/bookwyrm/models/readthrough.py +++ b/bookwyrm/models/readthrough.py @@ -32,13 +32,15 @@ class ReadThrough(BookWyrmModel): def save(self, *args, **kwargs): """update user active time""" - cache.delete(f"latest_read_through-{self.user_id}-{self.book_id}") - self.user.update_active_date() # an active readthrough must have an unset finish date if self.finish_date or self.stopped_date: self.is_active = False + super().save(*args, **kwargs) + cache.delete(f"latest_read_through-{self.user_id}-{self.book_id}") + self.user.update_active_date() + def create_update(self): """add update to the readthrough""" if self.progress: diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index 3386a02dc..745ff78b6 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -38,14 +38,16 @@ class UserRelationship(BookWyrmModel): def save(self, *args, **kwargs): """clear the template cache""" - clear_cache(self.user_subject, self.user_object) super().save(*args, **kwargs) + clear_cache(self.user_subject, self.user_object) + def delete(self, *args, **kwargs): """clear the template cache""" - clear_cache(self.user_subject, self.user_object) super().delete(*args, **kwargs) + clear_cache(self.user_subject, self.user_object) + class Meta: """relationships should be unique""" diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py index 4b4e3cd8d..77c2d26d9 100644 --- a/bookwyrm/models/shelf.py +++ b/bookwyrm/models/shelf.py @@ -44,6 +44,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): """set the identifier""" super().save(*args, priority=priority, **kwargs) if not self.identifier: + # this needs the auto increment ID from the save() above self.identifier = self.get_identifier() super().save(*args, **kwargs, broadcast=False) @@ -103,7 +104,11 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel): def save(self, *args, priority=BROADCAST, **kwargs): if not self.user: self.user = self.shelf.user - if self.id and self.user.local: + + is_update = self.id is not None + super().save(*args, priority=priority, **kwargs) + + if is_update and self.user.local: # remove all caches related to all editions of this book cache.delete_many( [ @@ -111,7 +116,6 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel): for book in self.book.parent_work.editions.all() ] ) - super().save(*args, priority=priority, **kwargs) def delete(self, *args, **kwargs): if self.id and self.user.local: diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index 36e6bb128..89d6ef395 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -139,13 +139,15 @@ class SiteSettings(SiteModel): def save(self, *args, **kwargs): """if require_confirm_email is disabled, make sure no users are pending, if enabled, make sure invite_question_text is not empty""" + if not self.invite_question_text: + self.invite_question_text = "What is your favourite book?" + + super().save(*args, **kwargs) + if not self.require_confirm_email: User.objects.filter(is_active=False, deactivation_reason="pending").update( is_active=True, deactivation_reason=None ) - if not self.invite_question_text: - self.invite_question_text = "What is your favourite book?" - super().save(*args, **kwargs) class Theme(SiteModel): diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index d0c1e639b..5b953d077 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -459,9 +459,10 @@ class Review(BookStatus): def save(self, *args, **kwargs): """clear rating caches""" + super().save(*args, **kwargs) + if self.book.parent_work: cache.delete(f"book-rating-{self.book.parent_work.id}") - super().save(*args, **kwargs) class ReviewRating(Review): From 92a94d2fdcb06950681cf3619cce805eceb2e9ce Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 21:38:48 +0100 Subject: [PATCH 06/56] django.utils.timezone.utc alias is deprecated --- .../tests/activitystreams/test_abstractstream.py | 3 +-- bookwyrm/tests/activitystreams/test_signals.py | 8 ++++---- bookwyrm/tests/models/test_import_model.py | 2 +- bookwyrm/tests/templatetags/test_status_display.py | 14 +++++++++----- bookwyrm/tests/test_book_search.py | 3 ++- bookwyrm/tests/test_partial_date.py | 2 +- bookwyrm/tests/views/imports/test_import.py | 4 ++-- bookwyrm/tests/views/test_annual_summary.py | 4 ++-- bookwyrm/tests/views/test_readthrough.py | 3 +-- 9 files changed, 23 insertions(+), 20 deletions(-) diff --git a/bookwyrm/tests/activitystreams/test_abstractstream.py b/bookwyrm/tests/activitystreams/test_abstractstream.py index 3a95e2efa..addbd00f7 100644 --- a/bookwyrm/tests/activitystreams/test_abstractstream.py +++ b/bookwyrm/tests/activitystreams/test_abstractstream.py @@ -1,8 +1,7 @@ """ testing activitystreams """ -from datetime import datetime +from datetime import datetime, timezone from unittest.mock import patch from django.test import TestCase -from django.utils import timezone from bookwyrm import activitystreams, models diff --git a/bookwyrm/tests/activitystreams/test_signals.py b/bookwyrm/tests/activitystreams/test_signals.py index 77ac68e71..42bf26289 100644 --- a/bookwyrm/tests/activitystreams/test_signals.py +++ b/bookwyrm/tests/activitystreams/test_signals.py @@ -1,5 +1,5 @@ """ testing activitystreams """ -from datetime import datetime, timedelta +import datetime from unittest.mock import patch from django.test import TestCase @@ -71,8 +71,8 @@ class ActivitystreamsSignals(TestCase): user=self.remote_user, content="hi", privacy="public", - created_date=datetime(2022, 5, 16, tzinfo=timezone.utc), - published_date=datetime(2022, 5, 14, tzinfo=timezone.utc), + created_date=datetime.datetime(2022, 5, 16, tzinfo=datetime.timezone.utc), + published_date=datetime.datetime(2022, 5, 14, tzinfo=datetime.timezone.utc), ) with patch("bookwyrm.activitystreams.add_status_task.apply_async") as mock: activitystreams.add_status_on_create_command(models.Status, status, False) @@ -87,7 +87,7 @@ class ActivitystreamsSignals(TestCase): user=self.remote_user, content="hi", privacy="public", - published_date=timezone.now() - timedelta(days=1), + published_date=timezone.now() - datetime.timedelta(days=1), ) with patch("bookwyrm.activitystreams.add_status_task.apply_async") as mock: activitystreams.add_status_on_create_command(models.Status, status, False) diff --git a/bookwyrm/tests/models/test_import_model.py b/bookwyrm/tests/models/test_import_model.py index e591c33e8..5445a79db 100644 --- a/bookwyrm/tests/models/test_import_model.py +++ b/bookwyrm/tests/models/test_import_model.py @@ -1,10 +1,10 @@ """ testing models """ import datetime +from datetime import timezone import json import pathlib from unittest.mock import patch -from django.utils import timezone from django.test import TestCase import responses diff --git a/bookwyrm/tests/templatetags/test_status_display.py b/bookwyrm/tests/templatetags/test_status_display.py index 762f14ea8..338c30481 100644 --- a/bookwyrm/tests/templatetags/test_status_display.py +++ b/bookwyrm/tests/templatetags/test_status_display.py @@ -1,5 +1,5 @@ """ style fixes and lookups for templates """ -from datetime import datetime +import datetime from unittest.mock import patch from django.test import TestCase @@ -95,14 +95,18 @@ class StatusDisplayTags(TestCase): def test_get_published_date(self, *_): """date formatting""" - date = datetime(2020, 1, 1, 0, 0, tzinfo=timezone.utc) + date = datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) with patch("django.utils.timezone.now") as timezone_mock: - timezone_mock.return_value = datetime(2022, 1, 1, 0, 0, tzinfo=timezone.utc) + timezone_mock.return_value = datetime.datetime( + 2022, 1, 1, 0, 0, tzinfo=datetime.timezone.utc + ) result = status_display.get_published_date(date) self.assertEqual(result, "Jan. 1, 2020") - date = datetime(2022, 1, 1, 0, 0, tzinfo=timezone.utc) + date = datetime.datetime(2022, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) with patch("django.utils.timezone.now") as timezone_mock: - timezone_mock.return_value = datetime(2022, 1, 8, 0, 0, tzinfo=timezone.utc) + timezone_mock.return_value = datetime.datetime( + 2022, 1, 8, 0, 0, tzinfo=datetime.timezone.utc + ) result = status_display.get_published_date(date) self.assertEqual(result, "Jan 1") diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index 3673b9579..cc9a00154 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -1,8 +1,9 @@ """ test searching for books """ import datetime +from datetime import timezone + from django.db import connection from django.test import TestCase -from django.utils import timezone from bookwyrm import book_search, models from bookwyrm.connectors.abstract_connector import AbstractMinimalConnector diff --git a/bookwyrm/tests/test_partial_date.py b/bookwyrm/tests/test_partial_date.py index 364d00933..12d8c768d 100644 --- a/bookwyrm/tests/test_partial_date.py +++ b/bookwyrm/tests/test_partial_date.py @@ -1,10 +1,10 @@ """ test partial_date module """ import datetime +from datetime import timezone import unittest from django.core.exceptions import ValidationError -from django.utils import timezone from django.utils import translation from bookwyrm.utils import partial_date diff --git a/bookwyrm/tests/views/imports/test_import.py b/bookwyrm/tests/views/imports/test_import.py index f694b7bf5..763fcc19f 100644 --- a/bookwyrm/tests/views/imports/test_import.py +++ b/bookwyrm/tests/views/imports/test_import.py @@ -123,8 +123,8 @@ class ImportViews(TestCase): """Give people a sense of the timing""" models.ImportJob.objects.create( user=self.local_user, - created_date=datetime.datetime(2000, 1, 1), - updated_date=datetime.datetime(2001, 1, 1), + created_date=datetime.datetime(2000, 1, 1, tzinfo=datetime.timezone.utc), + updated_date=datetime.datetime(2001, 1, 1, tzinfo=datetime.timezone.utc), status="complete", complete=True, mappings={}, diff --git a/bookwyrm/tests/views/test_annual_summary.py b/bookwyrm/tests/views/test_annual_summary.py index db8389fc6..f5bd60085 100644 --- a/bookwyrm/tests/views/test_annual_summary.py +++ b/bookwyrm/tests/views/test_annual_summary.py @@ -1,5 +1,5 @@ """testing the annual summary page""" -from datetime import datetime +import datetime from unittest.mock import patch import pytz @@ -15,7 +15,7 @@ from bookwyrm.tests.validate_html import validate_html def make_date(*args): """helper function to easily generate a date obj""" - return datetime(*args, tzinfo=pytz.UTC) + return datetime.datetime(*args, tzinfo=pytz.UTC) class AnnualSummary(TestCase): diff --git a/bookwyrm/tests/views/test_readthrough.py b/bookwyrm/tests/views/test_readthrough.py index c71ee6c58..e85d4e6a4 100644 --- a/bookwyrm/tests/views/test_readthrough.py +++ b/bookwyrm/tests/views/test_readthrough.py @@ -1,8 +1,7 @@ """ tests updating reading progress """ -from datetime import datetime +from datetime import datetime, timezone from unittest.mock import patch from django.test import TestCase, Client -from django.utils import timezone from bookwyrm import models From 984d7fb7d8c9f63071a4f18b271c6d7fa78eed54 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 21:55:03 +0100 Subject: [PATCH 07/56] Update pytest-django to 4.8.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1a974b167..b0187502f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,7 +51,7 @@ mypy==1.5.1 pylint==2.15.0 pytest==6.2.5 pytest-cov==2.10.1 -pytest-django==4.1.0 +pytest-django==4.8.0 pytest-env==0.6.2 pytest-xdist==2.3.0 pytidylib==0.3.2 From 0007c86a2c43bb7049ff8b19ac3bf43bef01a081 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 21:56:39 +0100 Subject: [PATCH 08/56] Update environs to 11.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b0187502f..c14277629 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ django-redis==5.2.0 django-sass-processor==1.2.2 django-storages==1.13.2 django-storages[azure] -environs==9.5.0 +environs==11.0.0 flower==2.0.1 grpcio==1.57.0 # Not a direct dependency, pinned to get a security fix libsass==0.22.0 From cfcb8732357c767392c349b4f100b8db8558c205 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 21:58:49 +0100 Subject: [PATCH 09/56] Update pytest-cov to 5.0.0 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c14277629..cfda1dfc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,8 +49,8 @@ celery-types==0.18.0 django-stubs[compatible-mypy]==4.2.4 mypy==1.5.1 pylint==2.15.0 -pytest==6.2.5 -pytest-cov==2.10.1 +pytest==8.0.2 +pytest-cov==5.0.0 pytest-django==4.8.0 pytest-env==0.6.2 pytest-xdist==2.3.0 From 4fa823e8df275a959654a54f82a8109e21793907 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:13:09 +0100 Subject: [PATCH 10/56] Update django-storages to 1.14.2 The problem that boto3 closes files has been worked around in django-storages. --- bookwyrm/storage_backends.py | 40 +++------------------------- bookwyrm/views/preferences/export.py | 36 ++++++++++++------------- requirements.txt | 2 +- 3 files changed, 23 insertions(+), 55 deletions(-) diff --git a/bookwyrm/storage_backends.py b/bookwyrm/storage_backends.py index 87c29ae70..1695daa67 100644 --- a/bookwyrm/storage_backends.py +++ b/bookwyrm/storage_backends.py @@ -1,55 +1,23 @@ """Handles backends for storages""" -import os -from tempfile import SpooledTemporaryFile from django.core.files.storage import FileSystemStorage -from storages.backends.s3boto3 import S3Boto3Storage +from storages.backends.s3 import S3Storage from storages.backends.azure_storage import AzureStorage -class StaticStorage(S3Boto3Storage): # pylint: disable=abstract-method +class StaticStorage(S3Storage): # pylint: disable=abstract-method """Storage class for Static contents""" location = "static" default_acl = "public-read" -class ImagesStorage(S3Boto3Storage): # pylint: disable=abstract-method +class ImagesStorage(S3Storage): # pylint: disable=abstract-method """Storage class for Image files""" location = "images" default_acl = "public-read" file_overwrite = False - """ - This is our custom version of S3Boto3Storage that fixes a bug in - boto3 where the passed in file is closed upon upload. - From: - https://github.com/matthewwithanm/django-imagekit/issues/391#issuecomment-275367006 - https://github.com/boto/boto3/issues/929 - https://github.com/matthewwithanm/django-imagekit/issues/391 - """ - - def _save(self, name, content): - """ - We create a clone of the content file as when this is passed to - boto3 it wrongly closes the file upon upload where as the storage - backend expects it to still be open - """ - # Seek our content back to the start - content.seek(0, os.SEEK_SET) - - # Create a temporary file that will write to disk after a specified - # size. This file will be automatically deleted when closed by - # boto3 or after exiting the `with` statement if the boto3 is fixed - with SpooledTemporaryFile() as content_autoclose: - - # Write our original content into our copy that will be closed by boto3 - content_autoclose.write(content.read()) - - # Upload the object which will auto close the - # content_autoclose instance - return super()._save(name, content_autoclose) - class AzureStaticStorage(AzureStorage): # pylint: disable=abstract-method """Storage class for Static contents""" @@ -71,7 +39,7 @@ class ExportsFileStorage(FileSystemStorage): # pylint: disable=abstract-method overwrite_files = False -class ExportsS3Storage(S3Boto3Storage): # pylint: disable=abstract-method +class ExportsS3Storage(S3Storage): # pylint: disable=abstract-method """Storage class for exports contents with S3""" location = "exports" diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index de243586d..58c77b14c 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -14,9 +14,9 @@ from django.urls import reverse from django.utils.decorators import method_decorator from django.shortcuts import redirect -from storages.backends.s3boto3 import S3Boto3Storage +from storages.backends.s3 import S3Storage -from bookwyrm import models, storage_backends +from bookwyrm import models from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from bookwyrm import settings @@ -220,17 +220,16 @@ class ExportUser(View): class ExportArchive(View): """Serve the archive file""" - # TODO: how do we serve s3 files? def get(self, request, archive_id): """download user export file""" export = BookwyrmExportJob.objects.get(task_id=archive_id, user=request.user) - if isinstance(export.export_data.storage, storage_backends.ExportsS3Storage): + if settings.USE_S3: # make custom_domain None so we can sign the url # see https://github.com/jschneier/django-storages/issues/944 - storage = S3Boto3Storage(querystring_auth=True, custom_domain=None) + storage = S3Storage(querystring_auth=True, custom_domain=None) try: - url = S3Boto3Storage.url( + url = S3Storage.url( storage, f"/exports/{export.task_id}.tar.gz", expire=settings.S3_SIGNED_URL_EXPIRY, @@ -239,16 +238,17 @@ class ExportArchive(View): raise Http404() return redirect(url) - if isinstance(export.export_data.storage, storage_backends.ExportsFileStorage): - try: - return HttpResponse( - export.export_data, - content_type="application/gzip", - headers={ - "Content-Disposition": 'attachment; filename="bookwyrm-account-export.tar.gz"' # pylint: disable=line-too-long - }, - ) - except FileNotFoundError: - raise Http404() + if settings.USE_AZURE: + # not implemented + return HttpResponseServerError() - return HttpResponseServerError() + try: + return HttpResponse( + export.export_data, + content_type="application/gzip", + headers={ + "Content-Disposition": 'attachment; filename="bookwyrm-account-export.tar.gz"' # pylint: disable=line-too-long + }, + ) + except FileNotFoundError: + raise Http404() diff --git a/requirements.txt b/requirements.txt index cfda1dfc5..c3b33f94b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ django-oauth-toolkit==2.3.0 django-pgtrigger==4.11.0 django-redis==5.2.0 django-sass-processor==1.2.2 -django-storages==1.13.2 +django-storages==1.14.2 django-storages[azure] environs==11.0.0 flower==2.0.1 From b5ef9f62410e86f29ca3a22b6e756ad95c3a1ec1 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:22:33 +0100 Subject: [PATCH 11/56] Configure STORAGES using OPTIONS instead of subclassing --- .../migrations/0193_auto_20240128_0249.py | 4 +- bookwyrm/models/bookwyrm_export_job.py | 11 ++--- bookwyrm/settings.py | 36 +++++++++++--- bookwyrm/storage_backends.py | 47 ------------------- 4 files changed, 37 insertions(+), 61 deletions(-) delete mode 100644 bookwyrm/storage_backends.py diff --git a/bookwyrm/migrations/0193_auto_20240128_0249.py b/bookwyrm/migrations/0193_auto_20240128_0249.py index c1c0527b9..82e32ee48 100644 --- a/bookwyrm/migrations/0193_auto_20240128_0249.py +++ b/bookwyrm/migrations/0193_auto_20240128_0249.py @@ -1,9 +1,9 @@ # Generated by Django 3.2.23 on 2024-01-28 02:49 -import bookwyrm.storage_backends import django.core.serializers.json from django.db import migrations, models import django.db.models.deletion +from django.core.files.storage import storages class Migration(migrations.Migration): @@ -30,7 +30,7 @@ class Migration(migrations.Migration): name="export_data", field=models.FileField( null=True, - storage=bookwyrm.storage_backends.ExportsFileStorage, + storage=storages["exports"], upload_to="", ), ), diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index da79de6a8..f355c86a4 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -10,9 +10,9 @@ from django.db.models import BooleanField, FileField, JSONField from django.db.models import Q from django.core.serializers.json import DjangoJSONEncoder from django.core.files.base import ContentFile -from django.utils.module_loading import import_string +from django.core.files.storage import storages -from bookwyrm import settings, storage_backends +from bookwyrm import settings from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, ListItem from bookwyrm.models import Review, Comment, Quotation @@ -35,8 +35,7 @@ class BookwyrmAwsSession(BotoSession): def select_exports_storage(): """callable to allow for dependency on runtime configuration""" - cls = import_string(settings.EXPORTS_STORAGE) - return cls() + return storages["exports"] class BookwyrmExportJob(ParentJob): @@ -116,7 +115,7 @@ def create_archive_task(job_id): if settings.USE_S3: # Storage for writing temporary files - exports_storage = storage_backends.ExportsS3Storage() + exports_storage = storages["exports"] # Handle for creating the final archive s3_tar = S3Tar( @@ -136,7 +135,7 @@ def create_archive_task(job_id): ) # Add images to TAR - images_storage = storage_backends.ImagesStorage() + images_storage = storages["default"] if user.avatar: add_file_to_s3_tar(s3_tar, images_storage, user.avatar) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 0ad403ce5..61b45c423 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -392,13 +392,27 @@ if USE_S3: # Storages STORAGES = { "default": { - "BACKEND": "bookwyrm.storage_backends.ImagesStorage", + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + "location": "images", + "default_acl": "public-read", + "file_overwrite": False, + }, }, "staticfiles": { - "BACKEND": "bookwyrm.storage_backends.StaticStorage", + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + "location": "static", + "default_acl": "public-read", + }, }, "exports": { - "BACKEND": "bookwyrm.storage_backends.ExportsS3Storage", + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + "location": "images", + "default_acl": None, + "file_overwrite": False, + }, }, } # S3 Static settings @@ -431,10 +445,17 @@ elif USE_AZURE: # Storages STORAGES = { "default": { - "BACKEND": "bookwyrm.storage_backends.AzureImagesStorage", + "BACKEND": "storages.backends.azure_storage.AzureStorage", + "OPTIONS": { + "location": "images", + "overwrite_files": False, + }, }, "staticfiles": { - "BACKEND": "bookwyrm.storage_backends.AzureStaticStorage", + "BACKEND": "storages.backends.azure_storage.AzureStorage", + "OPTIONS": { + "location": "static", + }, }, "exports": { "BACKEND": None, # not implemented yet @@ -465,7 +486,10 @@ else: "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", }, "exports": { - "BACKEND": "bookwyrm.storage_backends.ExportsFileStorage", + "BACKEND": "django.core.files.storage.FileSystemStorage", + "OPTIONS": { + "location": "exports", + }, }, } # Static settings diff --git a/bookwyrm/storage_backends.py b/bookwyrm/storage_backends.py deleted file mode 100644 index 1695daa67..000000000 --- a/bookwyrm/storage_backends.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Handles backends for storages""" -from django.core.files.storage import FileSystemStorage -from storages.backends.s3 import S3Storage -from storages.backends.azure_storage import AzureStorage - - -class StaticStorage(S3Storage): # pylint: disable=abstract-method - """Storage class for Static contents""" - - location = "static" - default_acl = "public-read" - - -class ImagesStorage(S3Storage): # pylint: disable=abstract-method - """Storage class for Image files""" - - location = "images" - default_acl = "public-read" - file_overwrite = False - - -class AzureStaticStorage(AzureStorage): # pylint: disable=abstract-method - """Storage class for Static contents""" - - location = "static" - - -class AzureImagesStorage(AzureStorage): # pylint: disable=abstract-method - """Storage class for Image files""" - - location = "images" - overwrite_files = False - - -class ExportsFileStorage(FileSystemStorage): # pylint: disable=abstract-method - """Storage class for exports contents with local files""" - - location = "exports" - overwrite_files = False - - -class ExportsS3Storage(S3Storage): # pylint: disable=abstract-method - """Storage class for exports contents with S3""" - - location = "exports" - default_acl = None - overwrite_files = False From 23bf08900406c34f759b9c30af7494628ac16c02 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:28:50 +0100 Subject: [PATCH 12/56] Update boto3 to 1.34.74 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c3b33f94b..3bd59e277 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ aiohttp==3.9.4 bleach==5.0.1 -boto3==1.26.57 +boto3==1.34.74 bw-file-resubmit==0.6.0rc2 celery==5.3.1 colorthief==0.2.1 From 3dfbc44c9a4ce096d13715b7c602f28c03318186 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:30:34 +0100 Subject: [PATCH 13/56] Update django-celery-beat to 2.6.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3bd59e277..2b94fbcda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ bw-file-resubmit==0.6.0rc2 celery==5.3.1 colorthief==0.2.1 Django==4.2.11 -django-celery-beat==2.5.0 +django-celery-beat==2.6.0 django-compressor==4.4 django-csp==3.7 django-imagekit==4.1.0 From 16e1b17a33874c641265f263e864c3aa3498f773 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:32:04 +0100 Subject: [PATCH 14/56] Update django-csp to 3.8 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2b94fbcda..c925e39bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ colorthief==0.2.1 Django==4.2.11 django-celery-beat==2.6.0 django-compressor==4.4 -django-csp==3.7 +django-csp==3.8 django-imagekit==4.1.0 django-model-utils==4.3.1 django-oauth-toolkit==2.3.0 From ffb3549e067e231a9cd8c71c5e24ac1e84e9de8b Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:34:52 +0100 Subject: [PATCH 15/56] Update django-imagekit to 5.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c925e39bd..520f73f9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ Django==4.2.11 django-celery-beat==2.6.0 django-compressor==4.4 django-csp==3.8 -django-imagekit==4.1.0 +django-imagekit==5.0.0 django-model-utils==4.3.1 django-oauth-toolkit==2.3.0 django-pgtrigger==4.11.0 From 69c273486c4aad8f2390e25a4bd073af40deec49 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:36:11 +0100 Subject: [PATCH 16/56] Update django-model-utils to 4.4.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 520f73f9f..5fa9e01ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ django-celery-beat==2.6.0 django-compressor==4.4 django-csp==3.8 django-imagekit==5.0.0 -django-model-utils==4.3.1 +django-model-utils==4.4.0 django-oauth-toolkit==2.3.0 django-pgtrigger==4.11.0 django-redis==5.2.0 From e0a14ea2ba23335a558cd3472c139216d4049573 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:41:19 +0100 Subject: [PATCH 17/56] Update django-sass-processor to 1.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5fa9e01ab..8463f8c6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ django-model-utils==4.4.0 django-oauth-toolkit==2.3.0 django-pgtrigger==4.11.0 django-redis==5.2.0 -django-sass-processor==1.2.2 +django-sass-processor==1.4 django-storages==1.14.2 django-storages[azure] environs==11.0.0 From e9325b87984caf61f33a046d15def808f66dd0c1 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:41:56 +0100 Subject: [PATCH 18/56] Update libsass to 0.23.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8463f8c6f..54ce7f958 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ django-storages[azure] environs==11.0.0 flower==2.0.1 grpcio==1.57.0 # Not a direct dependency, pinned to get a security fix -libsass==0.22.0 +libsass==0.23.0 Markdown==3.4.1 opentelemetry-api==1.16.0 opentelemetry-exporter-otlp-proto-grpc==1.16.0 From 1276112214c145ec4d14eb4657ddf300d9446ea5 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:45:42 +0100 Subject: [PATCH 19/56] Update opentelemetry dependencies --- requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 54ce7f958..17f76c43d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,12 +21,12 @@ flower==2.0.1 grpcio==1.57.0 # Not a direct dependency, pinned to get a security fix libsass==0.23.0 Markdown==3.4.1 -opentelemetry-api==1.16.0 -opentelemetry-exporter-otlp-proto-grpc==1.16.0 -opentelemetry-instrumentation-celery==0.37b0 -opentelemetry-instrumentation-django==0.37b0 -opentelemetry-instrumentation-psycopg2==0.37b0 -opentelemetry-sdk==1.16.0 +opentelemetry-api==1.24.0 +opentelemetry-exporter-otlp-proto-grpc==1.24.0 +opentelemetry-instrumentation-celery==0.45b0 +opentelemetry-instrumentation-django==0.45b0 +opentelemetry-instrumentation-psycopg2==0.45b0 +opentelemetry-sdk==1.24.0 Pillow==10.3.0 pilkit>=3.0 # dependency of django-imagekit, 2.0 is incompatible with Pillow>=10 protobuf==3.20.* From 309147bd9864287a80785a59fd123e13d9184b85 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:57:37 +0100 Subject: [PATCH 20/56] Update pycryptodome to 3.20.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 17f76c43d..7302639d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ Pillow==10.3.0 pilkit>=3.0 # dependency of django-imagekit, 2.0 is incompatible with Pillow>=10 protobuf==3.20.* psycopg2==2.9.5 -pycryptodome==3.19.1 +pycryptodome==3.20.0 pyotp==2.8.0 python-dateutil==2.8.2 pytz>=2022.7 From c11725a5c89decf6a6671d4191334c9927a49769 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:58:29 +0100 Subject: [PATCH 21/56] Update pyotp to 2.9.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7302639d5..9fe9b6271 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ pilkit>=3.0 # dependency of django-imagekit, 2.0 is incompatible with Pillow>=10 protobuf==3.20.* psycopg2==2.9.5 pycryptodome==3.20.0 -pyotp==2.8.0 +pyotp==2.9.0 python-dateutil==2.8.2 pytz>=2022.7 qrcode==7.3.1 From 1f8ba4df3e5fd69fb4de3480fbfe062fa108c800 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 23:01:17 +0100 Subject: [PATCH 22/56] Update python-dateutil to 2.9.0.post0 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9fe9b6271..ebda181cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ protobuf==3.20.* psycopg2==2.9.5 pycryptodome==3.20.0 pyotp==2.9.0 -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 pytz>=2022.7 qrcode==7.3.1 redis==4.5.4 @@ -60,5 +60,5 @@ types-dataclasses==0.6.6 types-Markdown==3.4.2.10 types-Pillow==10.2.0.20240311 types-psycopg2==2.9.21.11 -types-python-dateutil==2.8.19.14 +types-python-dateutil==2.9.0.20240316 types-requests==2.31.0.2 From e1fd57a1d67add7fa212f3003830a22c3645bbb6 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 30 Mar 2024 20:30:29 +0100 Subject: [PATCH 23/56] Fix constructor arguments to SessionMiddleware in tests --- bookwyrm/tests/views/preferences/test_delete_user.py | 8 ++++---- bookwyrm/tests/views/preferences/test_move.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bookwyrm/tests/views/preferences/test_delete_user.py b/bookwyrm/tests/views/preferences/test_delete_user.py index 34ab52be4..68a1ebc3b 100644 --- a/bookwyrm/tests/views/preferences/test_delete_user.py +++ b/bookwyrm/tests/views/preferences/test_delete_user.py @@ -78,7 +78,7 @@ class DeleteUserViews(TestCase): form.data["password"] = "password" request = self.factory.post("", form.data) request.user = self.local_user - middleware = SessionMiddleware() + middleware = SessionMiddleware(request) middleware.process_request(request) request.session.save() @@ -105,7 +105,7 @@ class DeleteUserViews(TestCase): view = views.DeactivateUser.as_view() request = self.factory.post("") request.user = self.local_user - middleware = SessionMiddleware() + middleware = SessionMiddleware(request) middleware.process_request(request) request.session.save() @@ -137,7 +137,7 @@ class DeleteUserViews(TestCase): form.data["password"] = "password" request = self.factory.post("", form.data) request.user = self.local_user - middleware = SessionMiddleware() + middleware = SessionMiddleware(request) middleware.process_request(request) request.session.save() @@ -159,7 +159,7 @@ class DeleteUserViews(TestCase): form.data["password"] = "password" request = self.factory.post("", form.data) request.user = self.local_user - middleware = SessionMiddleware() + middleware = SessionMiddleware(request) middleware.process_request(request) request.session.save() diff --git a/bookwyrm/tests/views/preferences/test_move.py b/bookwyrm/tests/views/preferences/test_move.py index 15edf3638..6086d8184 100644 --- a/bookwyrm/tests/views/preferences/test_move.py +++ b/bookwyrm/tests/views/preferences/test_move.py @@ -101,7 +101,7 @@ class ViewsHelpers(TestCase): request = self.factory.post("", form.data) request.user = self.local_user - middleware = SessionMiddleware() + middleware = SessionMiddleware(request) middleware.process_request(request) request.session.save() From d80a0146bde8bda657bb126b0b62289687518f0f Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 30 Mar 2024 21:28:03 +0100 Subject: [PATCH 24/56] Update django-stubs to 4.2.7 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ebda181cd..f4ac2e370 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,7 @@ tornado==6.3.3 # Not a direct dependency, pinned to get a security fix # Dev black==22.* celery-types==0.18.0 -django-stubs[compatible-mypy]==4.2.4 +django-stubs[compatible-mypy]==4.2.7 mypy==1.5.1 pylint==2.15.0 pytest==8.0.2 From 869bc5a376edfec6283940c0d461282d950b2c5d Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 30 Mar 2024 21:30:33 +0100 Subject: [PATCH 25/56] Update mypy to 1.7.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f4ac2e370..b92d13a9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,7 +47,7 @@ tornado==6.3.3 # Not a direct dependency, pinned to get a security fix black==22.* celery-types==0.18.0 django-stubs[compatible-mypy]==4.2.7 -mypy==1.5.1 +mypy==1.7.1 pylint==2.15.0 pytest==8.0.2 pytest-cov==5.0.0 From 224fae7a87c70daf3d993c464d214568d4223423 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 30 Mar 2024 21:47:19 +0100 Subject: [PATCH 26/56] Fix mypy errors --- bookwyrm/connectors/connector_manager.py | 8 ++++++-- bookwyrm/connectors/inventaire.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index 1e1b3b554..ad68af1dc 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -145,7 +145,9 @@ def load_more_data(connector_id: str, book_id: str) -> None: """background the work of getting all 10,000 editions of LoTR""" connector_info = models.Connector.objects.get(id=connector_id) connector = load_connector(connector_info) - book = models.Book.objects.select_subclasses().get(id=book_id) + book = models.Book.objects.select_subclasses().get( # type: ignore[no-untyped-call] + id=book_id + ) connector.expand_book_data(book) @@ -156,7 +158,9 @@ def create_edition_task( """separate task for each of the 10,000 editions of LoTR""" connector_info = models.Connector.objects.get(id=connector_id) connector = load_connector(connector_info) - work = models.Work.objects.select_subclasses().get(id=work_id) + work = models.Work.objects.select_subclasses().get( # type: ignore[no-untyped-call] + id=work_id + ) connector.create_edition_from_data(work, data) diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py index c08bcdee1..249f6b9ca 100644 --- a/bookwyrm/connectors/inventaire.py +++ b/bookwyrm/connectors/inventaire.py @@ -229,7 +229,7 @@ class Connector(AbstractConnector): data = get_data(url) except ConnectorException: return "" - return data.get("extract", "") + return str(data.get("extract", "")) def get_remote_id_from_model(self, obj: models.BookDataModel) -> str: """use get_remote_id to figure out the link from a model obj""" From 624115bf1161e1fe1411c3ba2089c2b44c857d17 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 15:28:01 +0200 Subject: [PATCH 27/56] Use headers dict instead of HTTP_* kwargs or request.META --- bookwyrm/tests/test_signing.py | 12 +++---- bookwyrm/tests/views/inbox/test_inbox.py | 5 ++- bookwyrm/tests/views/test_helpers.py | 45 ++++++++++++++++++------ bookwyrm/tests/views/test_outbox.py | 2 +- bookwyrm/views/helpers.py | 2 +- 5 files changed, 47 insertions(+), 19 deletions(-) diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py index 79370844a..2e0105c1c 100644 --- a/bookwyrm/tests/test_signing.py +++ b/bookwyrm/tests/test_signing.py @@ -72,12 +72,12 @@ class Signature(TestCase): urlsplit(self.rat.inbox).path, data=data, content_type="application/json", - **{ - "HTTP_DATE": now, - "HTTP_SIGNATURE": signature, - "HTTP_DIGEST": digest, - "HTTP_CONTENT_TYPE": "application/activity+json; charset=utf-8", - "HTTP_HOST": NETLOC, + headers={ + "date": now, + "signature": signature, + "digest": digest, + "content-type": "application/activity+json; charset=utf-8", + "host": NETLOC, }, ) diff --git a/bookwyrm/tests/views/inbox/test_inbox.py b/bookwyrm/tests/views/inbox/test_inbox.py index 92ee8a43d..c29aa71a2 100644 --- a/bookwyrm/tests/views/inbox/test_inbox.py +++ b/bookwyrm/tests/views/inbox/test_inbox.py @@ -134,7 +134,10 @@ class Inbox(TestCase): """check for blocked servers""" request = self.factory.post( "", - HTTP_USER_AGENT="http.rb/4.4.1 (Mastodon/3.3.0; +https://mastodon.social/)", + headers={ + # pylint: disable-next=line-too-long + "user-agent": "http.rb/4.4.1 (Mastodon/3.3.0; +https://mastodon.social/)", + }, ) self.assertIsNone(views.inbox.raise_is_blocked_user_agent(request)) diff --git a/bookwyrm/tests/views/test_helpers.py b/bookwyrm/tests/views/test_helpers.py index a1c06bede..64241d2b4 100644 --- a/bookwyrm/tests/views/test_helpers.py +++ b/bookwyrm/tests/views/test_helpers.py @@ -113,11 +113,20 @@ class ViewsHelpers(TestCase): # pylint: disable=too-many-public-methods request = self.factory.get( "", {"q": "Test Book"}, - HTTP_USER_AGENT="http.rb/4.4.1 (Mastodon/3.3.0; +https://mastodon.social/)", + headers={ + # pylint: disable-next=line-too-long + "user-agent": "http.rb/4.4.1 (Mastodon/3.3.0; +https://mastodon.social/)", + }, ) self.assertFalse(views.helpers.is_bookwyrm_request(request)) - request = self.factory.get("", {"q": "Test Book"}, HTTP_USER_AGENT=USER_AGENT) + request = self.factory.get( + "", + {"q": "Test Book"}, + headers={ + "user-agent": USER_AGENT, + }, + ) self.assertTrue(views.helpers.is_bookwyrm_request(request)) def test_handle_remote_webfinger_invalid(self, *_): @@ -271,8 +280,12 @@ class ViewsHelpers(TestCase): # pylint: disable=too-many-public-methods def test_redirect_to_referer_outside_domain(self, *_): """safely send people on their way""" - request = self.factory.get("/path") - request.META = {"HTTP_REFERER": "http://outside.domain/name"} + request = self.factory.get( + "/path", + headers={ + "referer": "http://outside.domain/name", + }, + ) result = views.helpers.redirect_to_referer( request, "user-feed", self.local_user.localname ) @@ -280,21 +293,33 @@ class ViewsHelpers(TestCase): # pylint: disable=too-many-public-methods def test_redirect_to_referer_outside_domain_with_fallback(self, *_): """invalid domain with regular params for the redirect function""" - request = self.factory.get("/path") - request.META = {"HTTP_REFERER": "https://outside.domain/name"} + request = self.factory.get( + "/path", + headers={ + "referer": "http://outside.domain/name", + }, + ) result = views.helpers.redirect_to_referer(request) self.assertEqual(result.url, "/") def test_redirect_to_referer_valid_domain(self, *_): """redirect to within the app""" - request = self.factory.get("/path") - request.META = {"HTTP_REFERER": f"{BASE_URL}/and/a/path"} + request = self.factory.get( + "/path", + headers={ + "referer": f"{BASE_URL}/and/a/path", + }, + ) result = views.helpers.redirect_to_referer(request) self.assertEqual(result.url, f"{BASE_URL}/and/a/path") def test_redirect_to_referer_with_get_args(self, *_): """if the path has get params (like sort) they are preserved""" - request = self.factory.get("/path") - request.META = {"HTTP_REFERER": f"{BASE_URL}/and/a/path?sort=hello"} + request = self.factory.get( + "/path", + headers={ + "referer": f"{BASE_URL}/and/a/path?sort=hello", + }, + ) result = views.helpers.redirect_to_referer(request) self.assertEqual(result.url, f"{BASE_URL}/and/a/path?sort=hello") diff --git a/bookwyrm/tests/views/test_outbox.py b/bookwyrm/tests/views/test_outbox.py index b21d56c83..bbd4aa37b 100644 --- a/bookwyrm/tests/views/test_outbox.py +++ b/bookwyrm/tests/views/test_outbox.py @@ -122,7 +122,7 @@ class OutboxView(TestCase): privacy="public", ) - request = self.factory.get("", {"page": 1}, HTTP_USER_AGENT=USER_AGENT) + request = self.factory.get("", {"page": 1}, headers={"user-agent": USER_AGENT}) result = views.Outbox.as_view()(request, "mouse") data = json.loads(result.content) diff --git a/bookwyrm/views/helpers.py b/bookwyrm/views/helpers.py index bdff119f2..391788b0c 100644 --- a/bookwyrm/views/helpers.py +++ b/bookwyrm/views/helpers.py @@ -231,7 +231,7 @@ def maybe_redirect_local_path(request, model): def redirect_to_referer(request, *args, **kwargs): """Redirect to the referrer, if it's in our domain, with get params""" # make sure the refer is part of this instance - validated = validate_url_domain(request.META.get("HTTP_REFERER")) + validated = validate_url_domain(request.headers.get("referer", "")) if validated: return redirect(validated) From 1303f539c3adbcd1b353574085e805a272d199d7 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 15:36:22 +0200 Subject: [PATCH 28/56] Update psycopg to 2.9.9 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b92d13a9e..d610d1e33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ opentelemetry-sdk==1.24.0 Pillow==10.3.0 pilkit>=3.0 # dependency of django-imagekit, 2.0 is incompatible with Pillow>=10 protobuf==3.20.* -psycopg2==2.9.5 +psycopg2==2.9.9 pycryptodome==3.20.0 pyotp==2.9.0 python-dateutil==2.9.0.post0 @@ -59,6 +59,6 @@ types-bleach==6.0.0.4 types-dataclasses==0.6.6 types-Markdown==3.4.2.10 types-Pillow==10.2.0.20240311 -types-psycopg2==2.9.21.11 +types-psycopg2==2.9.21.20240311 types-python-dateutil==2.9.0.20240316 types-requests==2.31.0.2 From b6174d9101f19f49fd9fbc3c6b696d3e4a8010bd Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 18:30:53 +0200 Subject: [PATCH 29/56] Update bleach to 6.1.0 --- bookwyrm/utils/sanitizer.py | 4 ++-- requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bookwyrm/utils/sanitizer.py b/bookwyrm/utils/sanitizer.py index 4f5132c9e..1467ee3e1 100644 --- a/bookwyrm/utils/sanitizer.py +++ b/bookwyrm/utils/sanitizer.py @@ -6,7 +6,7 @@ def clean(input_text: str) -> str: """Run through "bleach" """ return bleach.clean( input_text, - tags=[ + tags={ "p", "blockquote", "br", @@ -20,7 +20,7 @@ def clean(input_text: str) -> str: "ul", "ol", "li", - ], + }, attributes=["href", "rel", "src", "alt", "data-mention"], strip=True, ) diff --git a/requirements.txt b/requirements.txt index d610d1e33..59ec8d2bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ aiohttp==3.9.4 -bleach==5.0.1 +bleach==6.1.0 boto3==1.34.74 bw-file-resubmit==0.6.0rc2 celery==5.3.1 @@ -55,7 +55,7 @@ pytest-django==4.8.0 pytest-env==0.6.2 pytest-xdist==2.3.0 pytidylib==0.3.2 -types-bleach==6.0.0.4 +types-bleach==6.1.0.20240331 types-dataclasses==0.6.6 types-Markdown==3.4.2.10 types-Pillow==10.2.0.20240311 From 9ebda3fbe8f6a4f8356e6cd52a8899113f60977e Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 18:33:59 +0200 Subject: [PATCH 30/56] Update celery to 5.3.6 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 59ec8d2bd..5fa5fd065 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ aiohttp==3.9.4 bleach==6.1.0 boto3==1.34.74 bw-file-resubmit==0.6.0rc2 -celery==5.3.1 +celery==5.3.6 colorthief==0.2.1 Django==4.2.11 django-celery-beat==2.6.0 @@ -45,7 +45,7 @@ tornado==6.3.3 # Not a direct dependency, pinned to get a security fix # Dev black==22.* -celery-types==0.18.0 +celery-types==0.22.0 django-stubs[compatible-mypy]==4.2.7 mypy==1.7.1 pylint==2.15.0 From 01b37026ebf1556b9ee63df17bad7efe4d4166f4 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 18:36:12 +0200 Subject: [PATCH 31/56] Update Markdown to 3.6 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5fa5fd065..e3c164e65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ environs==11.0.0 flower==2.0.1 grpcio==1.57.0 # Not a direct dependency, pinned to get a security fix libsass==0.23.0 -Markdown==3.4.1 +Markdown==3.6 opentelemetry-api==1.24.0 opentelemetry-exporter-otlp-proto-grpc==1.24.0 opentelemetry-instrumentation-celery==0.45b0 @@ -57,7 +57,7 @@ pytest-xdist==2.3.0 pytidylib==0.3.2 types-bleach==6.1.0.20240331 types-dataclasses==0.6.6 -types-Markdown==3.4.2.10 +types-Markdown==3.6.0.20240316 types-Pillow==10.2.0.20240311 types-psycopg2==2.9.21.20240311 types-python-dateutil==2.9.0.20240316 From e46bc2e9a10a53e04dceb207d157f0adf1e303a1 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 18:39:28 +0200 Subject: [PATCH 32/56] Update redis-py to 5.0.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e3c164e65..846df4fe0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ pyotp==2.9.0 python-dateutil==2.9.0.post0 pytz>=2022.7 qrcode==7.3.1 -redis==4.5.4 +redis==5.0.3 requests==2.31.0 responses==0.22.0 s3-tar==0.1.13 From 1474c0d3aa28a3f8932d997236dcf7d94f5f4f48 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 19:10:38 +0200 Subject: [PATCH 33/56] Remove protobuf as a direct dependency --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 846df4fe0..f87cc0282 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,6 @@ opentelemetry-instrumentation-psycopg2==0.45b0 opentelemetry-sdk==1.24.0 Pillow==10.3.0 pilkit>=3.0 # dependency of django-imagekit, 2.0 is incompatible with Pillow>=10 -protobuf==3.20.* psycopg2==2.9.9 pycryptodome==3.20.0 pyotp==2.9.0 From 2537886b4d9cfe5580d5d540bedc4b605a0c0ed5 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 19:11:27 +0200 Subject: [PATCH 34/56] Group version constraints for indirect dependencies and change to >= --- requirements.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index f87cc0282..c38c9c054 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,6 @@ django-storages==1.14.2 django-storages[azure] environs==11.0.0 flower==2.0.1 -grpcio==1.57.0 # Not a direct dependency, pinned to get a security fix libsass==0.23.0 Markdown==3.6 opentelemetry-api==1.24.0 @@ -39,8 +38,11 @@ redis==5.0.3 requests==2.31.0 responses==0.22.0 s3-tar==0.1.13 -setuptools>=65.5.1 # Not a direct dependency, pinned to get a security fix -tornado==6.3.3 # Not a direct dependency, pinned to get a security fix + +# Indirect dependencies with version constraints for security fixes +grpcio>=1.57.0 +setuptools>=65.5.1 +tornado>=6.3.3 # Dev black==22.* From 1cb86197d58585148be1bd3d5dd1c06c246c2112 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 20:23:39 +0200 Subject: [PATCH 35/56] Update types-requests to 2.31.0.20240311 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c38c9c054..e827eac13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -62,4 +62,4 @@ types-Markdown==3.6.0.20240316 types-Pillow==10.2.0.20240311 types-psycopg2==2.9.21.20240311 types-python-dateutil==2.9.0.20240316 -types-requests==2.31.0.2 +types-requests==2.31.0.20240311 From a1ff5a478ed73897f27599fc1bec7ff17dcd7599 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 20:24:53 +0200 Subject: [PATCH 36/56] Update types-Pillow to 10.2.0.20240331 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e827eac13..89817654d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -59,7 +59,7 @@ pytidylib==0.3.2 types-bleach==6.1.0.20240331 types-dataclasses==0.6.6 types-Markdown==3.6.0.20240316 -types-Pillow==10.2.0.20240311 +types-Pillow==10.2.0.20240331 types-psycopg2==2.9.21.20240311 types-python-dateutil==2.9.0.20240316 types-requests==2.31.0.20240311 From 039160e004e635c5f5e43b28596bb5316ab1bcc9 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 20:27:45 +0200 Subject: [PATCH 37/56] Update pytest-env to 1.1.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 89817654d..dcd3f701c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,7 +53,7 @@ pylint==2.15.0 pytest==8.0.2 pytest-cov==5.0.0 pytest-django==4.8.0 -pytest-env==0.6.2 +pytest-env==1.1.3 pytest-xdist==2.3.0 pytidylib==0.3.2 types-bleach==6.1.0.20240331 From f324a3cd1d7bf2bda9f4ac3258fb0d10081c861b Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 20:30:13 +0200 Subject: [PATCH 38/56] Update pytest-xdist to 3.5.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dcd3f701c..8d8bc7c0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ pytest==8.0.2 pytest-cov==5.0.0 pytest-django==4.8.0 pytest-env==1.1.3 -pytest-xdist==2.3.0 +pytest-xdist==3.5.0 pytidylib==0.3.2 types-bleach==6.1.0.20240331 types-dataclasses==0.6.6 From f6bbe673ca5675a9832452cbb1cdb49e6fb9c50a Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 20:32:06 +0200 Subject: [PATCH 39/56] Update responses to 0.25.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8d8bc7c0e..562f430bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ pytz>=2022.7 qrcode==7.3.1 redis==5.0.3 requests==2.31.0 -responses==0.22.0 +responses==0.25.0 s3-tar==0.1.13 # Indirect dependencies with version constraints for security fixes From 22986a08f0631468db8308f8177a2fdba5b3efc3 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 20:48:52 +0200 Subject: [PATCH 40/56] Update pytest to 8.1.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 562f430bc..04899876b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ celery-types==0.22.0 django-stubs[compatible-mypy]==4.2.7 mypy==1.7.1 pylint==2.15.0 -pytest==8.0.2 +pytest==8.1.1 pytest-cov==5.0.0 pytest-django==4.8.0 pytest-env==1.1.3 From 39c2a0feaebca8eb873278a17c302c5e76e20baf Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 20:53:19 +0200 Subject: [PATCH 41/56] Update qrcode to 7.4.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 04899876b..27b43873b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ pycryptodome==3.20.0 pyotp==2.9.0 python-dateutil==2.9.0.post0 pytz>=2022.7 -qrcode==7.3.1 +qrcode==7.4.2 redis==5.0.3 requests==2.31.0 responses==0.25.0 From 03ac846b5d312086aeccb27dc01cc6d5604dd47b Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 21:50:31 +0200 Subject: [PATCH 42/56] Migrate from pytz to zoneinfo --- bookwyrm/middleware/timezone_middleware.py | 10 +- .../0171_alter_user_preferred_timezone.py | 1 + .../0200_alter_user_preferred_timezone.py | 633 ++++++++++++++++++ bookwyrm/models/user.py | 7 +- .../tests/importers/test_goodreads_import.py | 3 +- bookwyrm/tests/importers/test_importer.py | 3 +- .../importers/test_librarything_import.py | 3 +- .../importers/test_openlibrary_import.py | 3 +- .../tests/importers/test_storygraph_import.py | 3 +- bookwyrm/tests/views/test_annual_summary.py | 3 +- bookwyrm/views/landing/register.py | 10 +- requirements.txt | 1 - 12 files changed, 654 insertions(+), 26 deletions(-) create mode 100644 bookwyrm/migrations/0200_alter_user_preferred_timezone.py diff --git a/bookwyrm/middleware/timezone_middleware.py b/bookwyrm/middleware/timezone_middleware.py index 5033397a5..3cf084154 100644 --- a/bookwyrm/middleware/timezone_middleware.py +++ b/bookwyrm/middleware/timezone_middleware.py @@ -1,5 +1,5 @@ """ Makes the app aware of the users timezone """ -import pytz +import zoneinfo from django.utils import timezone @@ -12,9 +12,7 @@ class TimezoneMiddleware: def __call__(self, request): if request.user.is_authenticated: - timezone.activate(pytz.timezone(request.user.preferred_timezone)) + timezone.activate(zoneinfo.ZoneInfo(request.user.preferred_timezone)) else: - timezone.activate(pytz.utc) - response = self.get_response(request) - timezone.deactivate() - return response + timezone.deactivate() + return self.get_response(request) diff --git a/bookwyrm/migrations/0171_alter_user_preferred_timezone.py b/bookwyrm/migrations/0171_alter_user_preferred_timezone.py index 7dcd9546c..8d1dff553 100644 --- a/bookwyrm/migrations/0171_alter_user_preferred_timezone.py +++ b/bookwyrm/migrations/0171_alter_user_preferred_timezone.py @@ -10,6 +10,7 @@ class Migration(migrations.Migration): ] operations = [ + # The new timezones are "Factory" and "localtime" migrations.AlterField( model_name="user", name="preferred_timezone", diff --git a/bookwyrm/migrations/0200_alter_user_preferred_timezone.py b/bookwyrm/migrations/0200_alter_user_preferred_timezone.py new file mode 100644 index 000000000..1b21c0f94 --- /dev/null +++ b/bookwyrm/migrations/0200_alter_user_preferred_timezone.py @@ -0,0 +1,633 @@ +# Generated by Django 4.2.11 on 2024-04-01 20:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0199_alter_userblocks_user_object_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="preferred_timezone", + field=models.CharField( + choices=[ + ("Africa/Abidjan", "Africa/Abidjan"), + ("Africa/Accra", "Africa/Accra"), + ("Africa/Addis_Ababa", "Africa/Addis_Ababa"), + ("Africa/Algiers", "Africa/Algiers"), + ("Africa/Asmara", "Africa/Asmara"), + ("Africa/Asmera", "Africa/Asmera"), + ("Africa/Bamako", "Africa/Bamako"), + ("Africa/Bangui", "Africa/Bangui"), + ("Africa/Banjul", "Africa/Banjul"), + ("Africa/Bissau", "Africa/Bissau"), + ("Africa/Blantyre", "Africa/Blantyre"), + ("Africa/Brazzaville", "Africa/Brazzaville"), + ("Africa/Bujumbura", "Africa/Bujumbura"), + ("Africa/Cairo", "Africa/Cairo"), + ("Africa/Casablanca", "Africa/Casablanca"), + ("Africa/Ceuta", "Africa/Ceuta"), + ("Africa/Conakry", "Africa/Conakry"), + ("Africa/Dakar", "Africa/Dakar"), + ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"), + ("Africa/Djibouti", "Africa/Djibouti"), + ("Africa/Douala", "Africa/Douala"), + ("Africa/El_Aaiun", "Africa/El_Aaiun"), + ("Africa/Freetown", "Africa/Freetown"), + ("Africa/Gaborone", "Africa/Gaborone"), + ("Africa/Harare", "Africa/Harare"), + ("Africa/Johannesburg", "Africa/Johannesburg"), + ("Africa/Juba", "Africa/Juba"), + ("Africa/Kampala", "Africa/Kampala"), + ("Africa/Khartoum", "Africa/Khartoum"), + ("Africa/Kigali", "Africa/Kigali"), + ("Africa/Kinshasa", "Africa/Kinshasa"), + ("Africa/Lagos", "Africa/Lagos"), + ("Africa/Libreville", "Africa/Libreville"), + ("Africa/Lome", "Africa/Lome"), + ("Africa/Luanda", "Africa/Luanda"), + ("Africa/Lubumbashi", "Africa/Lubumbashi"), + ("Africa/Lusaka", "Africa/Lusaka"), + ("Africa/Malabo", "Africa/Malabo"), + ("Africa/Maputo", "Africa/Maputo"), + ("Africa/Maseru", "Africa/Maseru"), + ("Africa/Mbabane", "Africa/Mbabane"), + ("Africa/Mogadishu", "Africa/Mogadishu"), + ("Africa/Monrovia", "Africa/Monrovia"), + ("Africa/Nairobi", "Africa/Nairobi"), + ("Africa/Ndjamena", "Africa/Ndjamena"), + ("Africa/Niamey", "Africa/Niamey"), + ("Africa/Nouakchott", "Africa/Nouakchott"), + ("Africa/Ouagadougou", "Africa/Ouagadougou"), + ("Africa/Porto-Novo", "Africa/Porto-Novo"), + ("Africa/Sao_Tome", "Africa/Sao_Tome"), + ("Africa/Timbuktu", "Africa/Timbuktu"), + ("Africa/Tripoli", "Africa/Tripoli"), + ("Africa/Tunis", "Africa/Tunis"), + ("Africa/Windhoek", "Africa/Windhoek"), + ("America/Adak", "America/Adak"), + ("America/Anchorage", "America/Anchorage"), + ("America/Anguilla", "America/Anguilla"), + ("America/Antigua", "America/Antigua"), + ("America/Araguaina", "America/Araguaina"), + ( + "America/Argentina/Buenos_Aires", + "America/Argentina/Buenos_Aires", + ), + ("America/Argentina/Catamarca", "America/Argentina/Catamarca"), + ( + "America/Argentina/ComodRivadavia", + "America/Argentina/ComodRivadavia", + ), + ("America/Argentina/Cordoba", "America/Argentina/Cordoba"), + ("America/Argentina/Jujuy", "America/Argentina/Jujuy"), + ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"), + ("America/Argentina/Mendoza", "America/Argentina/Mendoza"), + ( + "America/Argentina/Rio_Gallegos", + "America/Argentina/Rio_Gallegos", + ), + ("America/Argentina/Salta", "America/Argentina/Salta"), + ("America/Argentina/San_Juan", "America/Argentina/San_Juan"), + ("America/Argentina/San_Luis", "America/Argentina/San_Luis"), + ("America/Argentina/Tucuman", "America/Argentina/Tucuman"), + ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"), + ("America/Aruba", "America/Aruba"), + ("America/Asuncion", "America/Asuncion"), + ("America/Atikokan", "America/Atikokan"), + ("America/Atka", "America/Atka"), + ("America/Bahia", "America/Bahia"), + ("America/Bahia_Banderas", "America/Bahia_Banderas"), + ("America/Barbados", "America/Barbados"), + ("America/Belem", "America/Belem"), + ("America/Belize", "America/Belize"), + ("America/Blanc-Sablon", "America/Blanc-Sablon"), + ("America/Boa_Vista", "America/Boa_Vista"), + ("America/Bogota", "America/Bogota"), + ("America/Boise", "America/Boise"), + ("America/Buenos_Aires", "America/Buenos_Aires"), + ("America/Cambridge_Bay", "America/Cambridge_Bay"), + ("America/Campo_Grande", "America/Campo_Grande"), + ("America/Cancun", "America/Cancun"), + ("America/Caracas", "America/Caracas"), + ("America/Catamarca", "America/Catamarca"), + ("America/Cayenne", "America/Cayenne"), + ("America/Cayman", "America/Cayman"), + ("America/Chicago", "America/Chicago"), + ("America/Chihuahua", "America/Chihuahua"), + ("America/Ciudad_Juarez", "America/Ciudad_Juarez"), + ("America/Coral_Harbour", "America/Coral_Harbour"), + ("America/Cordoba", "America/Cordoba"), + ("America/Costa_Rica", "America/Costa_Rica"), + ("America/Creston", "America/Creston"), + ("America/Cuiaba", "America/Cuiaba"), + ("America/Curacao", "America/Curacao"), + ("America/Danmarkshavn", "America/Danmarkshavn"), + ("America/Dawson", "America/Dawson"), + ("America/Dawson_Creek", "America/Dawson_Creek"), + ("America/Denver", "America/Denver"), + ("America/Detroit", "America/Detroit"), + ("America/Dominica", "America/Dominica"), + ("America/Edmonton", "America/Edmonton"), + ("America/Eirunepe", "America/Eirunepe"), + ("America/El_Salvador", "America/El_Salvador"), + ("America/Ensenada", "America/Ensenada"), + ("America/Fort_Nelson", "America/Fort_Nelson"), + ("America/Fort_Wayne", "America/Fort_Wayne"), + ("America/Fortaleza", "America/Fortaleza"), + ("America/Glace_Bay", "America/Glace_Bay"), + ("America/Godthab", "America/Godthab"), + ("America/Goose_Bay", "America/Goose_Bay"), + ("America/Grand_Turk", "America/Grand_Turk"), + ("America/Grenada", "America/Grenada"), + ("America/Guadeloupe", "America/Guadeloupe"), + ("America/Guatemala", "America/Guatemala"), + ("America/Guayaquil", "America/Guayaquil"), + ("America/Guyana", "America/Guyana"), + ("America/Halifax", "America/Halifax"), + ("America/Havana", "America/Havana"), + ("America/Hermosillo", "America/Hermosillo"), + ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"), + ("America/Indiana/Knox", "America/Indiana/Knox"), + ("America/Indiana/Marengo", "America/Indiana/Marengo"), + ("America/Indiana/Petersburg", "America/Indiana/Petersburg"), + ("America/Indiana/Tell_City", "America/Indiana/Tell_City"), + ("America/Indiana/Vevay", "America/Indiana/Vevay"), + ("America/Indiana/Vincennes", "America/Indiana/Vincennes"), + ("America/Indiana/Winamac", "America/Indiana/Winamac"), + ("America/Indianapolis", "America/Indianapolis"), + ("America/Inuvik", "America/Inuvik"), + ("America/Iqaluit", "America/Iqaluit"), + ("America/Jamaica", "America/Jamaica"), + ("America/Jujuy", "America/Jujuy"), + ("America/Juneau", "America/Juneau"), + ("America/Kentucky/Louisville", "America/Kentucky/Louisville"), + ("America/Kentucky/Monticello", "America/Kentucky/Monticello"), + ("America/Knox_IN", "America/Knox_IN"), + ("America/Kralendijk", "America/Kralendijk"), + ("America/La_Paz", "America/La_Paz"), + ("America/Lima", "America/Lima"), + ("America/Los_Angeles", "America/Los_Angeles"), + ("America/Louisville", "America/Louisville"), + ("America/Lower_Princes", "America/Lower_Princes"), + ("America/Maceio", "America/Maceio"), + ("America/Managua", "America/Managua"), + ("America/Manaus", "America/Manaus"), + ("America/Marigot", "America/Marigot"), + ("America/Martinique", "America/Martinique"), + ("America/Matamoros", "America/Matamoros"), + ("America/Mazatlan", "America/Mazatlan"), + ("America/Mendoza", "America/Mendoza"), + ("America/Menominee", "America/Menominee"), + ("America/Merida", "America/Merida"), + ("America/Metlakatla", "America/Metlakatla"), + ("America/Mexico_City", "America/Mexico_City"), + ("America/Miquelon", "America/Miquelon"), + ("America/Moncton", "America/Moncton"), + ("America/Monterrey", "America/Monterrey"), + ("America/Montevideo", "America/Montevideo"), + ("America/Montreal", "America/Montreal"), + ("America/Montserrat", "America/Montserrat"), + ("America/Nassau", "America/Nassau"), + ("America/New_York", "America/New_York"), + ("America/Nipigon", "America/Nipigon"), + ("America/Nome", "America/Nome"), + ("America/Noronha", "America/Noronha"), + ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"), + ("America/North_Dakota/Center", "America/North_Dakota/Center"), + ( + "America/North_Dakota/New_Salem", + "America/North_Dakota/New_Salem", + ), + ("America/Nuuk", "America/Nuuk"), + ("America/Ojinaga", "America/Ojinaga"), + ("America/Panama", "America/Panama"), + ("America/Pangnirtung", "America/Pangnirtung"), + ("America/Paramaribo", "America/Paramaribo"), + ("America/Phoenix", "America/Phoenix"), + ("America/Port-au-Prince", "America/Port-au-Prince"), + ("America/Port_of_Spain", "America/Port_of_Spain"), + ("America/Porto_Acre", "America/Porto_Acre"), + ("America/Porto_Velho", "America/Porto_Velho"), + ("America/Puerto_Rico", "America/Puerto_Rico"), + ("America/Punta_Arenas", "America/Punta_Arenas"), + ("America/Rainy_River", "America/Rainy_River"), + ("America/Rankin_Inlet", "America/Rankin_Inlet"), + ("America/Recife", "America/Recife"), + ("America/Regina", "America/Regina"), + ("America/Resolute", "America/Resolute"), + ("America/Rio_Branco", "America/Rio_Branco"), + ("America/Rosario", "America/Rosario"), + ("America/Santa_Isabel", "America/Santa_Isabel"), + ("America/Santarem", "America/Santarem"), + ("America/Santiago", "America/Santiago"), + ("America/Santo_Domingo", "America/Santo_Domingo"), + ("America/Sao_Paulo", "America/Sao_Paulo"), + ("America/Scoresbysund", "America/Scoresbysund"), + ("America/Shiprock", "America/Shiprock"), + ("America/Sitka", "America/Sitka"), + ("America/St_Barthelemy", "America/St_Barthelemy"), + ("America/St_Johns", "America/St_Johns"), + ("America/St_Kitts", "America/St_Kitts"), + ("America/St_Lucia", "America/St_Lucia"), + ("America/St_Thomas", "America/St_Thomas"), + ("America/St_Vincent", "America/St_Vincent"), + ("America/Swift_Current", "America/Swift_Current"), + ("America/Tegucigalpa", "America/Tegucigalpa"), + ("America/Thule", "America/Thule"), + ("America/Thunder_Bay", "America/Thunder_Bay"), + ("America/Tijuana", "America/Tijuana"), + ("America/Toronto", "America/Toronto"), + ("America/Tortola", "America/Tortola"), + ("America/Vancouver", "America/Vancouver"), + ("America/Virgin", "America/Virgin"), + ("America/Whitehorse", "America/Whitehorse"), + ("America/Winnipeg", "America/Winnipeg"), + ("America/Yakutat", "America/Yakutat"), + ("America/Yellowknife", "America/Yellowknife"), + ("Antarctica/Casey", "Antarctica/Casey"), + ("Antarctica/Davis", "Antarctica/Davis"), + ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"), + ("Antarctica/Macquarie", "Antarctica/Macquarie"), + ("Antarctica/Mawson", "Antarctica/Mawson"), + ("Antarctica/McMurdo", "Antarctica/McMurdo"), + ("Antarctica/Palmer", "Antarctica/Palmer"), + ("Antarctica/Rothera", "Antarctica/Rothera"), + ("Antarctica/South_Pole", "Antarctica/South_Pole"), + ("Antarctica/Syowa", "Antarctica/Syowa"), + ("Antarctica/Troll", "Antarctica/Troll"), + ("Antarctica/Vostok", "Antarctica/Vostok"), + ("Arctic/Longyearbyen", "Arctic/Longyearbyen"), + ("Asia/Aden", "Asia/Aden"), + ("Asia/Almaty", "Asia/Almaty"), + ("Asia/Amman", "Asia/Amman"), + ("Asia/Anadyr", "Asia/Anadyr"), + ("Asia/Aqtau", "Asia/Aqtau"), + ("Asia/Aqtobe", "Asia/Aqtobe"), + ("Asia/Ashgabat", "Asia/Ashgabat"), + ("Asia/Ashkhabad", "Asia/Ashkhabad"), + ("Asia/Atyrau", "Asia/Atyrau"), + ("Asia/Baghdad", "Asia/Baghdad"), + ("Asia/Bahrain", "Asia/Bahrain"), + ("Asia/Baku", "Asia/Baku"), + ("Asia/Bangkok", "Asia/Bangkok"), + ("Asia/Barnaul", "Asia/Barnaul"), + ("Asia/Beirut", "Asia/Beirut"), + ("Asia/Bishkek", "Asia/Bishkek"), + ("Asia/Brunei", "Asia/Brunei"), + ("Asia/Calcutta", "Asia/Calcutta"), + ("Asia/Chita", "Asia/Chita"), + ("Asia/Choibalsan", "Asia/Choibalsan"), + ("Asia/Chongqing", "Asia/Chongqing"), + ("Asia/Chungking", "Asia/Chungking"), + ("Asia/Colombo", "Asia/Colombo"), + ("Asia/Dacca", "Asia/Dacca"), + ("Asia/Damascus", "Asia/Damascus"), + ("Asia/Dhaka", "Asia/Dhaka"), + ("Asia/Dili", "Asia/Dili"), + ("Asia/Dubai", "Asia/Dubai"), + ("Asia/Dushanbe", "Asia/Dushanbe"), + ("Asia/Famagusta", "Asia/Famagusta"), + ("Asia/Gaza", "Asia/Gaza"), + ("Asia/Harbin", "Asia/Harbin"), + ("Asia/Hebron", "Asia/Hebron"), + ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"), + ("Asia/Hong_Kong", "Asia/Hong_Kong"), + ("Asia/Hovd", "Asia/Hovd"), + ("Asia/Irkutsk", "Asia/Irkutsk"), + ("Asia/Istanbul", "Asia/Istanbul"), + ("Asia/Jakarta", "Asia/Jakarta"), + ("Asia/Jayapura", "Asia/Jayapura"), + ("Asia/Jerusalem", "Asia/Jerusalem"), + ("Asia/Kabul", "Asia/Kabul"), + ("Asia/Kamchatka", "Asia/Kamchatka"), + ("Asia/Karachi", "Asia/Karachi"), + ("Asia/Kashgar", "Asia/Kashgar"), + ("Asia/Kathmandu", "Asia/Kathmandu"), + ("Asia/Katmandu", "Asia/Katmandu"), + ("Asia/Khandyga", "Asia/Khandyga"), + ("Asia/Kolkata", "Asia/Kolkata"), + ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"), + ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"), + ("Asia/Kuching", "Asia/Kuching"), + ("Asia/Kuwait", "Asia/Kuwait"), + ("Asia/Macao", "Asia/Macao"), + ("Asia/Macau", "Asia/Macau"), + ("Asia/Magadan", "Asia/Magadan"), + ("Asia/Makassar", "Asia/Makassar"), + ("Asia/Manila", "Asia/Manila"), + ("Asia/Muscat", "Asia/Muscat"), + ("Asia/Nicosia", "Asia/Nicosia"), + ("Asia/Novokuznetsk", "Asia/Novokuznetsk"), + ("Asia/Novosibirsk", "Asia/Novosibirsk"), + ("Asia/Omsk", "Asia/Omsk"), + ("Asia/Oral", "Asia/Oral"), + ("Asia/Phnom_Penh", "Asia/Phnom_Penh"), + ("Asia/Pontianak", "Asia/Pontianak"), + ("Asia/Pyongyang", "Asia/Pyongyang"), + ("Asia/Qatar", "Asia/Qatar"), + ("Asia/Qostanay", "Asia/Qostanay"), + ("Asia/Qyzylorda", "Asia/Qyzylorda"), + ("Asia/Rangoon", "Asia/Rangoon"), + ("Asia/Riyadh", "Asia/Riyadh"), + ("Asia/Saigon", "Asia/Saigon"), + ("Asia/Sakhalin", "Asia/Sakhalin"), + ("Asia/Samarkand", "Asia/Samarkand"), + ("Asia/Seoul", "Asia/Seoul"), + ("Asia/Shanghai", "Asia/Shanghai"), + ("Asia/Singapore", "Asia/Singapore"), + ("Asia/Srednekolymsk", "Asia/Srednekolymsk"), + ("Asia/Taipei", "Asia/Taipei"), + ("Asia/Tashkent", "Asia/Tashkent"), + ("Asia/Tbilisi", "Asia/Tbilisi"), + ("Asia/Tehran", "Asia/Tehran"), + ("Asia/Tel_Aviv", "Asia/Tel_Aviv"), + ("Asia/Thimbu", "Asia/Thimbu"), + ("Asia/Thimphu", "Asia/Thimphu"), + ("Asia/Tokyo", "Asia/Tokyo"), + ("Asia/Tomsk", "Asia/Tomsk"), + ("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"), + ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"), + ("Asia/Ulan_Bator", "Asia/Ulan_Bator"), + ("Asia/Urumqi", "Asia/Urumqi"), + ("Asia/Ust-Nera", "Asia/Ust-Nera"), + ("Asia/Vientiane", "Asia/Vientiane"), + ("Asia/Vladivostok", "Asia/Vladivostok"), + ("Asia/Yakutsk", "Asia/Yakutsk"), + ("Asia/Yangon", "Asia/Yangon"), + ("Asia/Yekaterinburg", "Asia/Yekaterinburg"), + ("Asia/Yerevan", "Asia/Yerevan"), + ("Atlantic/Azores", "Atlantic/Azores"), + ("Atlantic/Bermuda", "Atlantic/Bermuda"), + ("Atlantic/Canary", "Atlantic/Canary"), + ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"), + ("Atlantic/Faeroe", "Atlantic/Faeroe"), + ("Atlantic/Faroe", "Atlantic/Faroe"), + ("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"), + ("Atlantic/Madeira", "Atlantic/Madeira"), + ("Atlantic/Reykjavik", "Atlantic/Reykjavik"), + ("Atlantic/South_Georgia", "Atlantic/South_Georgia"), + ("Atlantic/St_Helena", "Atlantic/St_Helena"), + ("Atlantic/Stanley", "Atlantic/Stanley"), + ("Australia/ACT", "Australia/ACT"), + ("Australia/Adelaide", "Australia/Adelaide"), + ("Australia/Brisbane", "Australia/Brisbane"), + ("Australia/Broken_Hill", "Australia/Broken_Hill"), + ("Australia/Canberra", "Australia/Canberra"), + ("Australia/Currie", "Australia/Currie"), + ("Australia/Darwin", "Australia/Darwin"), + ("Australia/Eucla", "Australia/Eucla"), + ("Australia/Hobart", "Australia/Hobart"), + ("Australia/LHI", "Australia/LHI"), + ("Australia/Lindeman", "Australia/Lindeman"), + ("Australia/Lord_Howe", "Australia/Lord_Howe"), + ("Australia/Melbourne", "Australia/Melbourne"), + ("Australia/NSW", "Australia/NSW"), + ("Australia/North", "Australia/North"), + ("Australia/Perth", "Australia/Perth"), + ("Australia/Queensland", "Australia/Queensland"), + ("Australia/South", "Australia/South"), + ("Australia/Sydney", "Australia/Sydney"), + ("Australia/Tasmania", "Australia/Tasmania"), + ("Australia/Victoria", "Australia/Victoria"), + ("Australia/West", "Australia/West"), + ("Australia/Yancowinna", "Australia/Yancowinna"), + ("Brazil/Acre", "Brazil/Acre"), + ("Brazil/DeNoronha", "Brazil/DeNoronha"), + ("Brazil/East", "Brazil/East"), + ("Brazil/West", "Brazil/West"), + ("CET", "CET"), + ("CST6CDT", "CST6CDT"), + ("Canada/Atlantic", "Canada/Atlantic"), + ("Canada/Central", "Canada/Central"), + ("Canada/Eastern", "Canada/Eastern"), + ("Canada/Mountain", "Canada/Mountain"), + ("Canada/Newfoundland", "Canada/Newfoundland"), + ("Canada/Pacific", "Canada/Pacific"), + ("Canada/Saskatchewan", "Canada/Saskatchewan"), + ("Canada/Yukon", "Canada/Yukon"), + ("Chile/Continental", "Chile/Continental"), + ("Chile/EasterIsland", "Chile/EasterIsland"), + ("Cuba", "Cuba"), + ("EET", "EET"), + ("EST", "EST"), + ("EST5EDT", "EST5EDT"), + ("Egypt", "Egypt"), + ("Eire", "Eire"), + ("Etc/GMT", "Etc/GMT"), + ("Etc/GMT+0", "Etc/GMT+0"), + ("Etc/GMT+1", "Etc/GMT+1"), + ("Etc/GMT+10", "Etc/GMT+10"), + ("Etc/GMT+11", "Etc/GMT+11"), + ("Etc/GMT+12", "Etc/GMT+12"), + ("Etc/GMT+2", "Etc/GMT+2"), + ("Etc/GMT+3", "Etc/GMT+3"), + ("Etc/GMT+4", "Etc/GMT+4"), + ("Etc/GMT+5", "Etc/GMT+5"), + ("Etc/GMT+6", "Etc/GMT+6"), + ("Etc/GMT+7", "Etc/GMT+7"), + ("Etc/GMT+8", "Etc/GMT+8"), + ("Etc/GMT+9", "Etc/GMT+9"), + ("Etc/GMT-0", "Etc/GMT-0"), + ("Etc/GMT-1", "Etc/GMT-1"), + ("Etc/GMT-10", "Etc/GMT-10"), + ("Etc/GMT-11", "Etc/GMT-11"), + ("Etc/GMT-12", "Etc/GMT-12"), + ("Etc/GMT-13", "Etc/GMT-13"), + ("Etc/GMT-14", "Etc/GMT-14"), + ("Etc/GMT-2", "Etc/GMT-2"), + ("Etc/GMT-3", "Etc/GMT-3"), + ("Etc/GMT-4", "Etc/GMT-4"), + ("Etc/GMT-5", "Etc/GMT-5"), + ("Etc/GMT-6", "Etc/GMT-6"), + ("Etc/GMT-7", "Etc/GMT-7"), + ("Etc/GMT-8", "Etc/GMT-8"), + ("Etc/GMT-9", "Etc/GMT-9"), + ("Etc/GMT0", "Etc/GMT0"), + ("Etc/Greenwich", "Etc/Greenwich"), + ("Etc/UCT", "Etc/UCT"), + ("Etc/UTC", "Etc/UTC"), + ("Etc/Universal", "Etc/Universal"), + ("Etc/Zulu", "Etc/Zulu"), + ("Europe/Amsterdam", "Europe/Amsterdam"), + ("Europe/Andorra", "Europe/Andorra"), + ("Europe/Astrakhan", "Europe/Astrakhan"), + ("Europe/Athens", "Europe/Athens"), + ("Europe/Belfast", "Europe/Belfast"), + ("Europe/Belgrade", "Europe/Belgrade"), + ("Europe/Berlin", "Europe/Berlin"), + ("Europe/Bratislava", "Europe/Bratislava"), + ("Europe/Brussels", "Europe/Brussels"), + ("Europe/Bucharest", "Europe/Bucharest"), + ("Europe/Budapest", "Europe/Budapest"), + ("Europe/Busingen", "Europe/Busingen"), + ("Europe/Chisinau", "Europe/Chisinau"), + ("Europe/Copenhagen", "Europe/Copenhagen"), + ("Europe/Dublin", "Europe/Dublin"), + ("Europe/Gibraltar", "Europe/Gibraltar"), + ("Europe/Guernsey", "Europe/Guernsey"), + ("Europe/Helsinki", "Europe/Helsinki"), + ("Europe/Isle_of_Man", "Europe/Isle_of_Man"), + ("Europe/Istanbul", "Europe/Istanbul"), + ("Europe/Jersey", "Europe/Jersey"), + ("Europe/Kaliningrad", "Europe/Kaliningrad"), + ("Europe/Kiev", "Europe/Kiev"), + ("Europe/Kirov", "Europe/Kirov"), + ("Europe/Kyiv", "Europe/Kyiv"), + ("Europe/Lisbon", "Europe/Lisbon"), + ("Europe/Ljubljana", "Europe/Ljubljana"), + ("Europe/London", "Europe/London"), + ("Europe/Luxembourg", "Europe/Luxembourg"), + ("Europe/Madrid", "Europe/Madrid"), + ("Europe/Malta", "Europe/Malta"), + ("Europe/Mariehamn", "Europe/Mariehamn"), + ("Europe/Minsk", "Europe/Minsk"), + ("Europe/Monaco", "Europe/Monaco"), + ("Europe/Moscow", "Europe/Moscow"), + ("Europe/Nicosia", "Europe/Nicosia"), + ("Europe/Oslo", "Europe/Oslo"), + ("Europe/Paris", "Europe/Paris"), + ("Europe/Podgorica", "Europe/Podgorica"), + ("Europe/Prague", "Europe/Prague"), + ("Europe/Riga", "Europe/Riga"), + ("Europe/Rome", "Europe/Rome"), + ("Europe/Samara", "Europe/Samara"), + ("Europe/San_Marino", "Europe/San_Marino"), + ("Europe/Sarajevo", "Europe/Sarajevo"), + ("Europe/Saratov", "Europe/Saratov"), + ("Europe/Simferopol", "Europe/Simferopol"), + ("Europe/Skopje", "Europe/Skopje"), + ("Europe/Sofia", "Europe/Sofia"), + ("Europe/Stockholm", "Europe/Stockholm"), + ("Europe/Tallinn", "Europe/Tallinn"), + ("Europe/Tirane", "Europe/Tirane"), + ("Europe/Tiraspol", "Europe/Tiraspol"), + ("Europe/Ulyanovsk", "Europe/Ulyanovsk"), + ("Europe/Uzhgorod", "Europe/Uzhgorod"), + ("Europe/Vaduz", "Europe/Vaduz"), + ("Europe/Vatican", "Europe/Vatican"), + ("Europe/Vienna", "Europe/Vienna"), + ("Europe/Vilnius", "Europe/Vilnius"), + ("Europe/Volgograd", "Europe/Volgograd"), + ("Europe/Warsaw", "Europe/Warsaw"), + ("Europe/Zagreb", "Europe/Zagreb"), + ("Europe/Zaporozhye", "Europe/Zaporozhye"), + ("Europe/Zurich", "Europe/Zurich"), + ("Factory", "Factory"), + ("GB", "GB"), + ("GB-Eire", "GB-Eire"), + ("GMT", "GMT"), + ("GMT+0", "GMT+0"), + ("GMT-0", "GMT-0"), + ("GMT0", "GMT0"), + ("Greenwich", "Greenwich"), + ("HST", "HST"), + ("Hongkong", "Hongkong"), + ("Iceland", "Iceland"), + ("Indian/Antananarivo", "Indian/Antananarivo"), + ("Indian/Chagos", "Indian/Chagos"), + ("Indian/Christmas", "Indian/Christmas"), + ("Indian/Cocos", "Indian/Cocos"), + ("Indian/Comoro", "Indian/Comoro"), + ("Indian/Kerguelen", "Indian/Kerguelen"), + ("Indian/Mahe", "Indian/Mahe"), + ("Indian/Maldives", "Indian/Maldives"), + ("Indian/Mauritius", "Indian/Mauritius"), + ("Indian/Mayotte", "Indian/Mayotte"), + ("Indian/Reunion", "Indian/Reunion"), + ("Iran", "Iran"), + ("Israel", "Israel"), + ("Jamaica", "Jamaica"), + ("Japan", "Japan"), + ("Kwajalein", "Kwajalein"), + ("Libya", "Libya"), + ("MET", "MET"), + ("MST", "MST"), + ("MST7MDT", "MST7MDT"), + ("Mexico/BajaNorte", "Mexico/BajaNorte"), + ("Mexico/BajaSur", "Mexico/BajaSur"), + ("Mexico/General", "Mexico/General"), + ("NZ", "NZ"), + ("NZ-CHAT", "NZ-CHAT"), + ("Navajo", "Navajo"), + ("PRC", "PRC"), + ("PST8PDT", "PST8PDT"), + ("Pacific/Apia", "Pacific/Apia"), + ("Pacific/Auckland", "Pacific/Auckland"), + ("Pacific/Bougainville", "Pacific/Bougainville"), + ("Pacific/Chatham", "Pacific/Chatham"), + ("Pacific/Chuuk", "Pacific/Chuuk"), + ("Pacific/Easter", "Pacific/Easter"), + ("Pacific/Efate", "Pacific/Efate"), + ("Pacific/Enderbury", "Pacific/Enderbury"), + ("Pacific/Fakaofo", "Pacific/Fakaofo"), + ("Pacific/Fiji", "Pacific/Fiji"), + ("Pacific/Funafuti", "Pacific/Funafuti"), + ("Pacific/Galapagos", "Pacific/Galapagos"), + ("Pacific/Gambier", "Pacific/Gambier"), + ("Pacific/Guadalcanal", "Pacific/Guadalcanal"), + ("Pacific/Guam", "Pacific/Guam"), + ("Pacific/Honolulu", "Pacific/Honolulu"), + ("Pacific/Johnston", "Pacific/Johnston"), + ("Pacific/Kanton", "Pacific/Kanton"), + ("Pacific/Kiritimati", "Pacific/Kiritimati"), + ("Pacific/Kosrae", "Pacific/Kosrae"), + ("Pacific/Kwajalein", "Pacific/Kwajalein"), + ("Pacific/Majuro", "Pacific/Majuro"), + ("Pacific/Marquesas", "Pacific/Marquesas"), + ("Pacific/Midway", "Pacific/Midway"), + ("Pacific/Nauru", "Pacific/Nauru"), + ("Pacific/Niue", "Pacific/Niue"), + ("Pacific/Norfolk", "Pacific/Norfolk"), + ("Pacific/Noumea", "Pacific/Noumea"), + ("Pacific/Pago_Pago", "Pacific/Pago_Pago"), + ("Pacific/Palau", "Pacific/Palau"), + ("Pacific/Pitcairn", "Pacific/Pitcairn"), + ("Pacific/Pohnpei", "Pacific/Pohnpei"), + ("Pacific/Ponape", "Pacific/Ponape"), + ("Pacific/Port_Moresby", "Pacific/Port_Moresby"), + ("Pacific/Rarotonga", "Pacific/Rarotonga"), + ("Pacific/Saipan", "Pacific/Saipan"), + ("Pacific/Samoa", "Pacific/Samoa"), + ("Pacific/Tahiti", "Pacific/Tahiti"), + ("Pacific/Tarawa", "Pacific/Tarawa"), + ("Pacific/Tongatapu", "Pacific/Tongatapu"), + ("Pacific/Truk", "Pacific/Truk"), + ("Pacific/Wake", "Pacific/Wake"), + ("Pacific/Wallis", "Pacific/Wallis"), + ("Pacific/Yap", "Pacific/Yap"), + ("Poland", "Poland"), + ("Portugal", "Portugal"), + ("ROC", "ROC"), + ("ROK", "ROK"), + ("Singapore", "Singapore"), + ("Turkey", "Turkey"), + ("UCT", "UCT"), + ("US/Alaska", "US/Alaska"), + ("US/Aleutian", "US/Aleutian"), + ("US/Arizona", "US/Arizona"), + ("US/Central", "US/Central"), + ("US/East-Indiana", "US/East-Indiana"), + ("US/Eastern", "US/Eastern"), + ("US/Hawaii", "US/Hawaii"), + ("US/Indiana-Starke", "US/Indiana-Starke"), + ("US/Michigan", "US/Michigan"), + ("US/Mountain", "US/Mountain"), + ("US/Pacific", "US/Pacific"), + ("US/Samoa", "US/Samoa"), + ("UTC", "UTC"), + ("Universal", "Universal"), + ("W-SU", "W-SU"), + ("WET", "WET"), + ("Zulu", "Zulu"), + ("localtime", "localtime"), + ], + default="UTC", + max_length=255, + ), + ), + ] diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index e2671b07f..6ef4b6e77 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -1,5 +1,7 @@ """ database schema for user data """ +import datetime import re +import zoneinfo from urllib.parse import urlparse from uuid import uuid4 @@ -12,7 +14,6 @@ from django.db import models, transaction, IntegrityError from django.utils import timezone from django.utils.translation import gettext_lazy as _ from model_utils import FieldTracker -import pytz from bookwyrm import activitypub from bookwyrm.connectors import get_data, ConnectorException @@ -165,8 +166,8 @@ class User(OrderedCollectionPageMixin, AbstractUser): summary_keys = models.JSONField(null=True) preferred_timezone = models.CharField( - choices=[(str(tz), str(tz)) for tz in pytz.all_timezones], - default=str(pytz.utc), + choices=[(str(tz), str(tz)) for tz in sorted(zoneinfo.available_timezones())], + default=str(datetime.timezone.utc), max_length=255, ) preferred_language = models.CharField( diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index 79d58085c..81169ff10 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -2,7 +2,6 @@ import pathlib from unittest.mock import patch import datetime -import pytz from django.test import TestCase @@ -13,7 +12,7 @@ from bookwyrm.models.import_job import handle_imported_book def make_date(*args): """helper function to easily generate a date obj""" - return datetime.datetime(*args, tzinfo=pytz.UTC) + return datetime.datetime(*args, tzinfo=datetime.timezone.utc) @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index 39aac22ff..da2e1b3d8 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -3,7 +3,6 @@ from collections import namedtuple import pathlib from unittest.mock import patch import datetime -import pytz from django.test import TestCase import responses @@ -16,7 +15,7 @@ from bookwyrm.models.import_job import handle_imported_book def make_date(*args): """helper function to easily generate a date obj""" - return datetime.datetime(*args, tzinfo=pytz.UTC) + return datetime.datetime(*args, tzinfo=datetime.timezone.utc) @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index 4d78d242a..6b145e2d6 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -2,7 +2,6 @@ import pathlib from unittest.mock import patch import datetime -import pytz from django.test import TestCase @@ -13,7 +12,7 @@ from bookwyrm.models.import_job import handle_imported_book def make_date(*args): """helper function to easily generate a date obj""" - return datetime.datetime(*args, tzinfo=pytz.UTC) + return datetime.datetime(*args, tzinfo=datetime.timezone.utc) @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") diff --git a/bookwyrm/tests/importers/test_openlibrary_import.py b/bookwyrm/tests/importers/test_openlibrary_import.py index 8f2f120ff..ceb83762d 100644 --- a/bookwyrm/tests/importers/test_openlibrary_import.py +++ b/bookwyrm/tests/importers/test_openlibrary_import.py @@ -2,7 +2,6 @@ import pathlib from unittest.mock import patch import datetime -import pytz from django.test import TestCase @@ -13,7 +12,7 @@ from bookwyrm.models.import_job import handle_imported_book def make_date(*args): """helper function to easily generate a date obj""" - return datetime.datetime(*args, tzinfo=pytz.UTC) + return datetime.datetime(*args, tzinfo=datetime.timezone.utc) @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index 3de2b13a0..859760085 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -2,7 +2,6 @@ import pathlib from unittest.mock import patch import datetime -import pytz from django.test import TestCase @@ -13,7 +12,7 @@ from bookwyrm.models.import_job import handle_imported_book def make_date(*args): """helper function to easily generate a date obj""" - return datetime.datetime(*args, tzinfo=pytz.UTC) + return datetime.datetime(*args, tzinfo=datetime.timezone.utc) @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") diff --git a/bookwyrm/tests/views/test_annual_summary.py b/bookwyrm/tests/views/test_annual_summary.py index f5bd60085..1ee4322f7 100644 --- a/bookwyrm/tests/views/test_annual_summary.py +++ b/bookwyrm/tests/views/test_annual_summary.py @@ -1,7 +1,6 @@ """testing the annual summary page""" import datetime from unittest.mock import patch -import pytz from django.contrib.auth.models import AnonymousUser from django.http import Http404 @@ -15,7 +14,7 @@ from bookwyrm.tests.validate_html import validate_html def make_date(*args): """helper function to easily generate a date obj""" - return datetime.datetime(*args, tzinfo=pytz.UTC) + return datetime.datetime(*args, tzinfo=datetime.timezone.utc) class AnnualSummary(TestCase): diff --git a/bookwyrm/views/landing/register.py b/bookwyrm/views/landing/register.py index 26d8e1813..9d9aedb50 100644 --- a/bookwyrm/views/landing/register.py +++ b/bookwyrm/views/landing/register.py @@ -1,5 +1,5 @@ """ class views for login/register views """ -import pytz +import zoneinfo from django.contrib.auth import login from django.core.exceptions import PermissionDenied from django.shortcuts import get_object_or_404, redirect @@ -57,9 +57,11 @@ class Register(View): email = form.data["email"] password = form.data["password"] try: - preferred_timezone = pytz.timezone(form.data.get("preferred_timezone")) - except pytz.exceptions.UnknownTimeZoneError: - preferred_timezone = pytz.utc + preferred_timezone = zoneinfo.ZoneInfo( + form.data.get("preferred_timezone", "") + ) + except (ValueError, zoneinfo.ZoneInfoNotFoundError): + preferred_timezone = zoneinfo.ZoneInfo("UTC") # make sure the email isn't blocked as spam email_domain = email.split("@")[-1] diff --git a/requirements.txt b/requirements.txt index 27b43873b..7f52a64f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,6 @@ psycopg2==2.9.9 pycryptodome==3.20.0 pyotp==2.9.0 python-dateutil==2.9.0.post0 -pytz>=2022.7 qrcode==7.4.2 redis==5.0.3 requests==2.31.0 From 2896219e8812a5588a9e2ccffbc522fb20e3094f Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 22:07:01 +0200 Subject: [PATCH 43/56] Switch from django-redis to the built-in Redis cache backend --- bookwyrm/settings.py | 5 +---- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 61b45c423..74bd363e5 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -257,11 +257,8 @@ if env.bool("USE_DUMMY_CACHE", False): else: CACHES = { "default": { - "BACKEND": "django_redis.cache.RedisCache", + "BACKEND": "django.core.cache.backends.redis.RedisCache", "LOCATION": REDIS_ACTIVITY_URL, - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - }, }, "file_resubmit": { "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", diff --git a/requirements.txt b/requirements.txt index 7f52a64f1..a867b4c02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,12 +12,12 @@ django-imagekit==5.0.0 django-model-utils==4.4.0 django-oauth-toolkit==2.3.0 django-pgtrigger==4.11.0 -django-redis==5.2.0 django-sass-processor==1.4 django-storages==1.14.2 django-storages[azure] environs==11.0.0 flower==2.0.1 +hiredis==2.3.2 libsass==0.23.0 Markdown==3.6 opentelemetry-api==1.24.0 From 051dab77bb08f4b0e9ba264e73f9b94b3ac499f6 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 1 Apr 2024 22:53:37 +0200 Subject: [PATCH 44/56] Replace deprecated CICharField with custom collation for case-insensitivity --- ...alter_hashtag_name_alter_user_localname.py | 39 +++++++++++++++++++ bookwyrm/models/hashtag.py | 5 ++- bookwyrm/models/user.py | 7 ++-- 3 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py diff --git a/bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py b/bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py new file mode 100644 index 000000000..4fe41ec35 --- /dev/null +++ b/bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.11 on 2024-04-01 21:09 + +import bookwyrm.models.fields +from django.db import migrations, models +from django.contrib.postgres.operations import CreateCollation + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0200_alter_user_preferred_timezone"), + ] + + operations = [ + CreateCollation( + "case_insensitive", + provider="icu", + locale="und-u-ks-level2", + deterministic=False, + ), + migrations.AlterField( + model_name="hashtag", + name="name", + field=bookwyrm.models.fields.CharField( + db_collation="case_insensitive", max_length=256 + ), + ), + migrations.AlterField( + model_name="user", + name="localname", + field=models.CharField( + db_collation="case_insensitive", + max_length=255, + null=True, + unique=True, + validators=[bookwyrm.models.fields.validate_localname], + ), + ), + ] diff --git a/bookwyrm/models/hashtag.py b/bookwyrm/models/hashtag.py index 7894a3528..5126f012d 100644 --- a/bookwyrm/models/hashtag.py +++ b/bookwyrm/models/hashtag.py @@ -2,18 +2,19 @@ from bookwyrm import activitypub from .activitypub_mixin import ActivitypubMixin from .base_model import BookWyrmModel -from .fields import CICharField +from .fields import CharField class Hashtag(ActivitypubMixin, BookWyrmModel): "a hashtag which can be used in statuses" - name = CICharField( + name = CharField( max_length=256, blank=False, null=False, activitypub_field="name", deduplication_field=True, + db_collation="case_insensitive", ) name_field = "name" diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 6ef4b6e77..73f1b28c6 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -7,7 +7,7 @@ from uuid import uuid4 from django.apps import apps from django.contrib.auth.models import AbstractUser -from django.contrib.postgres.fields import ArrayField, CICharField +from django.contrib.postgres.fields import ArrayField as DjangoArrayField from django.core.exceptions import PermissionDenied, ObjectDoesNotExist from django.dispatch import receiver from django.db import models, transaction, IntegrityError @@ -76,11 +76,12 @@ class User(OrderedCollectionPageMixin, AbstractUser): summary = fields.HtmlField(null=True, blank=True) local = models.BooleanField(default=False) bookwyrm_user = fields.BooleanField(default=True) - localname = CICharField( + localname = models.CharField( max_length=255, null=True, unique=True, validators=[fields.validate_localname], + db_collation="case_insensitive", ) # name is your display name, which you can change at will name = fields.CharField(max_length=100, null=True, blank=True) @@ -157,7 +158,7 @@ class User(OrderedCollectionPageMixin, AbstractUser): show_guided_tour = models.BooleanField(default=True) # feed options - feed_status_types = ArrayField( + feed_status_types = DjangoArrayField( models.CharField(max_length=10, blank=False, choices=FeedFilterChoices), size=8, default=get_feed_filter_choices, From f38622fdc9ffa047f2f45d953197646c1c714e97 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Tue, 2 Apr 2024 12:38:53 +0200 Subject: [PATCH 45/56] Define CSRF_TRUSTED_ORIGINS --- bookwyrm/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 74bd363e5..c075c9c87 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -366,6 +366,7 @@ if (USE_HTTPS and PORT == 443) or (not USE_HTTPS and PORT == 80): else: NETLOC = f"{DOMAIN}:{PORT}" BASE_URL = f"{PROTOCOL}://{NETLOC}" +CSRF_TRUSTED_ORIGINS = [BASE_URL] USER_AGENT = f"BookWyrm (BookWyrm/{VERSION}; +{BASE_URL})" From de67c732378eae668c3a06d760378b525cd4885d Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 10 Apr 2024 22:23:35 +0200 Subject: [PATCH 46/56] Add merge migration --- bookwyrm/migrations/0205_merge_20240410_2022.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 bookwyrm/migrations/0205_merge_20240410_2022.py diff --git a/bookwyrm/migrations/0205_merge_20240410_2022.py b/bookwyrm/migrations/0205_merge_20240410_2022.py new file mode 100644 index 000000000..294f48487 --- /dev/null +++ b/bookwyrm/migrations/0205_merge_20240410_2022.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.11 on 2024-04-10 20:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0201_alter_hashtag_name_alter_user_localname"), + ("bookwyrm", "0204_merge_20240409_1042"), + ] + + operations = [] From 77832cbec70642b2d2336d793f09a5f7f23f8415 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 15 Apr 2024 17:39:11 +0200 Subject: [PATCH 47/56] Add merge migration --- bookwyrm/migrations/0206_merge_20240415_1537.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 bookwyrm/migrations/0206_merge_20240415_1537.py diff --git a/bookwyrm/migrations/0206_merge_20240415_1537.py b/bookwyrm/migrations/0206_merge_20240415_1537.py new file mode 100644 index 000000000..454e69880 --- /dev/null +++ b/bookwyrm/migrations/0206_merge_20240415_1537.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.11 on 2024-04-15 15:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0205_merge_20240410_2022"), + ("bookwyrm", "0205_merge_20240413_0232"), + ] + + operations = [] From 7604d0acdb5e1bcabbd1a040f26ada9b0964d7d8 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Thu, 25 Apr 2024 10:31:24 +0200 Subject: [PATCH 48/56] Simplify ObjectMixin broadcast kwarg --- bookwyrm/models/activitypub_mixin.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index db737b8bc..0015c5fe1 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -206,14 +206,10 @@ class ObjectMixin(ActivitypubMixin): created: Optional[bool] = None, software: Any = None, priority: str = BROADCAST, + broadcast: bool = True, **kwargs: Any, ) -> None: """broadcast created/updated/deleted objects as appropriate""" - broadcast = kwargs.get("broadcast", True) - # this bonus kwarg would cause an error in the base save method - if "broadcast" in kwargs: - del kwargs["broadcast"] - created = created or not bool(self.id) # first off, we want to save normally no matter what super().save(*args, **kwargs) From a6c2ce15ddfbaca966c8a33f814e4c258e027365 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Thu, 25 Apr 2024 15:51:32 +0200 Subject: [PATCH 49/56] Early return --- bookwyrm/models/move.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/bookwyrm/models/move.py b/bookwyrm/models/move.py index b6b8655df..5038058b7 100644 --- a/bookwyrm/models/move.py +++ b/bookwyrm/models/move.py @@ -48,24 +48,21 @@ class MoveUser(Move): """update user info and broadcast it""" # only allow if the source is listed in the target's alsoKnownAs - if self.user in self.target.also_known_as.all(): - self.user.also_known_as.add(self.target.id) - self.user.update_active_date() - self.user.moved_to = self.target.remote_id - self.user.save(update_fields=["moved_to"]) - - if self.user.local: - kwargs[ - "broadcast" - ] = True # Only broadcast if we are initiating the Move - - super().save(*args, **kwargs) - - for follower in self.user.followers.all(): - if follower.local: - Notification.notify( - follower, self.user, notification_type=NotificationType.MOVE - ) - - else: + if self.user not in self.target.also_known_as.all(): raise PermissionDenied() + + self.user.also_known_as.add(self.target.id) + self.user.update_active_date() + self.user.moved_to = self.target.remote_id + self.user.save(update_fields=["moved_to"]) + + if self.user.local: + kwargs["broadcast"] = True # Only broadcast if we are initiating the Move + + super().save(*args, **kwargs) + + for follower in self.user.followers.all(): + if follower.local: + Notification.notify( + follower, self.user, notification_type=NotificationType.MOVE + ) From e7f95ef4c2054385f58f84cc3f93e486ad4e387b Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Thu, 25 Apr 2024 15:53:53 +0200 Subject: [PATCH 50/56] Modify update_fields in save() when modifying objects https://docs.djangoproject.com/en/5.0/releases/4.2/#setting-update-fields-in-model-save-may-now-be-required --- bookwyrm/models/author.py | 6 +++--- bookwyrm/models/book.py | 38 ++++++++++++++++++++++++---------- bookwyrm/models/link.py | 14 +++++++++---- bookwyrm/models/list.py | 8 +++++-- bookwyrm/models/readthrough.py | 9 ++++++-- bookwyrm/models/shelf.py | 15 +++++++++++--- bookwyrm/models/site.py | 7 +++++-- bookwyrm/models/status.py | 10 +++++---- bookwyrm/models/user.py | 29 ++++++++++++++++++++------ bookwyrm/utils/db.py | 14 ++++++++++++- 10 files changed, 112 insertions(+), 38 deletions(-) diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 8ea1858fd..20c4e9e00 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -1,7 +1,7 @@ """ database schema for info about authors """ import re -from typing import Tuple, Any +from typing import Any from django.db import models from django.contrib.postgres.indexes import GinIndex @@ -45,9 +45,9 @@ class Author(BookDataModel): ) bio = fields.HtmlField(null=True, blank=True) - def save(self, *args: Tuple[Any, ...], **kwargs: dict[str, Any]) -> None: + def save(self, *args: Any, **kwargs: Any) -> None: """normalize isni format""" - if self.isni: + if self.isni is not None: self.isni = re.sub(r"\s", "", self.isni) super().save(*args, **kwargs) diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 6fc447228..2e6377575 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -2,7 +2,7 @@ from itertools import chain import re -from typing import Any, Dict +from typing import Any, Dict, Optional, Iterable from typing_extensions import Self from django.contrib.postgres.search import SearchVectorField @@ -27,7 +27,7 @@ from bookwyrm.settings import ( ENABLE_PREVIEW_IMAGES, ENABLE_THUMBNAIL_GENERATION, ) -from bookwyrm.utils.db import format_trigger +from bookwyrm.utils.db import format_trigger, add_update_fields from .activitypub_mixin import OrderedCollectionPageMixin, ObjectMixin from .base_model import BookWyrmModel @@ -96,14 +96,19 @@ class BookDataModel(ObjectMixin, BookWyrmModel): abstract = True - def save(self, *args: Any, **kwargs: Any) -> None: + def save( + self, *args: Any, update_fields: Optional[Iterable[str]] = None, **kwargs: Any + ) -> None: """ensure that the remote_id is within this instance""" if self.id: self.remote_id = self.get_remote_id() + update_fields = add_update_fields(update_fields, "remote_id") else: self.origin_id = self.remote_id self.remote_id = None - super().save(*args, **kwargs) + update_fields = add_update_fields(update_fields, "origin_id", "remote_id") + + super().save(*args, update_fields=update_fields, **kwargs) # pylint: disable=arguments-differ def broadcast(self, activity, sender, software="bookwyrm", **kwargs): @@ -510,28 +515,39 @@ class Edition(Book): # max rank is 9 return rank - def save(self, *args: Any, **kwargs: Any) -> None: + def save( + self, *args: Any, update_fields: Optional[Iterable[str]] = None, **kwargs: Any + ) -> None: """set some fields on the edition object""" # calculate isbn 10/13 - if self.isbn_13 and self.isbn_13[:3] == "978" and not self.isbn_10: + if ( + self.isbn_10 is None + and self.isbn_13 is not None + and self.isbn_13[:3] == "978" + ): self.isbn_10 = isbn_13_to_10(self.isbn_13) - if self.isbn_10 and not self.isbn_13: + update_fields = add_update_fields(update_fields, "isbn_10") + if self.isbn_13 is None and self.isbn_10 is not None: self.isbn_13 = isbn_10_to_13(self.isbn_10) + update_fields = add_update_fields(update_fields, "isbn_13") # normalize isbn format - if self.isbn_10: + if self.isbn_10 is not None: self.isbn_10 = normalize_isbn(self.isbn_10) - if self.isbn_13: + if self.isbn_13 is not None: self.isbn_13 = normalize_isbn(self.isbn_13) # set rank - self.edition_rank = self.get_rank() + if (new := self.get_rank()) != self.edition_rank: + self.edition_rank = new + update_fields = add_update_fields(update_fields, "edition_rank") # Create sort title by removing articles from title if self.sort_title in [None, ""]: self.sort_title = self.guess_sort_title() + update_fields = add_update_fields(update_fields, "sort_title") - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) # clear author cache if self.id: diff --git a/bookwyrm/models/link.py b/bookwyrm/models/link.py index 67bf9c4d4..4519f0c81 100644 --- a/bookwyrm/models/link.py +++ b/bookwyrm/models/link.py @@ -1,4 +1,5 @@ """ outlink data """ +from typing import Optional, Iterable from urllib.parse import urlparse from django.core.exceptions import PermissionDenied @@ -6,6 +7,7 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from bookwyrm import activitypub +from bookwyrm.utils.db import add_update_fields from .activitypub_mixin import ActivitypubMixin from .base_model import BookWyrmModel from . import fields @@ -34,17 +36,19 @@ class Link(ActivitypubMixin, BookWyrmModel): """link name via the associated domain""" return self.domain.name - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """create a link""" # get or create the associated domain if not self.domain: domain = urlparse(self.url).hostname self.domain, _ = LinkDomain.objects.get_or_create(domain=domain) + update_fields = add_update_fields(update_fields, "domain") # this is never broadcast, the owning model broadcasts an update if "broadcast" in kwargs: del kwargs["broadcast"] - return super().save(*args, **kwargs) + + super().save(*args, update_fields=update_fields, **kwargs) AvailabilityChoices = [ @@ -88,8 +92,10 @@ class LinkDomain(BookWyrmModel): return raise PermissionDenied() - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """set a default name""" if not self.name: self.name = self.domain - super().save(*args, **kwargs) + update_fields = add_update_fields(update_fields, "name") + + super().save(*args, update_fields=update_fields, **kwargs) diff --git a/bookwyrm/models/list.py b/bookwyrm/models/list.py index d32a8da95..df7e8162c 100644 --- a/bookwyrm/models/list.py +++ b/bookwyrm/models/list.py @@ -1,4 +1,5 @@ """ make a list of books!! """ +from typing import Optional, Iterable import uuid from django.core.exceptions import PermissionDenied @@ -8,6 +9,7 @@ from django.utils import timezone from bookwyrm import activitypub from bookwyrm.settings import BASE_URL +from bookwyrm.utils.db import add_update_fields from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin from .base_model import BookWyrmModel @@ -124,11 +126,13 @@ class List(OrderedCollectionMixin, BookWyrmModel): group=None, curation="closed" ) - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """on save, update embed_key and avoid clash with existing code""" if not self.embed_key: self.embed_key = uuid.uuid4() - super().save(*args, **kwargs) + update_fields = add_update_fields(update_fields, "embed_key") + + super().save(*args, update_fields=update_fields, **kwargs) class ListItem(CollectionItemMixin, BookWyrmModel): diff --git a/bookwyrm/models/readthrough.py b/bookwyrm/models/readthrough.py index 910b2a7a9..7700b4a87 100644 --- a/bookwyrm/models/readthrough.py +++ b/bookwyrm/models/readthrough.py @@ -1,9 +1,13 @@ """ progress in a book """ +from typing import Optional, Iterable + from django.core import validators from django.core.cache import cache from django.db import models from django.db.models import F, Q +from bookwyrm.utils.db import add_update_fields + from .base_model import BookWyrmModel @@ -30,13 +34,14 @@ class ReadThrough(BookWyrmModel): stopped_date = models.DateTimeField(blank=True, null=True) is_active = models.BooleanField(default=True) - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """update user active time""" # an active readthrough must have an unset finish date if self.finish_date or self.stopped_date: self.is_active = False + update_fields = add_update_fields(update_fields, "is_active") - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) cache.delete(f"latest_read_through-{self.user_id}-{self.book_id}") self.user.update_active_date() diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py index 77c2d26d9..0b9ef2b09 100644 --- a/bookwyrm/models/shelf.py +++ b/bookwyrm/models/shelf.py @@ -1,5 +1,6 @@ """ puttin' books on shelves """ import re +from typing import Optional, Iterable from django.core.cache import cache from django.core.exceptions import PermissionDenied from django.db import models @@ -8,6 +9,7 @@ from django.utils import timezone from bookwyrm import activitypub from bookwyrm.settings import BASE_URL from bookwyrm.tasks import BROADCAST +from bookwyrm.utils.db import add_update_fields from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin from .base_model import BookWyrmModel from . import fields @@ -46,7 +48,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): if not self.identifier: # this needs the auto increment ID from the save() above self.identifier = self.get_identifier() - super().save(*args, **kwargs, broadcast=False) + super().save(*args, **kwargs, broadcast=False, update_fields={"identifier"}) def get_identifier(self): """custom-shelf-123 for the url""" @@ -101,12 +103,19 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel): activity_serializer = activitypub.ShelfItem collection_field = "shelf" - def save(self, *args, priority=BROADCAST, **kwargs): + def save( + self, + *args, + priority=BROADCAST, + update_fields: Optional[Iterable[str]] = None, + **kwargs, + ): if not self.user: self.user = self.shelf.user + update_fields = add_update_fields(update_fields, "user") is_update = self.id is not None - super().save(*args, priority=priority, **kwargs) + super().save(*args, priority=priority, update_fields=update_fields, **kwargs) if is_update and self.user.local: # remove all caches related to all editions of this book diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index 89d6ef395..6c2a73422 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -1,5 +1,6 @@ """ the particulars for this instance of BookWyrm """ import datetime +from typing import Optional, Iterable from urllib.parse import urljoin import uuid @@ -15,6 +16,7 @@ from bookwyrm.preview_images import generate_site_preview_image_task from bookwyrm.settings import BASE_URL, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL from bookwyrm.settings import RELEASE_API from bookwyrm.tasks import app, MISC +from bookwyrm.utils.db import add_update_fields from .base_model import BookWyrmModel, new_access_code from .user import User from .fields import get_absolute_url @@ -136,13 +138,14 @@ class SiteSettings(SiteModel): return get_absolute_url(uploaded) return urljoin(STATIC_FULL_URL, default_path) - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """if require_confirm_email is disabled, make sure no users are pending, if enabled, make sure invite_question_text is not empty""" if not self.invite_question_text: self.invite_question_text = "What is your favourite book?" + update_fields = add_update_fields(update_fields, "invite_question_text") - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) if not self.require_confirm_email: User.objects.filter(is_active=False, deactivation_reason="pending").update( diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 5b953d077..9dc60e647 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -1,6 +1,6 @@ """ models for storing different kinds of Activities """ from dataclasses import MISSING -from typing import Optional +from typing import Optional, Iterable import re from django.apps import apps @@ -20,6 +20,7 @@ from model_utils.managers import InheritanceManager from bookwyrm import activitypub from bookwyrm.preview_images import generate_edition_preview_image_task from bookwyrm.settings import ENABLE_PREVIEW_IMAGES +from bookwyrm.utils.db import add_update_fields from .activitypub_mixin import ActivitypubMixin, ActivityMixin from .activitypub_mixin import OrderedCollectionPageMixin from .base_model import BookWyrmModel @@ -85,12 +86,13 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): models.Index(fields=["thread_id"]), ] - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """save and notify""" - if self.reply_parent: + if self.thread_id is None and self.reply_parent: self.thread_id = self.reply_parent.thread_id or self.reply_parent_id + update_fields = add_update_fields(update_fields, "thread_id") - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) if not self.reply_parent: self.thread_id = self.id diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 73f1b28c6..d5120deac 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -2,6 +2,7 @@ import datetime import re import zoneinfo +from typing import Optional, Iterable from urllib.parse import urlparse from uuid import uuid4 @@ -24,6 +25,7 @@ from bookwyrm.settings import BASE_URL, ENABLE_PREVIEW_IMAGES, LANGUAGES from bookwyrm.signatures import create_key_pair from bookwyrm.tasks import app, MISC from bookwyrm.utils import regex +from bookwyrm.utils.db import add_update_fields from .activitypub_mixin import OrderedCollectionPageMixin, ActivitypubMixin from .base_model import BookWyrmModel, DeactivationReason, new_access_code from .federated_server import FederatedServer @@ -338,13 +340,14 @@ class User(OrderedCollectionPageMixin, AbstractUser): ] return activity_object - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """populate fields for new local users""" created = not bool(self.id) if not self.local and not re.match(regex.FULL_USERNAME, self.username): # generate a username that uses the domain (webfinger format) actor_parts = urlparse(self.remote_id) self.username = f"{self.username}@{actor_parts.hostname}" + update_fields = add_update_fields(update_fields, "username") # this user already exists, no need to populate fields if not created: @@ -353,12 +356,12 @@ class User(OrderedCollectionPageMixin, AbstractUser): elif not self.deactivation_date: self.deactivation_date = timezone.now() - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) return # this is a new remote user, we need to set their remote server field if not self.local: - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) transaction.on_commit(lambda: set_remote_server(self.id)) return @@ -370,8 +373,17 @@ class User(OrderedCollectionPageMixin, AbstractUser): self.shared_inbox = f"{BASE_URL}/inbox" self.outbox = f"{self.remote_id}/outbox" + update_fields = add_update_fields( + update_fields, + "remote_id", + "followers_url", + "inbox", + "shared_inbox", + "outbox", + ) + # an id needs to be set before we can proceed with related models - super().save(*args, **kwargs) + super().save(*args, update_fields=update_fields, **kwargs) # make users editors by default try: @@ -522,14 +534,19 @@ class KeyPair(ActivitypubMixin, BookWyrmModel): # self.owner is set by the OneToOneField on User return f"{self.owner.remote_id}/#main-key" - def save(self, *args, **kwargs): + def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): """create a key pair""" # no broadcasting happening here if "broadcast" in kwargs: del kwargs["broadcast"] + if not self.public_key: self.private_key, self.public_key = create_key_pair() - return super().save(*args, **kwargs) + update_fields = add_update_fields( + update_fields, "private_key", "public_key" + ) + + super().save(*args, update_fields=update_fields, **kwargs) @app.task(queue=MISC) diff --git a/bookwyrm/utils/db.py b/bookwyrm/utils/db.py index 2bb3b9ff6..fd2601deb 100644 --- a/bookwyrm/utils/db.py +++ b/bookwyrm/utils/db.py @@ -1,6 +1,6 @@ """ Database utilities """ -from typing import cast +from typing import Optional, Iterable, Set, cast import sqlparse # type: ignore @@ -21,3 +21,15 @@ def format_trigger(sql: str) -> str: identifier_case="lower", ), ) + + +def add_update_fields( + update_fields: Optional[Iterable[str]], *fields: str +) -> Optional[Set[str]]: + """ + Helper for adding fields to the update_fields kwarg when modifying an object + in a model's save() method. + + https://docs.djangoproject.com/en/5.0/releases/4.2/#setting-update-fields-in-model-save-may-now-be-required + """ + return set(fields).union(update_fields) if update_fields is not None else None From c32f9faaa0d6a1af47e24ccff91d3c65e6bfcefe Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 26 Apr 2024 13:41:01 +0200 Subject: [PATCH 51/56] Upgrade pylint to 2.17.7 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a867b4c02..615fdc824 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,7 @@ black==22.* celery-types==0.22.0 django-stubs[compatible-mypy]==4.2.7 mypy==1.7.1 -pylint==2.15.0 +pylint==2.17.7 pytest==8.1.1 pytest-cov==5.0.0 pytest-django==4.8.0 From acae063652ca180561150ebf7b379f294be4e88f Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 26 Apr 2024 13:59:16 +0200 Subject: [PATCH 52/56] Fix new warnings from pylint upgrade --- bookwyrm/activitypub/base_activity.py | 10 +++++----- bookwyrm/importers/calibre_import.py | 13 ++++--------- bookwyrm/models/activitypub_mixin.py | 2 +- bookwyrm/templatetags/utilities.py | 16 ++++++++-------- bookwyrm/tests/validate_html.py | 4 ++-- bookwyrm/views/annual_summary.py | 2 +- 6 files changed, 21 insertions(+), 26 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index efc9d8da2..4ddc8eb9a 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -400,11 +400,11 @@ def get_representative(): to sign outgoing HTTP GET requests""" return models.User.objects.get_or_create( username=f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}", - defaults=dict( - email="bookwyrm@localhost", - local=True, - localname=INSTANCE_ACTOR_USERNAME, - ), + defaults={ + "email": "bookwyrm@localhost", + "local": True, + "localname": INSTANCE_ACTOR_USERNAME, + }, )[0] diff --git a/bookwyrm/importers/calibre_import.py b/bookwyrm/importers/calibre_import.py index 5c22a539d..542175dd7 100644 --- a/bookwyrm/importers/calibre_import.py +++ b/bookwyrm/importers/calibre_import.py @@ -14,15 +14,10 @@ class CalibreImporter(Importer): def __init__(self, *args: Any, **kwargs: Any): # Add timestamp to row_mappings_guesses for date_added to avoid # integrity error - row_mappings_guesses = [] - - for field, mapping in self.row_mappings_guesses: - if field in ("date_added",): - row_mappings_guesses.append((field, mapping + ["timestamp"])) - else: - row_mappings_guesses.append((field, mapping)) - - self.row_mappings_guesses = row_mappings_guesses + self.row_mappings_guesses = [ + (field, mapping + (["timestamp"] if field == "date_added" else [])) + for field, mapping in self.row_mappings_guesses + ] super().__init__(*args, **kwargs) def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]: diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index 0015c5fe1..06ef373e6 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -169,7 +169,7 @@ class ActivitypubMixin: # filter users first by whether they're using the desired software # this lets us send book updates only to other bw servers if software: - queryset = queryset.filter(bookwyrm_user=(software == "bookwyrm")) + queryset = queryset.filter(bookwyrm_user=software == "bookwyrm") # if there's a user, we only want to send to the user's followers if user: queryset = queryset.filter(following=user) diff --git a/bookwyrm/templatetags/utilities.py b/bookwyrm/templatetags/utilities.py index bc87a60d7..ab597a22a 100644 --- a/bookwyrm/templatetags/utilities.py +++ b/bookwyrm/templatetags/utilities.py @@ -137,14 +137,14 @@ def get_file_size(nbytes): raw_size = float(nbytes) except (ValueError, TypeError): return repr(nbytes) - else: - if raw_size < 1024: - return f"{raw_size} bytes" - if raw_size < 1024**2: - return f"{raw_size/1024:.2f} KB" - if raw_size < 1024**3: - return f"{raw_size/1024**2:.2f} MB" - return f"{raw_size/1024**3:.2f} GB" + + if raw_size < 1024: + return f"{raw_size} bytes" + if raw_size < 1024**2: + return f"{raw_size/1024:.2f} KB" + if raw_size < 1024**3: + return f"{raw_size/1024**2:.2f} MB" + return f"{raw_size/1024**3:.2f} GB" @register.filter(name="get_user_permission") diff --git a/bookwyrm/tests/validate_html.py b/bookwyrm/tests/validate_html.py index 748b94d5f..11bc84880 100644 --- a/bookwyrm/tests/validate_html.py +++ b/bookwyrm/tests/validate_html.py @@ -35,7 +35,7 @@ def validate_html(html): e for e in errors.split("\n") if not any(exclude in e for exclude in excluded) ) if errors: - raise Exception(errors) + raise ValueError(errors) validator = HtmlValidator() # will raise exceptions @@ -62,6 +62,6 @@ class HtmlValidator(HTMLParser): # pylint: disable=abstract-method and "noreferrer" in value ): return - raise Exception( + raise ValueError( 'Links to a new tab must have rel="nofollow noopener noreferrer"' ) diff --git a/bookwyrm/views/annual_summary.py b/bookwyrm/views/annual_summary.py index 703a2d2ab..21ac53992 100644 --- a/bookwyrm/views/annual_summary.py +++ b/bookwyrm/views/annual_summary.py @@ -225,4 +225,4 @@ def get_goal_status(user, year): if goal.privacy != "public": return None - return dict(**goal.progress, **{"goal": goal.goal}) + return {**goal.progress, **{"goal": goal.goal}} From 29f852b57e5cdfec2184e498c30fe7537e027572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Fri, 26 Apr 2024 15:36:11 -0300 Subject: [PATCH 53/56] consolidate multiple cache.delete() calls into cache.delete_many() --- bookwyrm/models/book.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 2e6377575..4ff377dbb 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -551,8 +551,12 @@ class Edition(Book): # clear author cache if self.id: - for author_id in self.authors.values_list("id", flat=True): - cache.delete(f"author-books-{author_id}") + cache.delete_many( + [ + f"author-books-{author_id}" + for author_id in self.authors.values_list("id", flat=True) + ] + ) @transaction.atomic def repair(self): From e6ee169c3e1dec189a2d7d34d66d2902ee327ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Wed, 3 Apr 2024 20:36:40 -0300 Subject: [PATCH 54/56] Narrow down bare `type: ignore` pragmas --- bookwyrm/utils/db.py | 2 +- bookwyrm/utils/partial_date.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bookwyrm/utils/db.py b/bookwyrm/utils/db.py index fd2601deb..7024d9e47 100644 --- a/bookwyrm/utils/db.py +++ b/bookwyrm/utils/db.py @@ -1,7 +1,7 @@ """ Database utilities """ from typing import Optional, Iterable, Set, cast -import sqlparse # type: ignore +import sqlparse # type: ignore[import-untyped] def format_trigger(sql: str) -> str: diff --git a/bookwyrm/utils/partial_date.py b/bookwyrm/utils/partial_date.py index 4c9391476..d5b3cabc3 100644 --- a/bookwyrm/utils/partial_date.py +++ b/bookwyrm/utils/partial_date.py @@ -222,17 +222,17 @@ class PartialDateDescriptor: return [("DAY", "Day prec."), ("MONTH", "Month prec."), ("YEAR", "Year prec.")] -class PartialDateModel(models.DateTimeField): # type: ignore +class PartialDateModel(models.DateTimeField): # type: ignore[type-arg] """a date field for Django models, using PartialDate as values""" descriptor_class = PartialDateDescriptor - def formfield(self, **kwargs): # type: ignore + def formfield(self, **kwargs): # type: ignore[no-untyped-def] kwargs.setdefault("form_class", PartialDateFormField) return super().formfield(**kwargs) # pylint: disable-next=arguments-renamed - def contribute_to_class(self, model, our_name_in_model, **kwargs): # type: ignore + def contribute_to_class(self, model, our_name_in_model, **kwargs): # type: ignore[no-untyped-def] # Define precision field. descriptor = self.descriptor_class(self) precision: models.Field[Optional[str], Optional[str]] = models.CharField( From 61d9e0c260d0981904f880cb555b6898dd7a7ae4 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 31 May 2024 16:49:34 +0200 Subject: [PATCH 55/56] Move comment to separate line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adeodato Simó <73768+dato@users.noreply.github.com> --- bookwyrm/views/preferences/export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index 58c77b14c..32e7db06e 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -247,7 +247,8 @@ class ExportArchive(View): export.export_data, content_type="application/gzip", headers={ - "Content-Disposition": 'attachment; filename="bookwyrm-account-export.tar.gz"' # pylint: disable=line-too-long + # pylint: disable=line-too-long + "Content-Disposition": 'attachment; filename="bookwyrm-account-export.tar.gz"' }, ) except FileNotFoundError: From eca246fc61e1c3c178734f525ad7af0f3822a51e Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 31 May 2024 16:59:24 +0200 Subject: [PATCH 56/56] Fix lint --- bookwyrm/utils/partial_date.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/utils/partial_date.py b/bookwyrm/utils/partial_date.py index d5b3cabc3..d20fc315e 100644 --- a/bookwyrm/utils/partial_date.py +++ b/bookwyrm/utils/partial_date.py @@ -231,7 +231,7 @@ class PartialDateModel(models.DateTimeField): # type: ignore[type-arg] kwargs.setdefault("form_class", PartialDateFormField) return super().formfield(**kwargs) - # pylint: disable-next=arguments-renamed + # pylint: disable-next=arguments-renamed,line-too-long def contribute_to_class(self, model, our_name_in_model, **kwargs): # type: ignore[no-untyped-def] # Define precision field. descriptor = self.descriptor_class(self)