From 8c34b23c35cbb32bfa2b02a8835cb879b87e3182 Mon Sep 17 00:00:00 2001 From: Joachim Date: Sat, 17 Dec 2022 11:39:10 +0100 Subject: [PATCH 1/8] Disable preview images generation for remote users --- bookwyrm/preview_images.py | 3 +++ bookwyrm/tests/test_preview_images.py | 24 +++++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/bookwyrm/preview_images.py b/bookwyrm/preview_images.py index d20145cd3..7c108237d 100644 --- a/bookwyrm/preview_images.py +++ b/bookwyrm/preview_images.py @@ -459,6 +459,9 @@ def generate_user_preview_image_task(user_id): user = models.User.objects.get(id=user_id) + if not user.local: + return + texts = { "text_one": user.display_name, "text_three": f"@{user.localname}@{settings.DOMAIN}", diff --git a/bookwyrm/tests/test_preview_images.py b/bookwyrm/tests/test_preview_images.py index 79ee195d7..3d248a541 100644 --- a/bookwyrm/tests/test_preview_images.py +++ b/bookwyrm/tests/test_preview_images.py @@ -46,6 +46,19 @@ class PreviewImages(TestCase): ), ) + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ), patch("bookwyrm.lists_stream.populate_lists_task.delay"): + self.remote_user = models.User.objects.create_user( + "rat", + "rat@rat.com", + "ratword", + local=False, + remote_id="https://example.com/users/rat", + inbox="https://example.com/users/rat/inbox", + outbox="https://example.com/users/rat/outbox", + ) + self.work = models.Work.objects.create(title="Test Work") self.edition = models.Edition.objects.create( title="Example Edition", @@ -114,7 +127,7 @@ class PreviewImages(TestCase): self.local_user.refresh_from_db() self.assertIsInstance(self.local_user.preview_image, ImageFieldFile) - self.assertIsNotNone(self.local_user.preview_image) + self.assertTrue(self.local_user.preview_image) self.assertEqual( self.local_user.preview_image.width, settings.PREVIEW_IMG_WIDTH ) @@ -122,6 +135,15 @@ class PreviewImages(TestCase): self.local_user.preview_image.height, settings.PREVIEW_IMG_HEIGHT ) + def test_remote_user_preview(self, *args, **kwargs): + """a remote user doesn’t get a user preview""" + generate_user_preview_image_task(self.remote_user.id) + + self.remote_user.refresh_from_db() + + self.assertFalse(self.remote_user.preview_image) + + def test_generate_user_preview_images_task(self, *args, **kwargs): """test task's external calls""" with patch("bookwyrm.preview_images.generate_preview_image") as generate_mock: From 09c095cfec30df8b41dc8254590f402f6af32217 Mon Sep 17 00:00:00 2001 From: Joachim Date: Sat, 17 Dec 2022 17:34:27 +0100 Subject: [PATCH 2/8] Add remove_remote_user_preview_images command --- .../remove_remote_user_preview_images.py | 40 +++++++++++++++++++ bookwyrm/preview_images.py | 22 ++++++++++ bw-dev | 4 ++ 3 files changed, 66 insertions(+) create mode 100644 bookwyrm/management/commands/remove_remote_user_preview_images.py diff --git a/bookwyrm/management/commands/remove_remote_user_preview_images.py b/bookwyrm/management/commands/remove_remote_user_preview_images.py new file mode 100644 index 000000000..a298e152c --- /dev/null +++ b/bookwyrm/management/commands/remove_remote_user_preview_images.py @@ -0,0 +1,40 @@ +""" Generate preview images """ +from django.core.management.base import BaseCommand +from django.db.models import Q + +from bookwyrm import models, preview_images + + +# pylint: disable=line-too-long +class Command(BaseCommand): + """Remove preview images for remote users""" + + help = "Remove preview images for remote users" + + # pylint: disable=no-self-use,unused-argument + def handle(self, *args, **options): + """generate preview images""" + self.stdout.write( + " | Hello! I will be removing preview images from remote users." + ) + self.stdout.write( + "πŸ§‘β€πŸš’ ⎨ This might take quite long if your instance has a lot of remote users." + ) + self.stdout.write(" | ✧ Thank you for your patience ✧") + + users = models.User.objects.filter(local=False).exclude(Q(preview_image='')|Q(preview_image=None)) + + if len(users) > 0: + self.stdout.write( + f" β†’ Remote user preview images ({len(users)}): ", ending="" + ) + for user in users: + preview_images.remove_user_preview_image_task.delay(user.id) + self.stdout.write(".", ending="") + self.stdout.write(" OK πŸ–Ό") + else: + self.stdout.write( + f" | There was no remote users with preview images." + ) + + self.stdout.write("πŸ§‘β€πŸš’ ⎨ I’m all done! ✧ Enjoy ✧") diff --git a/bookwyrm/preview_images.py b/bookwyrm/preview_images.py index 7c108237d..1a3e7f152 100644 --- a/bookwyrm/preview_images.py +++ b/bookwyrm/preview_images.py @@ -475,3 +475,25 @@ def generate_user_preview_image_task(user_id): image = generate_preview_image(texts=texts, picture=avatar) save_and_cleanup(image, instance=user) + + +@app.task(queue=LOW) +def remove_user_preview_image_task(user_id): + """remove preview_image for a user""" + if not settings.ENABLE_PREVIEW_IMAGES: + return + + user = models.User.objects.get(id=user_id) + + try: + file_name = user.preview_image.name + except ValueError: + file_name = None + + # Delete image in model + user.preview_image.delete(save=False) + user.save(broadcast=False, update_fields=["preview_image"]) + + # Delete image file + if file_name and default_storage.exists(file_name): + default_storage.delete(file_name) diff --git a/bw-dev b/bw-dev index ef5dec813..28510d15c 100755 --- a/bw-dev +++ b/bw-dev @@ -223,6 +223,9 @@ case "$CMD" in generate_preview_images) runweb python manage.py generate_preview_images "$@" ;; + remove_remote_user_preview_images) + runweb python manage.py remove_remote_user_preview_images + ;; copy_media_to_s3) awscommand "bookwyrm_media_volume:/images"\ "s3 cp /images s3://${AWS_STORAGE_BUCKET_NAME}/images\ @@ -300,6 +303,7 @@ case "$CMD" in echo " populate_suggestions" echo " generate_thumbnails" echo " generate_preview_images [--all]" + echo " remove_remote_user_preview_images" echo " copy_media_to_s3" echo " sync_media_to_s3" echo " set_cors_to_s3 [cors file]" From 75ea2cdbb4cdde41e130f51880332dd4d396fe5e Mon Sep 17 00:00:00 2001 From: Joachim Date: Sat, 17 Dec 2022 17:34:49 +0100 Subject: [PATCH 3/8] Fix generate_user_preview_image_task comment wording --- bookwyrm/preview_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/preview_images.py b/bookwyrm/preview_images.py index 1a3e7f152..8f27b47c4 100644 --- a/bookwyrm/preview_images.py +++ b/bookwyrm/preview_images.py @@ -453,7 +453,7 @@ def generate_edition_preview_image_task(book_id): @app.task(queue=LOW) def generate_user_preview_image_task(user_id): - """generate preview_image for a book""" + """generate preview_image for a user""" if not settings.ENABLE_PREVIEW_IMAGES: return From b570602a479212cac0a35d3293b465c965cbee26 Mon Sep 17 00:00:00 2001 From: Joachim Date: Sat, 17 Dec 2022 17:35:44 +0100 Subject: [PATCH 4/8] Update remove_remote_user_preview_images.py --- .../management/commands/remove_remote_user_preview_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/management/commands/remove_remote_user_preview_images.py b/bookwyrm/management/commands/remove_remote_user_preview_images.py index a298e152c..f940d4812 100644 --- a/bookwyrm/management/commands/remove_remote_user_preview_images.py +++ b/bookwyrm/management/commands/remove_remote_user_preview_images.py @@ -1,4 +1,4 @@ -""" Generate preview images """ +""" Remove preview images for remote users """ from django.core.management.base import BaseCommand from django.db.models import Q From cd940e2c407f52495f9ccbe7a74c1f9004cb4f9a Mon Sep 17 00:00:00 2001 From: Joachim Date: Sat, 17 Dec 2022 17:49:39 +0100 Subject: [PATCH 5/8] Add tests for remove_user_preview_image_task --- bookwyrm/tests/test_preview_images.py | 30 ++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/bookwyrm/tests/test_preview_images.py b/bookwyrm/tests/test_preview_images.py index 3d248a541..967dda7d5 100644 --- a/bookwyrm/tests/test_preview_images.py +++ b/bookwyrm/tests/test_preview_images.py @@ -14,6 +14,7 @@ from bookwyrm.preview_images import ( generate_edition_preview_image_task, generate_user_preview_image_task, generate_preview_image, + remove_user_preview_image_task, save_and_cleanup, ) @@ -59,6 +60,24 @@ class PreviewImages(TestCase): outbox="https://example.com/users/rat/outbox", ) + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ), patch("bookwyrm.lists_stream.populate_lists_task.delay"): + self.remote_user_with_preview = models.User.objects.create_user( + "badger@your.domain.here", + "badger@badger.com", + "badgeword", + local=False, + remote_id="https://example.com/users/badger", + inbox="https://example.com/users/badger/inbox", + outbox="https://example.com/users/badger/outbox", + avatar=SimpleUploadedFile( + avatar_file, + open(avatar_file, "rb").read(), + content_type="image/jpeg", + ), + ) + self.work = models.Work.objects.create(title="Test Work") self.edition = models.Edition.objects.create( title="Example Edition", @@ -127,7 +146,7 @@ class PreviewImages(TestCase): self.local_user.refresh_from_db() self.assertIsInstance(self.local_user.preview_image, ImageFieldFile) - self.assertTrue(self.local_user.preview_image) + self.assertIsNotNone(self.local_user.preview_image) self.assertEqual( self.local_user.preview_image.width, settings.PREVIEW_IMG_WIDTH ) @@ -151,3 +170,12 @@ class PreviewImages(TestCase): args = generate_mock.call_args.kwargs self.assertEqual(args["texts"]["text_one"], "possum") self.assertEqual(args["texts"]["text_three"], f"@possum@{settings.DOMAIN}") + + + def test_remove_user_preview_image_task(self, *args, **kwargs): + """you can delete the preview image for a (remote) user""" + remove_user_preview_image_task(self.remote_user_with_preview.id) + + self.remote_user_with_preview.refresh_from_db() + + self.assertFalse(self.remote_user_with_preview.preview_image) From 00c2930290ba7a495167701ffb4a9efaadbf398c Mon Sep 17 00:00:00 2001 From: Joachim Date: Sat, 17 Dec 2022 17:56:28 +0100 Subject: [PATCH 6/8] Black --- .../commands/remove_remote_user_preview_images.py | 8 ++++---- bookwyrm/tests/test_preview_images.py | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bookwyrm/management/commands/remove_remote_user_preview_images.py b/bookwyrm/management/commands/remove_remote_user_preview_images.py index f940d4812..d4dc131d8 100644 --- a/bookwyrm/management/commands/remove_remote_user_preview_images.py +++ b/bookwyrm/management/commands/remove_remote_user_preview_images.py @@ -22,7 +22,9 @@ class Command(BaseCommand): ) self.stdout.write(" | ✧ Thank you for your patience ✧") - users = models.User.objects.filter(local=False).exclude(Q(preview_image='')|Q(preview_image=None)) + users = models.User.objects.filter(local=False).exclude( + Q(preview_image="") | Q(preview_image=None) + ) if len(users) > 0: self.stdout.write( @@ -33,8 +35,6 @@ class Command(BaseCommand): self.stdout.write(".", ending="") self.stdout.write(" OK πŸ–Ό") else: - self.stdout.write( - f" | There was no remote users with preview images." - ) + self.stdout.write(f" | There was no remote users with preview images.") self.stdout.write("πŸ§‘β€πŸš’ ⎨ I’m all done! ✧ Enjoy ✧") diff --git a/bookwyrm/tests/test_preview_images.py b/bookwyrm/tests/test_preview_images.py index 967dda7d5..d1998bf3c 100644 --- a/bookwyrm/tests/test_preview_images.py +++ b/bookwyrm/tests/test_preview_images.py @@ -162,7 +162,6 @@ class PreviewImages(TestCase): self.assertFalse(self.remote_user.preview_image) - def test_generate_user_preview_images_task(self, *args, **kwargs): """test task's external calls""" with patch("bookwyrm.preview_images.generate_preview_image") as generate_mock: @@ -171,7 +170,6 @@ class PreviewImages(TestCase): self.assertEqual(args["texts"]["text_one"], "possum") self.assertEqual(args["texts"]["text_three"], f"@possum@{settings.DOMAIN}") - def test_remove_user_preview_image_task(self, *args, **kwargs): """you can delete the preview image for a (remote) user""" remove_user_preview_image_task(self.remote_user_with_preview.id) From e9abae9b7db7e34ef295ca2bf82c045433d1b885 Mon Sep 17 00:00:00 2001 From: Joachim Date: Sat, 17 Dec 2022 18:05:33 +0100 Subject: [PATCH 7/8] Add shell completion for remove_remote_user_preview_images command --- complete_bwdev.fish | 74 +++++++++++++++++++++++---------------------- complete_bwdev.sh | 1 + complete_bwdev.zsh | 1 + 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/complete_bwdev.fish b/complete_bwdev.fish index e680ede43..399ebc17b 100644 --- a/complete_bwdev.fish +++ b/complete_bwdev.fish @@ -31,6 +31,7 @@ populate_lists_streams \ populate_suggestions \ generate_thumbnails \ generate_preview_images \ +remove_remote_user_preview_images \ copy_media_to_s3 \ set_cors_to_s3 \ setup \ @@ -43,42 +44,43 @@ function __bw_complete -a cmds cmd desc complete -f -c bw-dev -n "not __fish_seen_subcommand_from $cmds" -a $cmd -d $desc end -__bw_complete "$commands" "up" "bring one or all service(s) up" -__bw_complete "$commands" "service_ports_web" "run command on the web container with its portsenabled and mapped" -__bw_complete "$commands" "initdb" "initialize database" -__bw_complete "$commands" "resetdb" "!! WARNING !! reset database" -__bw_complete "$commands" "makemigrations" "create new migrations" -__bw_complete "$commands" "migrate" "perform all migrations" -__bw_complete "$commands" "bash" "open up bash within the web container" -__bw_complete "$commands" "shell" "open the Python shell within the web container" -__bw_complete "$commands" "dbshell" "open the database shell within the web container" -__bw_complete "$commands" "restart_celery" "restart the celery container" -__bw_complete "$commands" "pytest" "run unit tests" -__bw_complete "$commands" "collectstatic" "copy changed static files into the installation" -__bw_complete "$commands" "makemessages" "extract all localizable messages from the code" -__bw_complete "$commands" "compilemessages" "compile .po localization files to .mo" -__bw_complete "$commands" "update_locales" "run makemessages and compilemessages for the en_US and additional locales" -__bw_complete "$commands" "build" "build the containers" -__bw_complete "$commands" "clean" "bring the cluster down and remove all containers" -__bw_complete "$commands" "black" "run Python code formatting tool" -__bw_complete "$commands" "prettier" "run JavaScript code formatting tool" -__bw_complete "$commands" "eslint" "run JavaScript linting tool" -__bw_complete "$commands" "stylelint" "run SCSS linting tool" -__bw_complete "$commands" "formatters" "run multiple formatter tools" -__bw_complete "$commands" "populate_streams" "populate the main streams" -__bw_complete "$commands" "populate_lists_streams" "populate streams for book lists" -__bw_complete "$commands" "populate_suggestions" "populate book suggestions" -__bw_complete "$commands" "generate_thumbnails" "generate book thumbnails" -__bw_complete "$commands" "generate_preview_images" "generate book preview images" -__bw_complete "$commands" "collectstatic_watch" "watch filesystem and copy changed static files" -__bw_complete "$commands" "copy_media_to_s3" "run the `s3 cp` command to copy media to a bucket on S3" -__bw_complete "$commands" "sync_media_to_s3" "run the `s3 sync` command to sync media with a bucket on S3" -__bw_complete "$commands" "set_cors_to_s3" "push a CORS configuration defined in .json to s3" -__bw_complete "$commands" "setup" "perform first-time setup" -__bw_complete "$commands" "admin_code" "get the admin code" -__bw_complete "$commands" "remove_2fa" "remove 2FA from user" -__bw_complete "$commands" "confirm_email" "manually confirm email of user and set active" -__bw_complete "$commands" "runweb" "run a command on the web container" +__bw_complete "$commands" "up" "bring one or all service(s) up" +__bw_complete "$commands" "service_ports_web" "run command on the web container with its portsenabled and mapped" +__bw_complete "$commands" "initdb" "initialize database" +__bw_complete "$commands" "resetdb" "!! WARNING !! reset database" +__bw_complete "$commands" "makemigrations" "create new migrations" +__bw_complete "$commands" "migrate" "perform all migrations" +__bw_complete "$commands" "bash" "open up bash within the web container" +__bw_complete "$commands" "shell" "open the Python shell within the web container" +__bw_complete "$commands" "dbshell" "open the database shell within the web container" +__bw_complete "$commands" "restart_celery" "restart the celery container" +__bw_complete "$commands" "pytest" "run unit tests" +__bw_complete "$commands" "collectstatic" "copy changed static files into the installation" +__bw_complete "$commands" "makemessages" "extract all localizable messages from the code" +__bw_complete "$commands" "compilemessages" "compile .po localization files to .mo" +__bw_complete "$commands" "update_locales" "run makemessages and compilemessages for the en_US and additional locales" +__bw_complete "$commands" "build" "build the containers" +__bw_complete "$commands" "clean" "bring the cluster down and remove all containers" +__bw_complete "$commands" "black" "run Python code formatting tool" +__bw_complete "$commands" "prettier" "run JavaScript code formatting tool" +__bw_complete "$commands" "eslint" "run JavaScript linting tool" +__bw_complete "$commands" "stylelint" "run SCSS linting tool" +__bw_complete "$commands" "formatters" "run multiple formatter tools" +__bw_complete "$commands" "populate_streams" "populate the main streams" +__bw_complete "$commands" "populate_lists_streams" "populate streams for book lists" +__bw_complete "$commands" "populate_suggestions" "populate book suggestions" +__bw_complete "$commands" "generate_thumbnails" "generate book thumbnails" +__bw_complete "$commands" "generate_preview_images" "generate site/book/user preview images" +__bw_complete "$commands" "remove_remote_user_preview_images" "remove preview images for remote users" +__bw_complete "$commands" "collectstatic_watch" "watch filesystem and copy changed static files" +__bw_complete "$commands" "copy_media_to_s3" "run the `s3 cp` command to copy media to a bucket on S3" +__bw_complete "$commands" "sync_media_to_s3" "run the `s3 sync` command to sync media with a bucket on S3" +__bw_complete "$commands" "set_cors_to_s3" "push a CORS configuration defined in .json to s3" +__bw_complete "$commands" "setup" "perform first-time setup" +__bw_complete "$commands" "admin_code" "get the admin code" +__bw_complete "$commands" "remove_2fa" "remove 2FA from user" +__bw_complete "$commands" "confirm_email" "manually confirm email of user and set active" +__bw_complete "$commands" "runweb" "run a command on the web container" function __bw_complete_subcommand -a cmd diff --git a/complete_bwdev.sh b/complete_bwdev.sh index 7976b2126..6e632173b 100644 --- a/complete_bwdev.sh +++ b/complete_bwdev.sh @@ -28,6 +28,7 @@ populate_lists_streams populate_suggestions generate_thumbnails generate_preview_images +remove_remote_user_preview_images copy_media_to_s3 set_cors_to_s3 setup diff --git a/complete_bwdev.zsh b/complete_bwdev.zsh index e9c8028c1..512218b5a 100644 --- a/complete_bwdev.zsh +++ b/complete_bwdev.zsh @@ -30,6 +30,7 @@ populate_lists_streams populate_suggestions generate_thumbnails generate_preview_images +remove_remote_user_preview_images copy_media_to_s3 set_cors_to_s3 setup From a44f427d84641c67d0bd1654592c8ed891a2cd7a Mon Sep 17 00:00:00 2001 From: Joachim Date: Mon, 19 Dec 2022 22:26:09 +0100 Subject: [PATCH 8/8] Add remote user test before the task is called --- bookwyrm/models/user.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 7df9da88d..f2a546737 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -526,6 +526,11 @@ def preview_image(instance, *args, **kwargs): """create preview images when user is updated""" if not ENABLE_PREVIEW_IMAGES: return + + # don't call the task for remote users + if not instance.local: + return + changed_fields = instance.field_tracker.changed() if len(changed_fields) > 0: