From a4bfcb34d5c2e53448700c5970b2c3634fc1245c Mon Sep 17 00:00:00 2001 From: Hugh Rundle Date: Sun, 15 Oct 2023 15:09:19 +1100 Subject: [PATCH 1/6] fix tests and clean up * cleans up some test logging * cleans up some commented-out code * adds export_job model tests * reconsiders some tests in export user view tests --- bookwyrm/models/bookwyrm_export_job.py | 8 - .../templates/preferences/export-user.html | 16 +- .../tests/models/test_bookwyrm_export_job.py | 233 ++++++++++++++++++ ...t_model.py => test_bookwyrm_import_job.py} | 27 +- .../views/preferences/test_export_user.py | 23 -- 5 files changed, 261 insertions(+), 46 deletions(-) create mode 100644 bookwyrm/tests/models/test_bookwyrm_export_job.py rename bookwyrm/tests/models/{test_bookwyrm_import_model.py => test_bookwyrm_import_job.py} (96%) diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index c262d9b5c..c3a0b652a 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -153,20 +153,12 @@ def json_export(user): comments = models.Comment.objects.filter(user=user, book=book["id"]).distinct() book["comments"] = list(comments.values()) - logger.error("FINAL COMMENTS") - logger.error(book["comments"]) # quotes quotes = models.Quotation.objects.filter(user=user, book=book["id"]).distinct() - # quote_statuses = models.Status.objects.filter( - # id__in=quotes, user=kwargs["user"] - # ).distinct() book["quotes"] = list(quotes.values()) - logger.error("FINAL QUOTES") - logger.error(book["quotes"]) - # append everything final_books.append(book) diff --git a/bookwyrm/templates/preferences/export-user.html b/bookwyrm/templates/preferences/export-user.html index 81f13bc22..393d8990e 100644 --- a/bookwyrm/templates/preferences/export-user.html +++ b/bookwyrm/templates/preferences/export-user.html @@ -12,15 +12,13 @@

{% trans "Your exported archive file will include all user data for import into another Bookwyrm server" %}

-

-

- {% csrf_token %} - -
-

+
+ {% csrf_token %} + +

{% trans "Recent Exports" %}

diff --git a/bookwyrm/tests/models/test_bookwyrm_export_job.py b/bookwyrm/tests/models/test_bookwyrm_export_job.py new file mode 100644 index 000000000..bd314e60e --- /dev/null +++ b/bookwyrm/tests/models/test_bookwyrm_export_job.py @@ -0,0 +1,233 @@ +import datetime +import time +import json +from unittest.mock import patch + +from django.core.serializers.json import DjangoJSONEncoder +from django.test import TestCase +from django.utils import timezone + +from bookwyrm import models +import bookwyrm.models.bookwyrm_export_job as export_job + + +class BookwyrmExport(TestCase): + """testing user export functions""" + + def setUp(self): + """lots of stuff to set up for a user export""" + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ), patch("bookwyrm.lists_stream.populate_lists_task.delay"), patch( + "bookwyrm.suggested_users.rerank_user_task.delay" + ), patch( + "bookwyrm.lists_stream.remove_list_task.delay" + ), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch( + "bookwyrm.activitystreams.add_book_statuses_task" + ): + + self.local_user = models.User.objects.create_user( + "mouse", + "mouse@mouse.mouse", + "password", + local=True, + localname="mouse", + name="Mouse", + summary="I'm a real bookmouse", + manually_approves_followers=False, + hide_follows=False, + show_goal=False, + show_suggested_users=False, + discoverable=True, + preferred_timezone="America/Los Angeles", + default_post_privacy="followers", + ) + + self.rat_user = models.User.objects.create_user( + "rat", "rat@rat.rat", "ratword", local=True, localname="rat" + ) + + self.badger_user = models.User.objects.create_user( + "badger", + "badger@badger.badger", + "badgerword", + local=True, + localname="badger", + ) + + models.AnnualGoal.objects.create( + user=self.local_user, + year=timezone.now().year, + goal=128937123, + privacy="followers", + ) + + self.list = models.List.objects.create( + name="My excellent list", + user=self.local_user, + remote_id="https://local.lists/1111", + ) + + self.saved_list = models.List.objects.create( + name="My cool list", + user=self.rat_user, + remote_id="https://local.lists/9999", + ) + + self.local_user.saved_lists.add(self.saved_list) + self.local_user.blocks.add(self.badger_user) + self.rat_user.followers.add(self.local_user) + + # book, edition, author + self.author = models.Author.objects.create(name="Sam Zhu") + self.work = models.Work.objects.create( + title="Example Work", remote_id="https://example.com/book/1" + ) + self.edition = models.Edition.objects.create( + title="Example Edition", parent_work=self.work + ) + + self.edition.authors.add(self.author) + + # readthrough + self.readthrough_start = timezone.now() + finish = self.readthrough_start + datetime.timedelta(days=1) + models.ReadThrough.objects.create( + user=self.local_user, + book=self.edition, + start_date=self.readthrough_start, + finish_date=finish, + ) + + # shelve + read_shelf = models.Shelf.objects.get( + user=self.local_user, identifier="read" + ) + models.ShelfBook.objects.create( + book=self.edition, shelf=read_shelf, user=self.local_user + ) + + # add to list + item = models.ListItem.objects.create( + book_list=self.list, + user=self.local_user, + book=self.edition, + approved=True, + order=1, + ) + + # review + models.Review.objects.create( + content="awesome", + name="my review", + rating=5, + user=self.local_user, + book=self.edition, + ) + # comment + models.Comment.objects.create( + content="ok so far", + user=self.local_user, + book=self.edition, + progress=15, + ) + # quote + models.Quotation.objects.create( + content="check this out", + quote="A rose by any other name", + user=self.local_user, + book=self.edition, + ) + + def test_json_export_user_settings(self): + """Test the json export function for basic user info""" + data = export_job.json_export(self.local_user) + user_data = json.loads(data)["user"] + self.assertEqual(user_data["username"], "mouse") + self.assertEqual(user_data["name"], "Mouse") + self.assertEqual(user_data["summary"], "I'm a real bookmouse") + self.assertEqual(user_data["manually_approves_followers"], False) + self.assertEqual(user_data["hide_follows"], False) + self.assertEqual(user_data["show_goal"], False) + self.assertEqual(user_data["show_suggested_users"], False) + self.assertEqual(user_data["discoverable"], True) + self.assertEqual(user_data["preferred_timezone"], "America/Los Angeles") + self.assertEqual(user_data["default_post_privacy"], "followers") + + def test_json_export_extended_user_data(self): + """Test the json export function for other non-book user info""" + data = export_job.json_export(self.local_user) + json_data = json.loads(data) + + # goal + self.assertEqual(len(json_data["goals"]), 1) + self.assertEqual(json_data["goals"][0]["goal"], 128937123) + self.assertEqual(json_data["goals"][0]["year"], timezone.now().year) + self.assertEqual(json_data["goals"][0]["privacy"], "followers") + + # saved lists + self.assertEqual(len(json_data["saved_lists"]), 1) + self.assertEqual(json_data["saved_lists"][0], "https://local.lists/9999") + + # follows + self.assertEqual(len(json_data["follows"]), 1) + self.assertEqual(json_data["follows"][0], "https://your.domain.here/user/rat") + # blocked users + self.assertEqual(len(json_data["blocked_users"]), 1) + self.assertEqual( + json_data["blocked_users"][0], "https://your.domain.here/user/badger" + ) + + def test_json_export_books(self): + """Test the json export function for extended user info""" + + data = export_job.json_export(self.local_user) + json_data = json.loads(data) + start_date = json_data["books"][0]["readthroughs"][0]["start_date"] + + self.assertEqual(len(json_data["books"]), 1) + self.assertEqual(json_data["books"][0]["title"], "Example Edition") + self.assertEqual(len(json_data["books"][0]["authors"]), 1) + self.assertEqual(json_data["books"][0]["authors"][0]["name"], "Sam Zhu") + self.assertEqual( + f'"{start_date}"', DjangoJSONEncoder().encode(self.readthrough_start) + ) + self.assertEqual(json_data["books"][0]["shelves"][0]["identifier"], "read") + self.assertEqual( + json_data["books"][0]["shelf_books"]["read"][0]["book_id"], self.edition.id + ) + + self.assertEqual(len(json_data["books"][0]["lists"]), 1) + self.assertEqual(json_data["books"][0]["lists"][0]["name"], "My excellent list") + self.assertEqual(len(json_data["books"][0]["list_items"]), 1) + self.assertEqual( + json_data["books"][0]["list_items"]["My excellent list"][0]["book_id"], + self.edition.id, + ) + + self.assertEqual(len(json_data["books"][0]["reviews"]), 1) + self.assertEqual(len(json_data["books"][0]["comments"]), 1) + self.assertEqual(len(json_data["books"][0]["quotes"]), 1) + + self.assertEqual(json_data["books"][0]["reviews"][0]["name"], "my review") + self.assertEqual(json_data["books"][0]["reviews"][0]["content"], "awesome") + self.assertEqual(json_data["books"][0]["reviews"][0]["rating"], "5.00") + + self.assertEqual(json_data["books"][0]["comments"][0]["content"], "ok so far") + self.assertEqual(json_data["books"][0]["comments"][0]["progress"], 15) + self.assertEqual(json_data["books"][0]["comments"][0]["progress_mode"], "PG") + + self.assertEqual( + json_data["books"][0]["quotes"][0]["content"], "check this out" + ) + self.assertEqual( + json_data["books"][0]["quotes"][0]["quote"], "A rose by any other name" + ) + + def test_tar_export(self): + """test the tar export function""" + + # TODO + pass diff --git a/bookwyrm/tests/models/test_bookwyrm_import_model.py b/bookwyrm/tests/models/test_bookwyrm_import_job.py similarity index 96% rename from bookwyrm/tests/models/test_bookwyrm_import_model.py rename to bookwyrm/tests/models/test_bookwyrm_import_job.py index 644cbd265..61713cd17 100644 --- a/bookwyrm/tests/models/test_bookwyrm_import_model.py +++ b/bookwyrm/tests/models/test_bookwyrm_import_job.py @@ -70,15 +70,28 @@ class BookwyrmImport(TestCase): self.tarfile = BookwyrmTarFile.open( mode="r:gz", fileobj=open(archive_file, "rb") ) - self.import_data = json.loads( - self.tarfile.read("archive.json").decode("utf-8") - ) + self.import_data = json.loads(self.tarfile.read("archive.json").decode("utf-8")) def test_update_user_profile(self): """Test update the user's profile from import data""" - # TODO once the tar is set up - pass + with patch("bookwyrm.suggested_users.remove_user_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + + models.bookwyrm_import_job.update_user_profile( + self.local_user, self.tarfile, self.import_data.get("user") + ) + self.local_user.refresh_from_db() + + self.assertEqual( + self.local_user.username, "mouse" + ) # username should not change + self.assertEqual(self.local_user.name, "Rat") + self.assertEqual( + self.local_user.summary, + "I love to make soup in Paris and eat pizza in New York", + ) def test_update_user_settings(self): """Test updating the user's settings from import data""" @@ -543,6 +556,8 @@ class BookwyrmImport(TestCase): self.assertEqual( models.ShelfBook.objects.filter(user=self.local_user.id).count(), 2 ) + + # check we didn't create an extra shelf self.assertEqual( - models.Shelf.objects.filter(user=self.local_user.id).count(), 2 + models.Shelf.objects.filter(user=self.local_user.id).count(), 4 ) diff --git a/bookwyrm/tests/views/preferences/test_export_user.py b/bookwyrm/tests/views/preferences/test_export_user.py index c7594749b..1483fc4ec 100644 --- a/bookwyrm/tests/views/preferences/test_export_user.py +++ b/bookwyrm/tests/views/preferences/test_export_user.py @@ -49,26 +49,3 @@ class ExportUserViews(TestCase): jobs = models.bookwyrm_export_job.BookwyrmExportJob.objects.count() self.assertEqual(jobs, 1) - - def test_download_export_user_file(self, *_): - """simple user export""" - - # TODO: need some help with this one - job = models.bookwyrm_export_job.BookwyrmExportJob.objects.create( - user=self.local_user - ) - MockTask = namedtuple("Task", ("id")) - with patch( - "bookwyrm.models.bookwyrm_export_job.start_export_task.delay" - ) as mock: - mock.return_value = MockTask(b'{"name": "mouse"}') - job.start_job() - - request = self.factory.get("") - request.user = self.local_user - job.refresh_from_db() - export = views.ExportArchive.as_view()(request, job.id) - self.assertIsInstance(export, HttpResponse) - self.assertEqual(export.status_code, 200) - # pylint: disable=line-too-long - self.assertEqual(export.content, b'{"name": "mouse"}') From 781b01a007b8248992e5c98ceda958ec520aed4c Mon Sep 17 00:00:00 2001 From: Hugh Rundle Date: Sat, 21 Oct 2023 19:43:44 +1100 Subject: [PATCH 2/6] add error handling and status for user exports * fix Safari not downloading with the correct filename * add FAILED status * don't provide download link for stopped jobs --- bookwyrm/models/bookwyrm_export_job.py | 18 ++++++++++-------- bookwyrm/models/job.py | 12 ++++++++++-- .../templates/preferences/export-user.html | 6 +++--- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index c3a0b652a..00cab7559 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -35,13 +35,14 @@ def start_export_task(**kwargs): # don't start the job if it was stopped from the UI if job.complete: return - - # This is where ChildJobs get made - job.export_data = ContentFile(b"", str(uuid4())) - - json_data = json_export(job.user) - tar_export(json_data, job.user, job.export_data) - + try: + # This is where ChildJobs get made + job.export_data = ContentFile(b"", str(uuid4())) + json_data = json_export(job.user) + tar_export(json_data, job.user, job.export_data) + except Exception as err: # pylint: disable=broad-except + logger.exception("Job %s Failed with error: %s", job.id, err) + job.set_status("failed") job.save(update_fields=["export_data"]) @@ -56,7 +57,8 @@ def tar_export(json_data: str, user, f): editions, books = get_books_for_user(user) for book in editions: - tar.add_image(book.cover) + if getattr(book, "cover", False): + tar.add_image(book.cover) f.close() diff --git a/bookwyrm/models/job.py b/bookwyrm/models/job.py index 6e8d0dc5c..7557c5855 100644 --- a/bookwyrm/models/job.py +++ b/bookwyrm/models/job.py @@ -19,6 +19,7 @@ class Job(models.Model): ACTIVE = "active", _("Active") COMPLETE = "complete", _("Complete") STOPPED = "stopped", _("Stopped") + FAILED = "failed", _("Failed") task_id = models.UUIDField(unique=True, null=True, blank=True) @@ -43,14 +44,17 @@ class Job(models.Model): self.save(update_fields=["status", "complete", "updated_date"]) - def stop_job(self): + def stop_job(self, reason=None): """Stop the job""" if self.complete: return self.__terminate_job() - self.status = self.Status.STOPPED + if reason and reason is "failed": + self.status = self.Status.FAILED + else: + self.status = self.Status.STOPPED self.complete = True self.updated_date = timezone.now() @@ -72,6 +76,10 @@ class Job(models.Model): self.stop_job() return + if status == self.Status.FAILED: + self.stop_job(reason="failed") + return + self.updated_date = timezone.now() self.status = status diff --git a/bookwyrm/templates/preferences/export-user.html b/bookwyrm/templates/preferences/export-user.html index 393d8990e..2dd3f6de3 100644 --- a/bookwyrm/templates/preferences/export-user.html +++ b/bookwyrm/templates/preferences/export-user.html @@ -48,8 +48,8 @@ {% for job in jobs %} - {% if job.complete %} -

{{ job.created_date }}

+ {% if job.complete and not job.status == "stopped" and not job.status == "failed" %} +

{{ job.created_date }}

{% else %}

{{ job.created_date }}

{% endif %} @@ -57,7 +57,7 @@ {{ job.updated_date }} Date: Sun, 22 Oct 2023 09:03:28 +1100 Subject: [PATCH 3/6] add notifs and error handling for user export/import --- .../migrations/0183_auto_20231021_2050.py | 34 ++++++++++++++ bookwyrm/models/bookwyrm_export_job.py | 3 +- bookwyrm/models/bookwyrm_import_job.py | 46 +++++++++++-------- bookwyrm/models/job.py | 2 +- bookwyrm/models/notification.py | 38 ++++++++++++++- bookwyrm/templates/import/import_user.html | 2 +- bookwyrm/templates/notifications/item.html | 4 ++ .../notifications/items/user_export.html | 11 +++++ .../notifications/items/user_import.html | 10 ++++ 9 files changed, 126 insertions(+), 24 deletions(-) create mode 100644 bookwyrm/migrations/0183_auto_20231021_2050.py create mode 100644 bookwyrm/templates/notifications/items/user_export.html create mode 100644 bookwyrm/templates/notifications/items/user_import.html diff --git a/bookwyrm/migrations/0183_auto_20231021_2050.py b/bookwyrm/migrations/0183_auto_20231021_2050.py new file mode 100644 index 000000000..201a9201a --- /dev/null +++ b/bookwyrm/migrations/0183_auto_20231021_2050.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.20 on 2023-10-21 20:50 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0182_merge_20230905_2240'), + ] + + operations = [ + migrations.AddField( + model_name='notification', + name='related_user_export', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='bookwyrm.bookwyrmexportjob'), + ), + migrations.AlterField( + model_name='childjob', + name='status', + field=models.CharField(choices=[('pending', 'Pending'), ('active', 'Active'), ('complete', 'Complete'), ('stopped', 'Stopped'), ('failed', 'Failed')], default='pending', max_length=50, null=True), + ), + migrations.AlterField( + model_name='notification', + name='notification_type', + field=models.CharField(choices=[('FAVORITE', 'Favorite'), ('REPLY', 'Reply'), ('MENTION', 'Mention'), ('TAG', 'Tag'), ('FOLLOW', 'Follow'), ('FOLLOW_REQUEST', 'Follow Request'), ('BOOST', 'Boost'), ('IMPORT', 'Import'), ('USER_IMPORT', 'User Import'), ('USER_EXPORT', 'User Export'), ('ADD', 'Add'), ('REPORT', 'Report'), ('LINK_DOMAIN', 'Link Domain'), ('INVITE', 'Invite'), ('ACCEPT', 'Accept'), ('JOIN', 'Join'), ('LEAVE', 'Leave'), ('REMOVE', 'Remove'), ('GROUP_PRIVACY', 'Group Privacy'), ('GROUP_NAME', 'Group Name'), ('GROUP_DESCRIPTION', 'Group Description')], max_length=255), + ), + migrations.AlterField( + model_name='parentjob', + name='status', + field=models.CharField(choices=[('pending', 'Pending'), ('active', 'Active'), ('complete', 'Complete'), ('stopped', 'Stopped'), ('failed', 'Failed')], default='pending', max_length=50, null=True), + ), + ] diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index 00cab7559..96e602cc9 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -41,8 +41,9 @@ def start_export_task(**kwargs): json_data = json_export(job.user) tar_export(json_data, job.user, job.export_data) except Exception as err: # pylint: disable=broad-except - logger.exception("Job %s Failed with error: %s", job.id, err) + logger.exception("User Export Job %s Failed with error: %s", job.id, err) job.set_status("failed") + job.set_status("complete") # need to explicitly set this here to trigger notifications job.save(update_fields=["export_data"]) diff --git a/bookwyrm/models/bookwyrm_import_job.py b/bookwyrm/models/bookwyrm_import_job.py index 696f8061a..73372829b 100644 --- a/bookwyrm/models/bookwyrm_import_job.py +++ b/bookwyrm/models/bookwyrm_import_job.py @@ -1,5 +1,6 @@ from functools import reduce import json +import logging import operator from django.db.models import FileField, JSONField, CharField @@ -18,7 +19,8 @@ from bookwyrm.models.job import ( create_child_job, ) from bookwyrm.utils.tar import BookwyrmTarFile -import json + +logger = logging.getLogger(__name__) class BookwyrmImportJob(ParentJob): @@ -43,27 +45,33 @@ def start_import_task(**kwargs): if job.complete: return - archive_file.open("rb") - with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: - job.import_data = json.loads(tar.read("archive.json").decode("utf-8")) + try: + archive_file.open("rb") + with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: + job.import_data = json.loads(tar.read("archive.json").decode("utf-8")) - if "include_user_profile" in job.required: - update_user_profile(job.user, tar, job.import_data.get("user")) - if "include_user_settings" in job.required: - update_user_settings(job.user, job.import_data.get("user")) - if "include_goals" in job.required: - update_goals(job.user, job.import_data.get("goals")) - if "include_saved_lists" in job.required: - upsert_saved_lists(job.user, job.import_data.get("saved_lists")) - if "include_follows" in job.required: - upsert_follows(job.user, job.import_data.get("follows")) - if "include_blocks" in job.required: - upsert_user_blocks(job.user, job.import_data.get("blocked_users")) + if "include_user_profile" in job.required: + update_user_profile(job.user, tar, job.import_data.get("user")) + if "include_user_settings" in job.required: + update_user_settings(job.user, job.import_data.get("user")) + if "include_goals" in job.required: + update_goals(job.user, job.import_data.get("goals")) + if "include_saved_lists" in job.required: + upsert_saved_lists(job.user, job.import_data.get("saved_lists")) + if "include_follows" in job.required: + upsert_follows(job.user, job.import_data.get("follows")) + if "include_blocks" in job.required: + upsert_user_blocks(job.user, job.import_data.get("blocked_users")) - process_books(job, tar) + process_books(job, tar) - job.save() - archive_file.close() + job.set_status("complete") # set here to trigger notifications + job.save() + archive_file.close() + + except Exception as err: # pylint: disable=broad-except + logger.exception("User Import Job %s Failed with error: %s", job.id, err) + job.set_status("failed") def process_books(job, tar): diff --git a/bookwyrm/models/job.py b/bookwyrm/models/job.py index 7557c5855..4ba4bc2d7 100644 --- a/bookwyrm/models/job.py +++ b/bookwyrm/models/job.py @@ -51,7 +51,7 @@ class Job(models.Model): self.__terminate_job() - if reason and reason is "failed": + if reason and reason == "failed": self.status = self.Status.FAILED else: self.status = self.Status.STOPPED diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py index 522038f9a..4c420a2e1 100644 --- a/bookwyrm/models/notification.py +++ b/bookwyrm/models/notification.py @@ -2,7 +2,8 @@ from django.db import models, transaction from django.dispatch import receiver from .base_model import BookWyrmModel -from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain +from . import Boost, Favorite, GroupMemberInvitation, ImportJob, BookwyrmImportJob, LinkDomain +from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from . import ListItem, Report, Status, User, UserFollowRequest @@ -22,6 +23,8 @@ class Notification(BookWyrmModel): # Imports IMPORT = "IMPORT" + USER_IMPORT = "USER_IMPORT" + USER_EXPORT = "USER_EXPORT" # List activity ADD = "ADD" @@ -44,7 +47,7 @@ class Notification(BookWyrmModel): NotificationType = models.TextChoices( # there has got be a better way to do this "NotificationType", - f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}", + f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {USER_IMPORT} {USER_EXPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}", ) user = models.ForeignKey("User", on_delete=models.CASCADE) @@ -61,6 +64,7 @@ class Notification(BookWyrmModel): ) related_status = models.ForeignKey("Status", on_delete=models.CASCADE, null=True) related_import = models.ForeignKey("ImportJob", on_delete=models.CASCADE, null=True) + related_user_export = models.ForeignKey("BookwyrmExportJob", on_delete=models.CASCADE, null=True) related_list_items = models.ManyToManyField( "ListItem", symmetrical=False, related_name="notifications" ) @@ -222,6 +226,36 @@ def notify_user_on_import_complete( related_import=instance, ) +@receiver(models.signals.post_save, sender=BookwyrmImportJob) +# pylint: disable=unused-argument +def notify_user_on_user_import_complete( + sender, instance, *args, update_fields=None, **kwargs +): + """we imported your user details! aren't you proud of us""" + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: + return + Notification.objects.create( + user=instance.user, + notification_type=Notification.USER_IMPORT + ) + +@receiver(models.signals.post_save, sender=BookwyrmExportJob) +# pylint: disable=unused-argument +def notify_user_on_user_export_complete( + sender, instance, *args, update_fields=None, **kwargs +): + """we imported your user details! aren't you proud of us""" + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: + print("RETURNING", instance.status) + return + print("NOTIFYING") + Notification.objects.create( + user=instance.user, + notification_type=Notification.USER_EXPORT, + related_user_export=instance, + ) @receiver(models.signals.post_save, sender=Report) @transaction.atomic diff --git a/bookwyrm/templates/import/import_user.html b/bookwyrm/templates/import/import_user.html index 86e99f657..e48f0198d 100644 --- a/bookwyrm/templates/import/import_user.html +++ b/bookwyrm/templates/import/import_user.html @@ -133,7 +133,7 @@ {{ job.updated_date }} +{% endblock %} + +{% block description %} + {% url 'prefs-export-file' notification.related_user_export.task_id as url %} + {% blocktrans %}Your user export is ready.{% endblocktrans %} +{% endblock %} diff --git a/bookwyrm/templates/notifications/items/user_import.html b/bookwyrm/templates/notifications/items/user_import.html new file mode 100644 index 000000000..e0b3ddaad --- /dev/null +++ b/bookwyrm/templates/notifications/items/user_import.html @@ -0,0 +1,10 @@ +{% extends 'notifications/items/layout.html' %} +{% load i18n %} + +{% block icon %} + +{% endblock %} + +{% block description %} + {% blocktrans %}Your user import is complete.{% endblocktrans %} +{% endblock %} From 836127f369d5e352c118d486944651f139066af3 Mon Sep 17 00:00:00 2001 From: Hugh Rundle Date: Sun, 22 Oct 2023 10:49:13 +1100 Subject: [PATCH 4/6] cooldown period for user exports add USER_EXPORT_COOLDOWN_HOURS setting for controlling user exports and imports --- bookwyrm/settings.py | 3 +++ bookwyrm/templates/import/import_user.html | 27 +++++-------------- .../templates/preferences/export-user.html | 9 +++++++ bookwyrm/views/imports/import_data.py | 7 ++++- bookwyrm/views/preferences/export.py | 9 ++++++- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 9a4c9b5a4..854f05973 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -423,3 +423,6 @@ if HTTP_X_FORWARDED_PROTO: # Do not change this setting unless you already have an existing # user with the same username - in which case you should change it! INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" + +# exports +USER_EXPORT_COOLDOWN_HOURS = 48 \ No newline at end of file diff --git a/bookwyrm/templates/import/import_user.html b/bookwyrm/templates/import/import_user.html index e48f0198d..1eee017fa 100644 --- a/bookwyrm/templates/import/import_user.html +++ b/bookwyrm/templates/import/import_user.html @@ -15,28 +15,12 @@ {% endif %} - {% if import_size_limit and import_limit_reset %} -
-

{% blocktrans %}Currently you are allowed to import one user every {{ user_import_limit_reset }} days.{% endblocktrans %}

-

{% blocktrans %}You have {{ allowed_imports }} left.{% endblocktrans %}

+ {% if next_available %} +
+

{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}

+

{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}

- {% endif %} - {% if recent_avg_hours or recent_avg_minutes %} -
-

- {% if recent_avg_hours %} - {% blocktrans trimmed with hours=recent_avg_hours|floatformat:0|intcomma %} - On average, recent imports have taken {{ hours }} hours. - {% endblocktrans %} - {% else %} - {% blocktrans trimmed with minutes=recent_avg_minutes|floatformat:0|intcomma %} - On average, recent imports have taken {{ minutes }} minutes. - {% endblocktrans %} - {% endif %} -

-
- {% endif %} - + {% else %}
{% csrf_token %} @@ -100,6 +84,7 @@

{% trans "You've reached the import limit." %}

{% endif%}
+ {% endif %}
diff --git a/bookwyrm/templates/preferences/export-user.html b/bookwyrm/templates/preferences/export-user.html index 2dd3f6de3..2f63c9e1c 100644 --- a/bookwyrm/templates/preferences/export-user.html +++ b/bookwyrm/templates/preferences/export-user.html @@ -9,6 +9,13 @@ {% block panel %}
+ {% if next_available %} +

+ {% blocktrans %} + You will be able to create a new export file at {{ next_available }} + {% endblocktrans %} +

+ {% else %}

{% trans "Your exported archive file will include all user data for import into another Bookwyrm server" %}

@@ -19,6 +26,8 @@ {% trans "Create user export file" %} + {% endif %} +

{% trans "Recent Exports" %}

diff --git a/bookwyrm/views/imports/import_data.py b/bookwyrm/views/imports/import_data.py index 69a87c0c2..aa561d367 100644 --- a/bookwyrm/views/imports/import_data.py +++ b/bookwyrm/views/imports/import_data.py @@ -23,7 +23,7 @@ from bookwyrm.importers import ( OpenLibraryImporter, ) from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob -from bookwyrm.settings import PAGE_LENGTH +from bookwyrm.settings import PAGE_LENGTH, USER_EXPORT_COOLDOWN_HOURS from bookwyrm.utils.cache import get_or_set # pylint: disable= no-self-use @@ -142,11 +142,16 @@ class UserImport(View): jobs = BookwyrmImportJob.objects.filter(user=request.user).order_by( "-created_date" ) + hours = USER_EXPORT_COOLDOWN_HOURS + allowed = jobs.first().created_date < timezone.now() - datetime.timedelta(hours=hours) + next_available = jobs.first().created_date + datetime.timedelta(hours=hours) if not allowed else False paginated = Paginator(jobs, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) data = { "import_form": forms.ImportUserForm(), "jobs": page, + "user_import_hours": hours, + "next_available": next_available, "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index 28e83051e..49b19aea8 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -1,4 +1,5 @@ """ Let users export their book data """ +from datetime import timedelta import csv import io @@ -7,11 +8,12 @@ from django.core.paginator import Paginator from django.db.models import Q from django.http import HttpResponse from django.template.response import TemplateResponse +from django.utils import timezone from django.views import View from django.utils.decorators import method_decorator from django.shortcuts import redirect -from bookwyrm import models +from bookwyrm import models, settings from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from bookwyrm.settings import PAGE_LENGTH @@ -101,10 +103,15 @@ class ExportUser(View): jobs = BookwyrmExportJob.objects.filter(user=request.user).order_by( "-created_date" ) + hours = settings.USER_EXPORT_COOLDOWN_HOURS + allowed = jobs.first().created_date < timezone.now() - timedelta(hours=hours) + next_available = jobs.first().created_date + timedelta(hours=hours) if not allowed else False + paginated = Paginator(jobs, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) data = { "jobs": page, + "next_available": next_available, "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), From a27c6525019839120c81f662aac98c786e6e1405 Mon Sep 17 00:00:00 2001 From: Hugh Rundle Date: Sun, 22 Oct 2023 15:07:49 +1100 Subject: [PATCH 5/6] admin view for user imports - makes user_import_time_limit a site setting rather than a value in settings.py (note this applies to exports as well as imports) - admins can change user_import_time_limit from UI - admins can cancel stuck user imports - disabling new imports also disables user imports --- ...184_sitesettings_user_import_time_limit.py | 18 ++ bookwyrm/models/site.py | 1 + bookwyrm/settings.py | 5 +- bookwyrm/templates/import/import_user.html | 146 ++++----- .../imports/complete_user_import_modal.html | 23 ++ .../templates/settings/imports/imports.html | 286 +++++++++++++----- bookwyrm/urls.py | 10 + bookwyrm/views/__init__.py | 2 + bookwyrm/views/admin/imports.py | 30 ++ bookwyrm/views/imports/import_data.py | 7 +- bookwyrm/views/preferences/export.py | 6 +- 11 files changed, 374 insertions(+), 160 deletions(-) create mode 100644 bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py create mode 100644 bookwyrm/templates/settings/imports/complete_user_import_modal.html diff --git a/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py b/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py new file mode 100644 index 000000000..a23161db1 --- /dev/null +++ b/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2023-10-22 02:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0183_auto_20231021_2050'), + ] + + operations = [ + migrations.AddField( + model_name='sitesettings', + name='user_import_time_limit', + field=models.IntegerField(default=48), + ), + ] diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index a27c4b70d..cce055999 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -96,6 +96,7 @@ class SiteSettings(SiteModel): imports_enabled = models.BooleanField(default=True) import_size_limit = models.IntegerField(default=0) import_limit_reset = models.IntegerField(default=0) + user_import_time_limit = models.IntegerField(default=48) field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"]) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 854f05973..f74ef0093 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -422,7 +422,4 @@ if HTTP_X_FORWARDED_PROTO: # Mastodon servers. # Do not change this setting unless you already have an existing # user with the same username - in which case you should change it! -INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" - -# exports -USER_EXPORT_COOLDOWN_HOURS = 48 \ No newline at end of file +INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" \ No newline at end of file diff --git a/bookwyrm/templates/import/import_user.html b/bookwyrm/templates/import/import_user.html index 1eee017fa..8e7bb1a09 100644 --- a/bookwyrm/templates/import/import_user.html +++ b/bookwyrm/templates/import/import_user.html @@ -10,81 +10,89 @@ {% if invalid %}
- {% trans "Not a valid JSON file" %} + {% trans "Not a valid import file" %}
{% endif %} + {% if not site.imports_enabled %} +
+

+ +

+

+ {% trans "Imports are temporarily disabled; thank you for your patience." %} +

+
+ {% elif next_available %} +
+

{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}

+

{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}

+
+ {% else %} +
+ {% csrf_token %} - {% if next_available %} -
-

{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}

-

{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}

+
+
+
+ + {{ import_form.archive_file }}
+
+

{% trans "Importing this file will overwrite any data you currently have saved." %}

+

{% trans "Deselect any data you do not wish to include in your import. Books will always be imported" %}

+
+
+ +
+
+ + + + + + + + + + + + +
+
+
+ {% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %} + {% else %} - - {% csrf_token %} - -
-
-
- - {{ import_form.archive_file }} -
-
-

{% trans "Importing this file will overwrite any data you currently have saved." %}

-

{% trans "Deselect any data you do not wish to include in your import. Books will always be imported" %}

-
-
- -
-
- - - - - - - - - - - - -
-
-
- {% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %} - - {% else %} - -

{% trans "You've reached the import limit." %}

- {% endif%} - - {% endif %} + +

{% trans "You've reached the import limit." %}

+ {% endif%} + + {% endif %}
diff --git a/bookwyrm/templates/settings/imports/complete_user_import_modal.html b/bookwyrm/templates/settings/imports/complete_user_import_modal.html new file mode 100644 index 000000000..74004b7a2 --- /dev/null +++ b/bookwyrm/templates/settings/imports/complete_user_import_modal.html @@ -0,0 +1,23 @@ +{% extends 'components/modal.html' %} +{% load i18n %} + +{% block modal-title %}{% trans "Stop import?" %}{% endblock %} + +{% block modal-body %} +{% trans "This action will stop the user import before it is complete and cannot be un-done" %} +{% endblock %} + +{% block modal-footer %} +
+ {% csrf_token %} + +
+ + +
+
+{% endblock %} diff --git a/bookwyrm/templates/settings/imports/imports.html b/bookwyrm/templates/settings/imports/imports.html index 8819220fb..09d12b04a 100644 --- a/bookwyrm/templates/settings/imports/imports.html +++ b/bookwyrm/templates/settings/imports/imports.html @@ -29,6 +29,7 @@
{% trans "This is only intended to be used when things have gone very wrong with imports and you need to pause the feature while addressing issues." %} {% trans "While imports are disabled, users will not be allowed to start new imports, but existing imports will not be affected." %} + {% trans "This setting prevents both book imports and user imports." %}
{% csrf_token %}
@@ -89,91 +90,214 @@
+
+ + + {% trans "Limit how often users can import and export" %} + + + +
+
+ {% trans "Some users might try to run user imports or exports very frequently, which you want to limit." %} + {% trans "Set the value to 0 to not enforce any limit." %} +
+
+ + + + {% csrf_token %} +
+ +
+
+
+
-
- +

{% trans "Book Imports" %}

+
+
+ +
+ +
+ + + {% url 'settings-imports' status as url %} + + + + {% if status != "active" %} + + {% endif %} + + + + + {% if status == "active" %} + + {% endif %} + + {% for import in imports %} + + + + + {% if status != "active" %} + + {% endif %} + + + + + {% if status == "active" %} + + {% endif %} + + {% endfor %} + {% if not imports %} + + + + {% endif %} +
+ {% trans "ID" %} + + {% trans "User" as text %} + {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %} + + {% trans "Date Created" as text %} + {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %} + + {% trans "Date Updated" %} + + {% trans "Items" %} + + {% trans "Pending items" %} + + {% trans "Successful items" %} + + {% trans "Failed items" %} + {% trans "Actions" %}
{{ import.id }} + {{ import.user|username }} + {{ import.created_date }}{{ import.updated_date }}{{ import.item_count|intcomma }}{{ import.pending_item_count|intcomma }}{{ import.successful_item_count|intcomma }}{{ import.failed_item_count|intcomma }} + {% join "complete" import.id as modal_id %} + + {% include "settings/imports/complete_import_modal.html" with id=modal_id %} +
+ {% trans "No matching imports found." %} +
+
+ + {% include 'snippets/pagination.html' with page=imports path=request.path %} +
-
- - - {% url 'settings-imports' status as url %} - - - - {% if status != "active" %} - +
+

{% trans "User Imports" %}

+
+
+ +
+
+ +
+
- {% trans "ID" %} - - {% trans "User" as text %} - {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %} - - {% trans "Date Created" as text %} - {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %} - - {% trans "Date Updated" %} -
+ + {% url 'settings-imports' status as url %} + + + + {% if status != "active" %} + + {% endif %} + + {% if status == "active" %} + + {% else %} + + {% endif %} + + {% for import in user_imports %} + + + + + {% if status != "active" %} + + {% endif %} + {% if status == "active" %} + + {% else %} + + + {% endif %} + + {% endfor %} + {% if not user_imports %} + + + {% endif %} - - - - - {% if status == "active" %} - - {% endif %} - - {% for import in imports %} - - - - - {% if status != "active" %} - - {% endif %} - - - - - {% if status == "active" %} - - {% endif %} - - {% endfor %} - {% if not imports %} - - - - {% endif %} -
+ {% trans "ID" %} + + {% trans "User" as text %} + {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %} + + {% trans "Date Created" as text %} + {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %} + + {% trans "Date Updated" %} + {% trans "Actions" %}{% trans "Status" %}
{{ import.id }} + {{ import.user|username }} + {{ import.created_date }}{{ import.updated_date }} + {% join "complete" import.id as modal_id %} + + {% include "settings/imports/complete_user_import_modal.html" with id=modal_id %} + + {{ import.status }}
+ {% trans "No matching imports found." %} +
- {% trans "Items" %} - - {% trans "Pending items" %} - - {% trans "Successful items" %} - - {% trans "Failed items" %} - {% trans "Actions" %}
{{ import.id }} - {{ import.user|username }} - {{ import.created_date }}{{ import.updated_date }}{{ import.item_count|intcomma }}{{ import.pending_item_count|intcomma }}{{ import.successful_item_count|intcomma }}{{ import.failed_item_count|intcomma }} - {% join "complete" import.id as modal_id %} - - {% include "settings/imports/complete_import_modal.html" with id=modal_id %} -
- {% trans "No matching imports found." %} -
+ +
+ + {% include 'snippets/pagination.html' with page=user_imports path=request.path %} + {% endblock %}
- -{% include 'snippets/pagination.html' with page=imports path=request.path %} -{% endblock %} - diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 5b83acb85..2871ef282 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -316,6 +316,11 @@ urlpatterns = [ views.ImportList.as_view(), name="settings-imports-complete", ), + re_path( + r"^settings/user-imports/(?P\d+)/complete/?$", + views.set_user_import_completed, + name="settings-user-import-complete", + ), re_path( r"^settings/imports/disable/?$", views.disable_imports, @@ -331,6 +336,11 @@ urlpatterns = [ views.set_import_size_limit, name="settings-imports-set-limit", ), + re_path( + r"^settings/user-imports/set-limit/?$", + views.set_user_import_limit, + name="settings-user-imports-set-limit", + ), re_path( r"^settings/celery/?$", views.CeleryStatus.as_view(), name="settings-celery" ), diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index c044200e3..d98dffdcc 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -16,6 +16,8 @@ from .admin.imports import ( disable_imports, enable_imports, set_import_size_limit, + set_user_import_completed, + set_user_import_limit ) from .admin.ip_blocklist import IPBlocklist from .admin.invite import ManageInvites, Invite, InviteRequest diff --git a/bookwyrm/views/admin/imports.py b/bookwyrm/views/admin/imports.py index 7ae190ce8..4da7acf0e 100644 --- a/bookwyrm/views/admin/imports.py +++ b/bookwyrm/views/admin/imports.py @@ -40,9 +40,17 @@ class ImportList(View): paginated = Paginator(imports, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) + user_imports = models.BookwyrmImportJob.objects.filter(complete=complete).order_by( + "created_date" + ) + + user_paginated = Paginator(user_imports, PAGE_LENGTH) + user_page = user_paginated.get_page(request.GET.get("page")) + site_settings = models.SiteSettings.objects.get() data = { "imports": page, + "user_imports": user_page, "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), @@ -50,6 +58,7 @@ class ImportList(View): "sort": sort, "import_size_limit": site_settings.import_size_limit, "import_limit_reset": site_settings.import_limit_reset, + "user_import_time_limit": site_settings.user_import_time_limit, } return TemplateResponse(request, "settings/imports/imports.html", data) @@ -95,3 +104,24 @@ def set_import_size_limit(request): site.import_limit_reset = import_limit_reset site.save(update_fields=["import_size_limit", "import_limit_reset"]) return redirect("settings-imports") + +@require_POST +@login_required +@permission_required("bookwyrm.moderate_user", raise_exception=True) +# pylint: disable=unused-argument +def set_user_import_completed(request, import_id): + """Mark a user import as complete""" + import_job = get_object_or_404(models.BookwyrmImportJob, id=import_id) + import_job.stop_job() + return redirect("settings-imports") + + +@require_POST +@permission_required("bookwyrm.edit_instance_settings", raise_exception=True) +# pylint: disable=unused-argument +def set_user_import_limit(request): + """Limit how ofter users can import and export their account""" + site = models.SiteSettings.objects.get() + site.user_import_time_limit = int(request.POST.get("limit")) + site.save(update_fields=["user_import_time_limit"]) + return redirect("settings-imports") \ No newline at end of file diff --git a/bookwyrm/views/imports/import_data.py b/bookwyrm/views/imports/import_data.py index aa561d367..4fd50d9ce 100644 --- a/bookwyrm/views/imports/import_data.py +++ b/bookwyrm/views/imports/import_data.py @@ -23,7 +23,7 @@ from bookwyrm.importers import ( OpenLibraryImporter, ) from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob -from bookwyrm.settings import PAGE_LENGTH, USER_EXPORT_COOLDOWN_HOURS +from bookwyrm.settings import PAGE_LENGTH from bookwyrm.utils.cache import get_or_set # pylint: disable= no-self-use @@ -142,8 +142,9 @@ class UserImport(View): jobs = BookwyrmImportJob.objects.filter(user=request.user).order_by( "-created_date" ) - hours = USER_EXPORT_COOLDOWN_HOURS - allowed = jobs.first().created_date < timezone.now() - datetime.timedelta(hours=hours) + site = models.SiteSettings.objects.get() + hours = site.user_import_time_limit + allowed = jobs.first().created_date < timezone.now() - datetime.timedelta(hours=hours) if jobs.first() else True next_available = jobs.first().created_date + datetime.timedelta(hours=hours) if not allowed else False paginated = Paginator(jobs, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index 49b19aea8..5e70f896e 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -103,10 +103,10 @@ class ExportUser(View): jobs = BookwyrmExportJob.objects.filter(user=request.user).order_by( "-created_date" ) - hours = settings.USER_EXPORT_COOLDOWN_HOURS - allowed = jobs.first().created_date < timezone.now() - timedelta(hours=hours) + site = models.SiteSettings.objects.get() + hours = site.user_import_time_limit + allowed = jobs.first().created_date < timezone.now() - timedelta(hours=hours) if jobs.first() else True next_available = jobs.first().created_date + timedelta(hours=hours) if not allowed else False - paginated = Paginator(jobs, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) data = { From b34a49117263d9136e94669d9edef5bd04ae8df2 Mon Sep 17 00:00:00 2001 From: Hugh Rundle Date: Sun, 22 Oct 2023 15:34:25 +1100 Subject: [PATCH 6/6] run black --- .../migrations/0183_auto_20231021_2050.py | 77 +++++++++++++++---- ...184_sitesettings_user_import_time_limit.py | 6 +- bookwyrm/models/bookwyrm_export_job.py | 4 +- bookwyrm/models/bookwyrm_import_job.py | 2 +- bookwyrm/models/notification.py | 19 ++++- bookwyrm/settings.py | 2 +- bookwyrm/views/__init__.py | 2 +- bookwyrm/views/admin/imports.py | 9 ++- bookwyrm/views/imports/import_data.py | 12 ++- bookwyrm/views/preferences/export.py | 10 ++- 10 files changed, 111 insertions(+), 32 deletions(-) diff --git a/bookwyrm/migrations/0183_auto_20231021_2050.py b/bookwyrm/migrations/0183_auto_20231021_2050.py index 201a9201a..c960fe5bd 100644 --- a/bookwyrm/migrations/0183_auto_20231021_2050.py +++ b/bookwyrm/migrations/0183_auto_20231021_2050.py @@ -7,28 +7,79 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('bookwyrm', '0182_merge_20230905_2240'), + ("bookwyrm", "0182_merge_20230905_2240"), ] operations = [ migrations.AddField( - model_name='notification', - name='related_user_export', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='bookwyrm.bookwyrmexportjob'), + model_name="notification", + name="related_user_export", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="bookwyrm.bookwyrmexportjob", + ), ), migrations.AlterField( - model_name='childjob', - name='status', - field=models.CharField(choices=[('pending', 'Pending'), ('active', 'Active'), ('complete', 'Complete'), ('stopped', 'Stopped'), ('failed', 'Failed')], default='pending', max_length=50, null=True), + model_name="childjob", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ("failed", "Failed"), + ], + default="pending", + max_length=50, + null=True, + ), ), migrations.AlterField( - model_name='notification', - name='notification_type', - field=models.CharField(choices=[('FAVORITE', 'Favorite'), ('REPLY', 'Reply'), ('MENTION', 'Mention'), ('TAG', 'Tag'), ('FOLLOW', 'Follow'), ('FOLLOW_REQUEST', 'Follow Request'), ('BOOST', 'Boost'), ('IMPORT', 'Import'), ('USER_IMPORT', 'User Import'), ('USER_EXPORT', 'User Export'), ('ADD', 'Add'), ('REPORT', 'Report'), ('LINK_DOMAIN', 'Link Domain'), ('INVITE', 'Invite'), ('ACCEPT', 'Accept'), ('JOIN', 'Join'), ('LEAVE', 'Leave'), ('REMOVE', 'Remove'), ('GROUP_PRIVACY', 'Group Privacy'), ('GROUP_NAME', 'Group Name'), ('GROUP_DESCRIPTION', 'Group Description')], max_length=255), + model_name="notification", + name="notification_type", + field=models.CharField( + choices=[ + ("FAVORITE", "Favorite"), + ("REPLY", "Reply"), + ("MENTION", "Mention"), + ("TAG", "Tag"), + ("FOLLOW", "Follow"), + ("FOLLOW_REQUEST", "Follow Request"), + ("BOOST", "Boost"), + ("IMPORT", "Import"), + ("USER_IMPORT", "User Import"), + ("USER_EXPORT", "User Export"), + ("ADD", "Add"), + ("REPORT", "Report"), + ("LINK_DOMAIN", "Link Domain"), + ("INVITE", "Invite"), + ("ACCEPT", "Accept"), + ("JOIN", "Join"), + ("LEAVE", "Leave"), + ("REMOVE", "Remove"), + ("GROUP_PRIVACY", "Group Privacy"), + ("GROUP_NAME", "Group Name"), + ("GROUP_DESCRIPTION", "Group Description"), + ], + max_length=255, + ), ), migrations.AlterField( - model_name='parentjob', - name='status', - field=models.CharField(choices=[('pending', 'Pending'), ('active', 'Active'), ('complete', 'Complete'), ('stopped', 'Stopped'), ('failed', 'Failed')], default='pending', max_length=50, null=True), + model_name="parentjob", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ("failed", "Failed"), + ], + default="pending", + max_length=50, + null=True, + ), ), ] diff --git a/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py b/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py index a23161db1..24b4dad37 100644 --- a/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py +++ b/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py @@ -6,13 +6,13 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('bookwyrm', '0183_auto_20231021_2050'), + ("bookwyrm", "0183_auto_20231021_2050"), ] operations = [ migrations.AddField( - model_name='sitesettings', - name='user_import_time_limit', + model_name="sitesettings", + name="user_import_time_limit", field=models.IntegerField(default=48), ), ] diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index 96e602cc9..65f209905 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -43,7 +43,9 @@ def start_export_task(**kwargs): except Exception as err: # pylint: disable=broad-except logger.exception("User Export Job %s Failed with error: %s", job.id, err) job.set_status("failed") - job.set_status("complete") # need to explicitly set this here to trigger notifications + job.set_status( + "complete" + ) # need to explicitly set this here to trigger notifications job.save(update_fields=["export_data"]) diff --git a/bookwyrm/models/bookwyrm_import_job.py b/bookwyrm/models/bookwyrm_import_job.py index 73372829b..6e71aa4b5 100644 --- a/bookwyrm/models/bookwyrm_import_job.py +++ b/bookwyrm/models/bookwyrm_import_job.py @@ -65,7 +65,7 @@ def start_import_task(**kwargs): process_books(job, tar) - job.set_status("complete") # set here to trigger notifications + job.set_status("complete") # set here to trigger notifications job.save() archive_file.close() diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py index 4c420a2e1..c8140bce9 100644 --- a/bookwyrm/models/notification.py +++ b/bookwyrm/models/notification.py @@ -2,7 +2,14 @@ from django.db import models, transaction from django.dispatch import receiver from .base_model import BookWyrmModel -from . import Boost, Favorite, GroupMemberInvitation, ImportJob, BookwyrmImportJob, LinkDomain +from . import ( + Boost, + Favorite, + GroupMemberInvitation, + ImportJob, + BookwyrmImportJob, + LinkDomain, +) from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from . import ListItem, Report, Status, User, UserFollowRequest @@ -64,7 +71,9 @@ class Notification(BookWyrmModel): ) related_status = models.ForeignKey("Status", on_delete=models.CASCADE, null=True) related_import = models.ForeignKey("ImportJob", on_delete=models.CASCADE, null=True) - related_user_export = models.ForeignKey("BookwyrmExportJob", on_delete=models.CASCADE, null=True) + related_user_export = models.ForeignKey( + "BookwyrmExportJob", on_delete=models.CASCADE, null=True + ) related_list_items = models.ManyToManyField( "ListItem", symmetrical=False, related_name="notifications" ) @@ -226,6 +235,7 @@ def notify_user_on_import_complete( related_import=instance, ) + @receiver(models.signals.post_save, sender=BookwyrmImportJob) # pylint: disable=unused-argument def notify_user_on_user_import_complete( @@ -236,10 +246,10 @@ def notify_user_on_user_import_complete( if not instance.complete or "complete" not in update_fields: return Notification.objects.create( - user=instance.user, - notification_type=Notification.USER_IMPORT + user=instance.user, notification_type=Notification.USER_IMPORT ) + @receiver(models.signals.post_save, sender=BookwyrmExportJob) # pylint: disable=unused-argument def notify_user_on_user_export_complete( @@ -257,6 +267,7 @@ def notify_user_on_user_export_complete( related_user_export=instance, ) + @receiver(models.signals.post_save, sender=Report) @transaction.atomic # pylint: disable=unused-argument diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index f74ef0093..9a4c9b5a4 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -422,4 +422,4 @@ if HTTP_X_FORWARDED_PROTO: # Mastodon servers. # Do not change this setting unless you already have an existing # user with the same username - in which case you should change it! -INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" \ No newline at end of file +INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index d98dffdcc..2746ab9f9 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -17,7 +17,7 @@ from .admin.imports import ( enable_imports, set_import_size_limit, set_user_import_completed, - set_user_import_limit + set_user_import_limit, ) from .admin.ip_blocklist import IPBlocklist from .admin.invite import ManageInvites, Invite, InviteRequest diff --git a/bookwyrm/views/admin/imports.py b/bookwyrm/views/admin/imports.py index 4da7acf0e..a85d6c79e 100644 --- a/bookwyrm/views/admin/imports.py +++ b/bookwyrm/views/admin/imports.py @@ -40,9 +40,9 @@ class ImportList(View): paginated = Paginator(imports, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) - user_imports = models.BookwyrmImportJob.objects.filter(complete=complete).order_by( - "created_date" - ) + user_imports = models.BookwyrmImportJob.objects.filter( + complete=complete + ).order_by("created_date") user_paginated = Paginator(user_imports, PAGE_LENGTH) user_page = user_paginated.get_page(request.GET.get("page")) @@ -105,6 +105,7 @@ def set_import_size_limit(request): site.save(update_fields=["import_size_limit", "import_limit_reset"]) return redirect("settings-imports") + @require_POST @login_required @permission_required("bookwyrm.moderate_user", raise_exception=True) @@ -124,4 +125,4 @@ def set_user_import_limit(request): site = models.SiteSettings.objects.get() site.user_import_time_limit = int(request.POST.get("limit")) site.save(update_fields=["user_import_time_limit"]) - return redirect("settings-imports") \ No newline at end of file + return redirect("settings-imports") diff --git a/bookwyrm/views/imports/import_data.py b/bookwyrm/views/imports/import_data.py index 4fd50d9ce..1a9085ce1 100644 --- a/bookwyrm/views/imports/import_data.py +++ b/bookwyrm/views/imports/import_data.py @@ -144,8 +144,16 @@ class UserImport(View): ) site = models.SiteSettings.objects.get() hours = site.user_import_time_limit - allowed = jobs.first().created_date < timezone.now() - datetime.timedelta(hours=hours) if jobs.first() else True - next_available = jobs.first().created_date + datetime.timedelta(hours=hours) if not allowed else False + allowed = ( + jobs.first().created_date < timezone.now() - datetime.timedelta(hours=hours) + if jobs.first() + else True + ) + next_available = ( + jobs.first().created_date + datetime.timedelta(hours=hours) + if not allowed + else False + ) paginated = Paginator(jobs, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) data = { diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index 5e70f896e..037b8dbdc 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -105,8 +105,14 @@ class ExportUser(View): ) site = models.SiteSettings.objects.get() hours = site.user_import_time_limit - allowed = jobs.first().created_date < timezone.now() - timedelta(hours=hours) if jobs.first() else True - next_available = jobs.first().created_date + timedelta(hours=hours) if not allowed else False + allowed = ( + jobs.first().created_date < timezone.now() - timedelta(hours=hours) + if jobs.first() + else True + ) + next_available = ( + jobs.first().created_date + timedelta(hours=hours) if not allowed else False + ) paginated = Paginator(jobs, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) data = {