diff --git a/bookwyrm/migrations/0183_auto_20231021_2050.py b/bookwyrm/migrations/0183_auto_20231021_2050.py new file mode 100644 index 000000000..c960fe5bd --- /dev/null +++ b/bookwyrm/migrations/0183_auto_20231021_2050.py @@ -0,0 +1,85 @@ +# Generated by Django 3.2.20 on 2023-10-21 20:50 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0182_merge_20230905_2240"), + ] + + operations = [ + migrations.AddField( + model_name="notification", + name="related_user_export", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="bookwyrm.bookwyrmexportjob", + ), + ), + migrations.AlterField( + model_name="childjob", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ("failed", "Failed"), + ], + default="pending", + max_length=50, + null=True, + ), + ), + migrations.AlterField( + model_name="notification", + name="notification_type", + field=models.CharField( + choices=[ + ("FAVORITE", "Favorite"), + ("REPLY", "Reply"), + ("MENTION", "Mention"), + ("TAG", "Tag"), + ("FOLLOW", "Follow"), + ("FOLLOW_REQUEST", "Follow Request"), + ("BOOST", "Boost"), + ("IMPORT", "Import"), + ("USER_IMPORT", "User Import"), + ("USER_EXPORT", "User Export"), + ("ADD", "Add"), + ("REPORT", "Report"), + ("LINK_DOMAIN", "Link Domain"), + ("INVITE", "Invite"), + ("ACCEPT", "Accept"), + ("JOIN", "Join"), + ("LEAVE", "Leave"), + ("REMOVE", "Remove"), + ("GROUP_PRIVACY", "Group Privacy"), + ("GROUP_NAME", "Group Name"), + ("GROUP_DESCRIPTION", "Group Description"), + ], + max_length=255, + ), + ), + migrations.AlterField( + model_name="parentjob", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ("failed", "Failed"), + ], + default="pending", + max_length=50, + null=True, + ), + ), + ] diff --git a/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py b/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py new file mode 100644 index 000000000..24b4dad37 --- /dev/null +++ b/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2023-10-22 02:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0183_auto_20231021_2050"), + ] + + operations = [ + migrations.AddField( + model_name="sitesettings", + name="user_import_time_limit", + field=models.IntegerField(default=48), + ), + ] diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index c262d9b5c..65f209905 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -35,13 +35,17 @@ def start_export_task(**kwargs): # don't start the job if it was stopped from the UI if job.complete: return - - # This is where ChildJobs get made - job.export_data = ContentFile(b"", str(uuid4())) - - json_data = json_export(job.user) - tar_export(json_data, job.user, job.export_data) - + try: + # This is where ChildJobs get made + job.export_data = ContentFile(b"", str(uuid4())) + json_data = json_export(job.user) + tar_export(json_data, job.user, job.export_data) + except Exception as err: # pylint: disable=broad-except + logger.exception("User Export Job %s Failed with error: %s", job.id, err) + job.set_status("failed") + job.set_status( + "complete" + ) # need to explicitly set this here to trigger notifications job.save(update_fields=["export_data"]) @@ -56,7 +60,8 @@ def tar_export(json_data: str, user, f): editions, books = get_books_for_user(user) for book in editions: - tar.add_image(book.cover) + if getattr(book, "cover", False): + tar.add_image(book.cover) f.close() @@ -153,20 +158,12 @@ def json_export(user): comments = models.Comment.objects.filter(user=user, book=book["id"]).distinct() book["comments"] = list(comments.values()) - logger.error("FINAL COMMENTS") - logger.error(book["comments"]) # quotes quotes = models.Quotation.objects.filter(user=user, book=book["id"]).distinct() - # quote_statuses = models.Status.objects.filter( - # id__in=quotes, user=kwargs["user"] - # ).distinct() book["quotes"] = list(quotes.values()) - logger.error("FINAL QUOTES") - logger.error(book["quotes"]) - # append everything final_books.append(book) diff --git a/bookwyrm/models/bookwyrm_import_job.py b/bookwyrm/models/bookwyrm_import_job.py index 696f8061a..6e71aa4b5 100644 --- a/bookwyrm/models/bookwyrm_import_job.py +++ b/bookwyrm/models/bookwyrm_import_job.py @@ -1,5 +1,6 @@ from functools import reduce import json +import logging import operator from django.db.models import FileField, JSONField, CharField @@ -18,7 +19,8 @@ from bookwyrm.models.job import ( create_child_job, ) from bookwyrm.utils.tar import BookwyrmTarFile -import json + +logger = logging.getLogger(__name__) class BookwyrmImportJob(ParentJob): @@ -43,27 +45,33 @@ def start_import_task(**kwargs): if job.complete: return - archive_file.open("rb") - with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: - job.import_data = json.loads(tar.read("archive.json").decode("utf-8")) + try: + archive_file.open("rb") + with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: + job.import_data = json.loads(tar.read("archive.json").decode("utf-8")) - if "include_user_profile" in job.required: - update_user_profile(job.user, tar, job.import_data.get("user")) - if "include_user_settings" in job.required: - update_user_settings(job.user, job.import_data.get("user")) - if "include_goals" in job.required: - update_goals(job.user, job.import_data.get("goals")) - if "include_saved_lists" in job.required: - upsert_saved_lists(job.user, job.import_data.get("saved_lists")) - if "include_follows" in job.required: - upsert_follows(job.user, job.import_data.get("follows")) - if "include_blocks" in job.required: - upsert_user_blocks(job.user, job.import_data.get("blocked_users")) + if "include_user_profile" in job.required: + update_user_profile(job.user, tar, job.import_data.get("user")) + if "include_user_settings" in job.required: + update_user_settings(job.user, job.import_data.get("user")) + if "include_goals" in job.required: + update_goals(job.user, job.import_data.get("goals")) + if "include_saved_lists" in job.required: + upsert_saved_lists(job.user, job.import_data.get("saved_lists")) + if "include_follows" in job.required: + upsert_follows(job.user, job.import_data.get("follows")) + if "include_blocks" in job.required: + upsert_user_blocks(job.user, job.import_data.get("blocked_users")) - process_books(job, tar) + process_books(job, tar) - job.save() - archive_file.close() + job.set_status("complete") # set here to trigger notifications + job.save() + archive_file.close() + + except Exception as err: # pylint: disable=broad-except + logger.exception("User Import Job %s Failed with error: %s", job.id, err) + job.set_status("failed") def process_books(job, tar): diff --git a/bookwyrm/models/job.py b/bookwyrm/models/job.py index 6e8d0dc5c..4ba4bc2d7 100644 --- a/bookwyrm/models/job.py +++ b/bookwyrm/models/job.py @@ -19,6 +19,7 @@ class Job(models.Model): ACTIVE = "active", _("Active") COMPLETE = "complete", _("Complete") STOPPED = "stopped", _("Stopped") + FAILED = "failed", _("Failed") task_id = models.UUIDField(unique=True, null=True, blank=True) @@ -43,14 +44,17 @@ class Job(models.Model): self.save(update_fields=["status", "complete", "updated_date"]) - def stop_job(self): + def stop_job(self, reason=None): """Stop the job""" if self.complete: return self.__terminate_job() - self.status = self.Status.STOPPED + if reason and reason == "failed": + self.status = self.Status.FAILED + else: + self.status = self.Status.STOPPED self.complete = True self.updated_date = timezone.now() @@ -72,6 +76,10 @@ class Job(models.Model): self.stop_job() return + if status == self.Status.FAILED: + self.stop_job(reason="failed") + return + self.updated_date = timezone.now() self.status = status diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py index 522038f9a..c8140bce9 100644 --- a/bookwyrm/models/notification.py +++ b/bookwyrm/models/notification.py @@ -2,7 +2,15 @@ from django.db import models, transaction from django.dispatch import receiver from .base_model import BookWyrmModel -from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain +from . import ( + Boost, + Favorite, + GroupMemberInvitation, + ImportJob, + BookwyrmImportJob, + LinkDomain, +) +from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from . import ListItem, Report, Status, User, UserFollowRequest @@ -22,6 +30,8 @@ class Notification(BookWyrmModel): # Imports IMPORT = "IMPORT" + USER_IMPORT = "USER_IMPORT" + USER_EXPORT = "USER_EXPORT" # List activity ADD = "ADD" @@ -44,7 +54,7 @@ class Notification(BookWyrmModel): NotificationType = models.TextChoices( # there has got be a better way to do this "NotificationType", - f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}", + f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {USER_IMPORT} {USER_EXPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}", ) user = models.ForeignKey("User", on_delete=models.CASCADE) @@ -61,6 +71,9 @@ class Notification(BookWyrmModel): ) related_status = models.ForeignKey("Status", on_delete=models.CASCADE, null=True) related_import = models.ForeignKey("ImportJob", on_delete=models.CASCADE, null=True) + related_user_export = models.ForeignKey( + "BookwyrmExportJob", on_delete=models.CASCADE, null=True + ) related_list_items = models.ManyToManyField( "ListItem", symmetrical=False, related_name="notifications" ) @@ -223,6 +236,38 @@ def notify_user_on_import_complete( ) +@receiver(models.signals.post_save, sender=BookwyrmImportJob) +# pylint: disable=unused-argument +def notify_user_on_user_import_complete( + sender, instance, *args, update_fields=None, **kwargs +): + """we imported your user details! aren't you proud of us""" + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: + return + Notification.objects.create( + user=instance.user, notification_type=Notification.USER_IMPORT + ) + + +@receiver(models.signals.post_save, sender=BookwyrmExportJob) +# pylint: disable=unused-argument +def notify_user_on_user_export_complete( + sender, instance, *args, update_fields=None, **kwargs +): + """we imported your user details! aren't you proud of us""" + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: + print("RETURNING", instance.status) + return + print("NOTIFYING") + Notification.objects.create( + user=instance.user, + notification_type=Notification.USER_EXPORT, + related_user_export=instance, + ) + + @receiver(models.signals.post_save, sender=Report) @transaction.atomic # pylint: disable=unused-argument diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index a27c4b70d..cce055999 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -96,6 +96,7 @@ class SiteSettings(SiteModel): imports_enabled = models.BooleanField(default=True) import_size_limit = models.IntegerField(default=0) import_limit_reset = models.IntegerField(default=0) + user_import_time_limit = models.IntegerField(default=48) field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"]) diff --git a/bookwyrm/templates/import/import_user.html b/bookwyrm/templates/import/import_user.html index 86e99f657..8e7bb1a09 100644 --- a/bookwyrm/templates/import/import_user.html +++ b/bookwyrm/templates/import/import_user.html @@ -10,96 +10,89 @@ {% if invalid %}
- {% trans "Not a valid JSON file" %} + {% trans "Not a valid import file" %}
{% endif %} - - {% if import_size_limit and import_limit_reset %} -
-

{% blocktrans %}Currently you are allowed to import one user every {{ user_import_limit_reset }} days.{% endblocktrans %}

-

{% blocktrans %}You have {{ allowed_imports }} left.{% endblocktrans %}

-
- {% endif %} - {% if recent_avg_hours or recent_avg_minutes %} -
-

- {% if recent_avg_hours %} - {% blocktrans trimmed with hours=recent_avg_hours|floatformat:0|intcomma %} - On average, recent imports have taken {{ hours }} hours. - {% endblocktrans %} - {% else %} - {% blocktrans trimmed with minutes=recent_avg_minutes|floatformat:0|intcomma %} - On average, recent imports have taken {{ minutes }} minutes. - {% endblocktrans %} - {% endif %} -

+ {% if not site.imports_enabled %} +
+

+ +

+

+ {% trans "Imports are temporarily disabled; thank you for your patience." %} +

+
+ {% elif next_available %} +
+

{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}

+

{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}

- {% endif %} + {% else %} +
+ {% csrf_token %} - - {% csrf_token %} +
+
+
+ + {{ import_form.archive_file }} +
+
+

{% trans "Importing this file will overwrite any data you currently have saved." %}

+

{% trans "Deselect any data you do not wish to include in your import. Books will always be imported" %}

+
+
-
-
-
- - {{ import_form.archive_file }} -
-
-

{% trans "Importing this file will overwrite any data you currently have saved." %}

-

{% trans "Deselect any data you do not wish to include in your import. Books will always be imported" %}

-
+
+
+ + + + + + + + + + + +
- -
-
- - - - - - - - - - - - -
-
-
- {% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %} - - {% else %} - -

{% trans "You've reached the import limit." %}

- {% endif%} - +
+
+ {% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %} + + {% else %} + +

{% trans "You've reached the import limit." %}

+ {% endif%} + + {% endif %}
@@ -133,7 +126,7 @@ {{ job.updated_date }} +{% endblock %} + +{% block description %} + {% url 'prefs-export-file' notification.related_user_export.task_id as url %} + {% blocktrans %}Your user export is ready.{% endblocktrans %} +{% endblock %} diff --git a/bookwyrm/templates/notifications/items/user_import.html b/bookwyrm/templates/notifications/items/user_import.html new file mode 100644 index 000000000..e0b3ddaad --- /dev/null +++ b/bookwyrm/templates/notifications/items/user_import.html @@ -0,0 +1,10 @@ +{% extends 'notifications/items/layout.html' %} +{% load i18n %} + +{% block icon %} + +{% endblock %} + +{% block description %} + {% blocktrans %}Your user import is complete.{% endblocktrans %} +{% endblock %} diff --git a/bookwyrm/templates/preferences/export-user.html b/bookwyrm/templates/preferences/export-user.html index 81f13bc22..2f63c9e1c 100644 --- a/bookwyrm/templates/preferences/export-user.html +++ b/bookwyrm/templates/preferences/export-user.html @@ -9,18 +9,25 @@ {% block panel %}
+ {% if next_available %} +

+ {% blocktrans %} + You will be able to create a new export file at {{ next_available }} + {% endblocktrans %} +

+ {% else %}

{% trans "Your exported archive file will include all user data for import into another Bookwyrm server" %}

-

-

- {% csrf_token %} - -
-

+
+ {% csrf_token %} + +
+ {% endif %} +

{% trans "Recent Exports" %}

@@ -50,8 +57,8 @@ {% for job in jobs %} - {% if job.complete %} -

{{ job.created_date }}

+ {% if job.complete and not job.status == "stopped" and not job.status == "failed" %} +

{{ job.created_date }}

{% else %}

{{ job.created_date }}

{% endif %} @@ -59,7 +66,7 @@ {{ job.updated_date }} + {% csrf_token %} + +
+ + +
+ +{% endblock %} diff --git a/bookwyrm/templates/settings/imports/imports.html b/bookwyrm/templates/settings/imports/imports.html index 8819220fb..09d12b04a 100644 --- a/bookwyrm/templates/settings/imports/imports.html +++ b/bookwyrm/templates/settings/imports/imports.html @@ -29,6 +29,7 @@
{% trans "This is only intended to be used when things have gone very wrong with imports and you need to pause the feature while addressing issues." %} {% trans "While imports are disabled, users will not be allowed to start new imports, but existing imports will not be affected." %} + {% trans "This setting prevents both book imports and user imports." %}
{% csrf_token %}
@@ -89,91 +90,214 @@
+
+ + + {% trans "Limit how often users can import and export" %} + + + +
+
+ {% trans "Some users might try to run user imports or exports very frequently, which you want to limit." %} + {% trans "Set the value to 0 to not enforce any limit." %} +
+
+ + + + {% csrf_token %} +
+ +
+
+
+
-
- +

{% trans "Book Imports" %}

+
+
+ +
+ +
+ + + {% url 'settings-imports' status as url %} + + + + {% if status != "active" %} + + {% endif %} + + + + + {% if status == "active" %} + + {% endif %} + + {% for import in imports %} + + + + + {% if status != "active" %} + + {% endif %} + + + + + {% if status == "active" %} + + {% endif %} + + {% endfor %} + {% if not imports %} + + + + {% endif %} +
+ {% trans "ID" %} + + {% trans "User" as text %} + {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %} + + {% trans "Date Created" as text %} + {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %} + + {% trans "Date Updated" %} + + {% trans "Items" %} + + {% trans "Pending items" %} + + {% trans "Successful items" %} + + {% trans "Failed items" %} + {% trans "Actions" %}
{{ import.id }} + {{ import.user|username }} + {{ import.created_date }}{{ import.updated_date }}{{ import.item_count|intcomma }}{{ import.pending_item_count|intcomma }}{{ import.successful_item_count|intcomma }}{{ import.failed_item_count|intcomma }} + {% join "complete" import.id as modal_id %} + + {% include "settings/imports/complete_import_modal.html" with id=modal_id %} +
+ {% trans "No matching imports found." %} +
+
+ + {% include 'snippets/pagination.html' with page=imports path=request.path %} +
-
- - - {% url 'settings-imports' status as url %} - - - - {% if status != "active" %} - +
+

{% trans "User Imports" %}

+
+
+ +
+
+ +
+
- {% trans "ID" %} - - {% trans "User" as text %} - {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %} - - {% trans "Date Created" as text %} - {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %} - - {% trans "Date Updated" %} -
+ + {% url 'settings-imports' status as url %} + + + + {% if status != "active" %} + + {% endif %} + + {% if status == "active" %} + + {% else %} + + {% endif %} + + {% for import in user_imports %} + + + + + {% if status != "active" %} + + {% endif %} + {% if status == "active" %} + + {% else %} + + + {% endif %} + + {% endfor %} + {% if not user_imports %} + + + {% endif %} - - - - - {% if status == "active" %} - - {% endif %} - - {% for import in imports %} - - - - - {% if status != "active" %} - - {% endif %} - - - - - {% if status == "active" %} - - {% endif %} - - {% endfor %} - {% if not imports %} - - - - {% endif %} -
+ {% trans "ID" %} + + {% trans "User" as text %} + {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %} + + {% trans "Date Created" as text %} + {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %} + + {% trans "Date Updated" %} + {% trans "Actions" %}{% trans "Status" %}
{{ import.id }} + {{ import.user|username }} + {{ import.created_date }}{{ import.updated_date }} + {% join "complete" import.id as modal_id %} + + {% include "settings/imports/complete_user_import_modal.html" with id=modal_id %} + + {{ import.status }}
+ {% trans "No matching imports found." %} +
- {% trans "Items" %} - - {% trans "Pending items" %} - - {% trans "Successful items" %} - - {% trans "Failed items" %} - {% trans "Actions" %}
{{ import.id }} - {{ import.user|username }} - {{ import.created_date }}{{ import.updated_date }}{{ import.item_count|intcomma }}{{ import.pending_item_count|intcomma }}{{ import.successful_item_count|intcomma }}{{ import.failed_item_count|intcomma }} - {% join "complete" import.id as modal_id %} - - {% include "settings/imports/complete_import_modal.html" with id=modal_id %} -
- {% trans "No matching imports found." %} -
+ +
+ + {% include 'snippets/pagination.html' with page=user_imports path=request.path %} + {% endblock %}
- -{% include 'snippets/pagination.html' with page=imports path=request.path %} -{% endblock %} - diff --git a/bookwyrm/tests/models/test_bookwyrm_export_job.py b/bookwyrm/tests/models/test_bookwyrm_export_job.py new file mode 100644 index 000000000..bd314e60e --- /dev/null +++ b/bookwyrm/tests/models/test_bookwyrm_export_job.py @@ -0,0 +1,233 @@ +import datetime +import time +import json +from unittest.mock import patch + +from django.core.serializers.json import DjangoJSONEncoder +from django.test import TestCase +from django.utils import timezone + +from bookwyrm import models +import bookwyrm.models.bookwyrm_export_job as export_job + + +class BookwyrmExport(TestCase): + """testing user export functions""" + + def setUp(self): + """lots of stuff to set up for a user export""" + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ), patch("bookwyrm.lists_stream.populate_lists_task.delay"), patch( + "bookwyrm.suggested_users.rerank_user_task.delay" + ), patch( + "bookwyrm.lists_stream.remove_list_task.delay" + ), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch( + "bookwyrm.activitystreams.add_book_statuses_task" + ): + + self.local_user = models.User.objects.create_user( + "mouse", + "mouse@mouse.mouse", + "password", + local=True, + localname="mouse", + name="Mouse", + summary="I'm a real bookmouse", + manually_approves_followers=False, + hide_follows=False, + show_goal=False, + show_suggested_users=False, + discoverable=True, + preferred_timezone="America/Los Angeles", + default_post_privacy="followers", + ) + + self.rat_user = models.User.objects.create_user( + "rat", "rat@rat.rat", "ratword", local=True, localname="rat" + ) + + self.badger_user = models.User.objects.create_user( + "badger", + "badger@badger.badger", + "badgerword", + local=True, + localname="badger", + ) + + models.AnnualGoal.objects.create( + user=self.local_user, + year=timezone.now().year, + goal=128937123, + privacy="followers", + ) + + self.list = models.List.objects.create( + name="My excellent list", + user=self.local_user, + remote_id="https://local.lists/1111", + ) + + self.saved_list = models.List.objects.create( + name="My cool list", + user=self.rat_user, + remote_id="https://local.lists/9999", + ) + + self.local_user.saved_lists.add(self.saved_list) + self.local_user.blocks.add(self.badger_user) + self.rat_user.followers.add(self.local_user) + + # book, edition, author + self.author = models.Author.objects.create(name="Sam Zhu") + self.work = models.Work.objects.create( + title="Example Work", remote_id="https://example.com/book/1" + ) + self.edition = models.Edition.objects.create( + title="Example Edition", parent_work=self.work + ) + + self.edition.authors.add(self.author) + + # readthrough + self.readthrough_start = timezone.now() + finish = self.readthrough_start + datetime.timedelta(days=1) + models.ReadThrough.objects.create( + user=self.local_user, + book=self.edition, + start_date=self.readthrough_start, + finish_date=finish, + ) + + # shelve + read_shelf = models.Shelf.objects.get( + user=self.local_user, identifier="read" + ) + models.ShelfBook.objects.create( + book=self.edition, shelf=read_shelf, user=self.local_user + ) + + # add to list + item = models.ListItem.objects.create( + book_list=self.list, + user=self.local_user, + book=self.edition, + approved=True, + order=1, + ) + + # review + models.Review.objects.create( + content="awesome", + name="my review", + rating=5, + user=self.local_user, + book=self.edition, + ) + # comment + models.Comment.objects.create( + content="ok so far", + user=self.local_user, + book=self.edition, + progress=15, + ) + # quote + models.Quotation.objects.create( + content="check this out", + quote="A rose by any other name", + user=self.local_user, + book=self.edition, + ) + + def test_json_export_user_settings(self): + """Test the json export function for basic user info""" + data = export_job.json_export(self.local_user) + user_data = json.loads(data)["user"] + self.assertEqual(user_data["username"], "mouse") + self.assertEqual(user_data["name"], "Mouse") + self.assertEqual(user_data["summary"], "I'm a real bookmouse") + self.assertEqual(user_data["manually_approves_followers"], False) + self.assertEqual(user_data["hide_follows"], False) + self.assertEqual(user_data["show_goal"], False) + self.assertEqual(user_data["show_suggested_users"], False) + self.assertEqual(user_data["discoverable"], True) + self.assertEqual(user_data["preferred_timezone"], "America/Los Angeles") + self.assertEqual(user_data["default_post_privacy"], "followers") + + def test_json_export_extended_user_data(self): + """Test the json export function for other non-book user info""" + data = export_job.json_export(self.local_user) + json_data = json.loads(data) + + # goal + self.assertEqual(len(json_data["goals"]), 1) + self.assertEqual(json_data["goals"][0]["goal"], 128937123) + self.assertEqual(json_data["goals"][0]["year"], timezone.now().year) + self.assertEqual(json_data["goals"][0]["privacy"], "followers") + + # saved lists + self.assertEqual(len(json_data["saved_lists"]), 1) + self.assertEqual(json_data["saved_lists"][0], "https://local.lists/9999") + + # follows + self.assertEqual(len(json_data["follows"]), 1) + self.assertEqual(json_data["follows"][0], "https://your.domain.here/user/rat") + # blocked users + self.assertEqual(len(json_data["blocked_users"]), 1) + self.assertEqual( + json_data["blocked_users"][0], "https://your.domain.here/user/badger" + ) + + def test_json_export_books(self): + """Test the json export function for extended user info""" + + data = export_job.json_export(self.local_user) + json_data = json.loads(data) + start_date = json_data["books"][0]["readthroughs"][0]["start_date"] + + self.assertEqual(len(json_data["books"]), 1) + self.assertEqual(json_data["books"][0]["title"], "Example Edition") + self.assertEqual(len(json_data["books"][0]["authors"]), 1) + self.assertEqual(json_data["books"][0]["authors"][0]["name"], "Sam Zhu") + self.assertEqual( + f'"{start_date}"', DjangoJSONEncoder().encode(self.readthrough_start) + ) + self.assertEqual(json_data["books"][0]["shelves"][0]["identifier"], "read") + self.assertEqual( + json_data["books"][0]["shelf_books"]["read"][0]["book_id"], self.edition.id + ) + + self.assertEqual(len(json_data["books"][0]["lists"]), 1) + self.assertEqual(json_data["books"][0]["lists"][0]["name"], "My excellent list") + self.assertEqual(len(json_data["books"][0]["list_items"]), 1) + self.assertEqual( + json_data["books"][0]["list_items"]["My excellent list"][0]["book_id"], + self.edition.id, + ) + + self.assertEqual(len(json_data["books"][0]["reviews"]), 1) + self.assertEqual(len(json_data["books"][0]["comments"]), 1) + self.assertEqual(len(json_data["books"][0]["quotes"]), 1) + + self.assertEqual(json_data["books"][0]["reviews"][0]["name"], "my review") + self.assertEqual(json_data["books"][0]["reviews"][0]["content"], "awesome") + self.assertEqual(json_data["books"][0]["reviews"][0]["rating"], "5.00") + + self.assertEqual(json_data["books"][0]["comments"][0]["content"], "ok so far") + self.assertEqual(json_data["books"][0]["comments"][0]["progress"], 15) + self.assertEqual(json_data["books"][0]["comments"][0]["progress_mode"], "PG") + + self.assertEqual( + json_data["books"][0]["quotes"][0]["content"], "check this out" + ) + self.assertEqual( + json_data["books"][0]["quotes"][0]["quote"], "A rose by any other name" + ) + + def test_tar_export(self): + """test the tar export function""" + + # TODO + pass diff --git a/bookwyrm/tests/models/test_bookwyrm_import_model.py b/bookwyrm/tests/models/test_bookwyrm_import_job.py similarity index 96% rename from bookwyrm/tests/models/test_bookwyrm_import_model.py rename to bookwyrm/tests/models/test_bookwyrm_import_job.py index 644cbd265..61713cd17 100644 --- a/bookwyrm/tests/models/test_bookwyrm_import_model.py +++ b/bookwyrm/tests/models/test_bookwyrm_import_job.py @@ -70,15 +70,28 @@ class BookwyrmImport(TestCase): self.tarfile = BookwyrmTarFile.open( mode="r:gz", fileobj=open(archive_file, "rb") ) - self.import_data = json.loads( - self.tarfile.read("archive.json").decode("utf-8") - ) + self.import_data = json.loads(self.tarfile.read("archive.json").decode("utf-8")) def test_update_user_profile(self): """Test update the user's profile from import data""" - # TODO once the tar is set up - pass + with patch("bookwyrm.suggested_users.remove_user_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + + models.bookwyrm_import_job.update_user_profile( + self.local_user, self.tarfile, self.import_data.get("user") + ) + self.local_user.refresh_from_db() + + self.assertEqual( + self.local_user.username, "mouse" + ) # username should not change + self.assertEqual(self.local_user.name, "Rat") + self.assertEqual( + self.local_user.summary, + "I love to make soup in Paris and eat pizza in New York", + ) def test_update_user_settings(self): """Test updating the user's settings from import data""" @@ -543,6 +556,8 @@ class BookwyrmImport(TestCase): self.assertEqual( models.ShelfBook.objects.filter(user=self.local_user.id).count(), 2 ) + + # check we didn't create an extra shelf self.assertEqual( - models.Shelf.objects.filter(user=self.local_user.id).count(), 2 + models.Shelf.objects.filter(user=self.local_user.id).count(), 4 ) diff --git a/bookwyrm/tests/views/preferences/test_export_user.py b/bookwyrm/tests/views/preferences/test_export_user.py index c7594749b..1483fc4ec 100644 --- a/bookwyrm/tests/views/preferences/test_export_user.py +++ b/bookwyrm/tests/views/preferences/test_export_user.py @@ -49,26 +49,3 @@ class ExportUserViews(TestCase): jobs = models.bookwyrm_export_job.BookwyrmExportJob.objects.count() self.assertEqual(jobs, 1) - - def test_download_export_user_file(self, *_): - """simple user export""" - - # TODO: need some help with this one - job = models.bookwyrm_export_job.BookwyrmExportJob.objects.create( - user=self.local_user - ) - MockTask = namedtuple("Task", ("id")) - with patch( - "bookwyrm.models.bookwyrm_export_job.start_export_task.delay" - ) as mock: - mock.return_value = MockTask(b'{"name": "mouse"}') - job.start_job() - - request = self.factory.get("") - request.user = self.local_user - job.refresh_from_db() - export = views.ExportArchive.as_view()(request, job.id) - self.assertIsInstance(export, HttpResponse) - self.assertEqual(export.status_code, 200) - # pylint: disable=line-too-long - self.assertEqual(export.content, b'{"name": "mouse"}') diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 8b2893da2..4c6bdf151 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -317,6 +317,11 @@ urlpatterns = [ views.ImportList.as_view(), name="settings-imports-complete", ), + re_path( + r"^settings/user-imports/(?P\d+)/complete/?$", + views.set_user_import_completed, + name="settings-user-import-complete", + ), re_path( r"^settings/imports/disable/?$", views.disable_imports, @@ -332,6 +337,11 @@ urlpatterns = [ views.set_import_size_limit, name="settings-imports-set-limit", ), + re_path( + r"^settings/user-imports/set-limit/?$", + views.set_user_import_limit, + name="settings-user-imports-set-limit", + ), re_path( r"^settings/celery/?$", views.CeleryStatus.as_view(), name="settings-celery" ), diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index c044200e3..2746ab9f9 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -16,6 +16,8 @@ from .admin.imports import ( disable_imports, enable_imports, set_import_size_limit, + set_user_import_completed, + set_user_import_limit, ) from .admin.ip_blocklist import IPBlocklist from .admin.invite import ManageInvites, Invite, InviteRequest diff --git a/bookwyrm/views/admin/imports.py b/bookwyrm/views/admin/imports.py index 7ae190ce8..a85d6c79e 100644 --- a/bookwyrm/views/admin/imports.py +++ b/bookwyrm/views/admin/imports.py @@ -40,9 +40,17 @@ class ImportList(View): paginated = Paginator(imports, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) + user_imports = models.BookwyrmImportJob.objects.filter( + complete=complete + ).order_by("created_date") + + user_paginated = Paginator(user_imports, PAGE_LENGTH) + user_page = user_paginated.get_page(request.GET.get("page")) + site_settings = models.SiteSettings.objects.get() data = { "imports": page, + "user_imports": user_page, "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), @@ -50,6 +58,7 @@ class ImportList(View): "sort": sort, "import_size_limit": site_settings.import_size_limit, "import_limit_reset": site_settings.import_limit_reset, + "user_import_time_limit": site_settings.user_import_time_limit, } return TemplateResponse(request, "settings/imports/imports.html", data) @@ -95,3 +104,25 @@ def set_import_size_limit(request): site.import_limit_reset = import_limit_reset site.save(update_fields=["import_size_limit", "import_limit_reset"]) return redirect("settings-imports") + + +@require_POST +@login_required +@permission_required("bookwyrm.moderate_user", raise_exception=True) +# pylint: disable=unused-argument +def set_user_import_completed(request, import_id): + """Mark a user import as complete""" + import_job = get_object_or_404(models.BookwyrmImportJob, id=import_id) + import_job.stop_job() + return redirect("settings-imports") + + +@require_POST +@permission_required("bookwyrm.edit_instance_settings", raise_exception=True) +# pylint: disable=unused-argument +def set_user_import_limit(request): + """Limit how ofter users can import and export their account""" + site = models.SiteSettings.objects.get() + site.user_import_time_limit = int(request.POST.get("limit")) + site.save(update_fields=["user_import_time_limit"]) + return redirect("settings-imports") diff --git a/bookwyrm/views/imports/import_data.py b/bookwyrm/views/imports/import_data.py index 69a87c0c2..1a9085ce1 100644 --- a/bookwyrm/views/imports/import_data.py +++ b/bookwyrm/views/imports/import_data.py @@ -142,11 +142,25 @@ class UserImport(View): jobs = BookwyrmImportJob.objects.filter(user=request.user).order_by( "-created_date" ) + site = models.SiteSettings.objects.get() + hours = site.user_import_time_limit + allowed = ( + jobs.first().created_date < timezone.now() - datetime.timedelta(hours=hours) + if jobs.first() + else True + ) + next_available = ( + jobs.first().created_date + datetime.timedelta(hours=hours) + if not allowed + else False + ) paginated = Paginator(jobs, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) data = { "import_form": forms.ImportUserForm(), "jobs": page, + "user_import_hours": hours, + "next_available": next_available, "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index 28e83051e..037b8dbdc 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -1,4 +1,5 @@ """ Let users export their book data """ +from datetime import timedelta import csv import io @@ -7,11 +8,12 @@ from django.core.paginator import Paginator from django.db.models import Q from django.http import HttpResponse from django.template.response import TemplateResponse +from django.utils import timezone from django.views import View from django.utils.decorators import method_decorator from django.shortcuts import redirect -from bookwyrm import models +from bookwyrm import models, settings from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from bookwyrm.settings import PAGE_LENGTH @@ -101,10 +103,21 @@ class ExportUser(View): jobs = BookwyrmExportJob.objects.filter(user=request.user).order_by( "-created_date" ) + site = models.SiteSettings.objects.get() + hours = site.user_import_time_limit + allowed = ( + jobs.first().created_date < timezone.now() - timedelta(hours=hours) + if jobs.first() + else True + ) + next_available = ( + jobs.first().created_date + timedelta(hours=hours) if not allowed else False + ) paginated = Paginator(jobs, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) data = { "jobs": page, + "next_available": next_available, "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ),