diff --git a/bookwyrm/forms/forms.py b/bookwyrm/forms/forms.py index ea6093750..3d555f308 100644 --- a/bookwyrm/forms/forms.py +++ b/bookwyrm/forms/forms.py @@ -25,6 +25,10 @@ class ImportForm(forms.Form): csv_file = forms.FileField() +class ImportUserForm(forms.Form): + archive_file = forms.FileField() + + class ShelfForm(CustomForm): class Meta: model = models.Shelf diff --git a/bookwyrm/importers/__init__.py b/bookwyrm/importers/__init__.py index 6ce50f160..8e92872f2 100644 --- a/bookwyrm/importers/__init__.py +++ b/bookwyrm/importers/__init__.py @@ -1,6 +1,7 @@ """ import classes """ from .importer import Importer +from .bookwyrm_import import BookwyrmImporter from .calibre_import import CalibreImporter from .goodreads_import import GoodreadsImporter from .librarything_import import LibrarythingImporter diff --git a/bookwyrm/importers/bookwyrm_import.py b/bookwyrm/importers/bookwyrm_import.py new file mode 100644 index 000000000..206cd6219 --- /dev/null +++ b/bookwyrm/importers/bookwyrm_import.py @@ -0,0 +1,24 @@ +"""Import data from Bookwyrm export files""" +from django.http import QueryDict + +from bookwyrm.models import User +from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob + + +class BookwyrmImporter: + """Import a Bookwyrm User export file. + This is kind of a combination of an importer and a connector. + """ + + # pylint: disable=no-self-use + def process_import( + self, user: User, archive_file: bytes, settings: QueryDict + ) -> BookwyrmImportJob: + """import user data from a Bookwyrm export file""" + + required = [k for k in settings if settings.get(k) == "on"] + + job = BookwyrmImportJob.objects.create( + user=user, archive_file=archive_file, required=required + ) + return job diff --git a/bookwyrm/migrations/0186_auto_20231116_0048.py b/bookwyrm/migrations/0186_auto_20231116_0048.py new file mode 100644 index 000000000..e3b9da4fe --- /dev/null +++ b/bookwyrm/migrations/0186_auto_20231116_0048.py @@ -0,0 +1,212 @@ +# Generated by Django 3.2.20 on 2023-11-16 00:48 + +from django.conf import settings +import django.contrib.postgres.fields +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0185_alter_notification_notification_type"), + ] + + operations = [ + migrations.CreateModel( + name="ParentJob", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("task_id", models.UUIDField(blank=True, null=True, unique=True)), + ( + "created_date", + models.DateTimeField(default=django.utils.timezone.now), + ), + ( + "updated_date", + models.DateTimeField(default=django.utils.timezone.now), + ), + ("complete", models.BooleanField(default=False)), + ( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ("failed", "Failed"), + ], + default="pending", + max_length=50, + null=True, + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "abstract": False, + }, + ), + migrations.AddField( + model_name="sitesettings", + name="user_import_time_limit", + field=models.IntegerField(default=48), + ), + migrations.AlterField( + model_name="notification", + name="notification_type", + field=models.CharField( + choices=[ + ("FAVORITE", "Favorite"), + ("BOOST", "Boost"), + ("REPLY", "Reply"), + ("MENTION", "Mention"), + ("TAG", "Tag"), + ("FOLLOW", "Follow"), + ("FOLLOW_REQUEST", "Follow Request"), + ("IMPORT", "Import"), + ("USER_IMPORT", "User Import"), + ("USER_EXPORT", "User Export"), + ("ADD", "Add"), + ("REPORT", "Report"), + ("LINK_DOMAIN", "Link Domain"), + ("INVITE", "Invite"), + ("ACCEPT", "Accept"), + ("JOIN", "Join"), + ("LEAVE", "Leave"), + ("REMOVE", "Remove"), + ("GROUP_PRIVACY", "Group Privacy"), + ("GROUP_NAME", "Group Name"), + ("GROUP_DESCRIPTION", "Group Description"), + ("MOVE", "Move"), + ], + max_length=255, + ), + ), + migrations.CreateModel( + name="BookwyrmExportJob", + fields=[ + ( + "parentjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.parentjob", + ), + ), + ("export_data", models.FileField(null=True, upload_to="")), + ], + options={ + "abstract": False, + }, + bases=("bookwyrm.parentjob",), + ), + migrations.CreateModel( + name="BookwyrmImportJob", + fields=[ + ( + "parentjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.parentjob", + ), + ), + ("archive_file", models.FileField(blank=True, null=True, upload_to="")), + ("import_data", models.JSONField(null=True)), + ( + "required", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(blank=True, max_length=50), + blank=True, + size=None, + ), + ), + ], + options={ + "abstract": False, + }, + bases=("bookwyrm.parentjob",), + ), + migrations.CreateModel( + name="ChildJob", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("task_id", models.UUIDField(blank=True, null=True, unique=True)), + ( + "created_date", + models.DateTimeField(default=django.utils.timezone.now), + ), + ( + "updated_date", + models.DateTimeField(default=django.utils.timezone.now), + ), + ("complete", models.BooleanField(default=False)), + ( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ("failed", "Failed"), + ], + default="pending", + max_length=50, + null=True, + ), + ), + ( + "parent_job", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="child_jobs", + to="bookwyrm.parentjob", + ), + ), + ], + options={ + "abstract": False, + }, + ), + migrations.AddField( + model_name="notification", + name="related_user_export", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="bookwyrm.bookwyrmexportjob", + ), + ), + ] diff --git a/bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py b/bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py new file mode 100644 index 000000000..eb6238f6e --- /dev/null +++ b/bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.23 on 2023-11-22 10:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0186_auto_20231116_0048"), + ("bookwyrm", "0188_theme_loads"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0190_alter_notification_notification_type.py b/bookwyrm/migrations/0190_alter_notification_notification_type.py new file mode 100644 index 000000000..aff54c77b --- /dev/null +++ b/bookwyrm/migrations/0190_alter_notification_notification_type.py @@ -0,0 +1,45 @@ +# Generated by Django 3.2.23 on 2023-11-23 19:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0189_merge_0186_auto_20231116_0048_0188_theme_loads"), + ] + + operations = [ + migrations.AlterField( + model_name="notification", + name="notification_type", + field=models.CharField( + choices=[ + ("FAVORITE", "Favorite"), + ("BOOST", "Boost"), + ("REPLY", "Reply"), + ("MENTION", "Mention"), + ("TAG", "Tag"), + ("FOLLOW", "Follow"), + ("FOLLOW_REQUEST", "Follow Request"), + ("IMPORT", "Import"), + ("USER_IMPORT", "User Import"), + ("USER_EXPORT", "User Export"), + ("ADD", "Add"), + ("REPORT", "Report"), + ("LINK_DOMAIN", "Link Domain"), + ("INVITE_REQUEST", "Invite Request"), + ("INVITE", "Invite"), + ("ACCEPT", "Accept"), + ("JOIN", "Join"), + ("LEAVE", "Leave"), + ("REMOVE", "Remove"), + ("GROUP_PRIVACY", "Group Privacy"), + ("GROUP_NAME", "Group Name"), + ("GROUP_DESCRIPTION", "Group Description"), + ("MOVE", "Move"), + ], + max_length=255, + ), + ), + ] diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py index e1c862f2b..6bb99c7f2 100644 --- a/bookwyrm/models/__init__.py +++ b/bookwyrm/models/__init__.py @@ -26,6 +26,8 @@ from .federated_server import FederatedServer from .group import Group, GroupMember, GroupMemberInvitation from .import_job import ImportJob, ImportItem +from .bookwyrm_import_job import BookwyrmImportJob +from .bookwyrm_export_job import BookwyrmExportJob from .move import MoveUser diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py new file mode 100644 index 000000000..1f6085e0c --- /dev/null +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -0,0 +1,232 @@ +"""Export user account to tar.gz file for import into another Bookwyrm instance""" + +import dataclasses +import logging +from uuid import uuid4 + +from django.db.models import FileField +from django.db.models import Q +from django.core.serializers.json import DjangoJSONEncoder +from django.core.files.base import ContentFile + +from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, List, ListItem +from bookwyrm.models import Review, Comment, Quotation +from bookwyrm.models import Edition +from bookwyrm.models import UserFollows, User, UserBlocks +from bookwyrm.models.job import ParentJob, ParentTask +from bookwyrm.tasks import app, IMPORTS +from bookwyrm.utils.tar import BookwyrmTarFile + +logger = logging.getLogger(__name__) + + +class BookwyrmExportJob(ParentJob): + """entry for a specific request to export a bookwyrm user""" + + export_data = FileField(null=True) + + def start_job(self): + """Start the job""" + start_export_task.delay(job_id=self.id, no_children=True) + + return self + + +@app.task(queue=IMPORTS, base=ParentTask) +def start_export_task(**kwargs): + """trigger the child tasks for each row""" + job = BookwyrmExportJob.objects.get(id=kwargs["job_id"]) + + # don't start the job if it was stopped from the UI + if job.complete: + return + try: + # This is where ChildJobs get made + job.export_data = ContentFile(b"", str(uuid4())) + json_data = json_export(job.user) + tar_export(json_data, job.user, job.export_data) + job.save(update_fields=["export_data"]) + except Exception as err: # pylint: disable=broad-except + logger.exception("User Export Job %s Failed with error: %s", job.id, err) + job.set_status("failed") + + job.set_status("complete") + + +def tar_export(json_data: str, user, file): + """wrap the export information in a tar file""" + file.open("wb") + with BookwyrmTarFile.open(mode="w:gz", fileobj=file) as tar: + tar.write_bytes(json_data.encode("utf-8")) + + # Add avatar image if present + if getattr(user, "avatar", False): + tar.add_image(user.avatar, filename="avatar") + + editions = get_books_for_user(user) + for book in editions: + if getattr(book, "cover", False): + tar.add_image(book.cover) + + file.close() + + +def json_export( + user, +): # pylint: disable=too-many-locals, too-many-statements, too-many-branches + """Generate an export for a user""" + + # User as AP object + exported_user = user.to_activity() + # I don't love this but it prevents a JSON encoding error + # when there is no user image + if isinstance( + exported_user["icon"], + dataclasses._MISSING_TYPE, # pylint: disable=protected-access + ): + exported_user["icon"] = {} + else: + # change the URL to be relative to the JSON file + file_type = exported_user["icon"]["url"].rsplit(".", maxsplit=1)[-1] + filename = f"avatar.{file_type}" + exported_user["icon"]["url"] = filename + + # Additional settings - can't be serialized as AP + vals = [ + "show_goal", + "preferred_timezone", + "default_post_privacy", + "show_suggested_users", + ] + exported_user["settings"] = {} + for k in vals: + exported_user["settings"][k] = getattr(user, k) + + # Reading goals - can't be serialized as AP + reading_goals = AnnualGoal.objects.filter(user=user).distinct() + exported_user["goals"] = [] + for goal in reading_goals: + exported_user["goals"].append( + {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy} + ) + + # Reading history - can't be serialized as AP + readthroughs = ReadThrough.objects.filter(user=user).distinct().values() + readthroughs = list(readthroughs) + + # Books + editions = get_books_for_user(user) + exported_user["books"] = [] + + for edition in editions: + book = {} + book["work"] = edition.parent_work.to_activity() + book["edition"] = edition.to_activity() + + if book["edition"].get("cover"): + # change the URL to be relative to the JSON file + filename = book["edition"]["cover"]["url"].rsplit("/", maxsplit=1)[-1] + book["edition"]["cover"]["url"] = f"covers/{filename}" + + # authors + book["authors"] = [] + for author in edition.authors.all(): + book["authors"].append(author.to_activity()) + + # Shelves this book is on + # Every ShelfItem is this book so we don't other serializing + book["shelves"] = [] + shelf_books = ( + ShelfBook.objects.select_related("shelf") + .filter(user=user, book=edition) + .distinct() + ) + + for shelfbook in shelf_books: + book["shelves"].append(shelfbook.shelf.to_activity()) + + # Lists and ListItems + # ListItems include "notes" and "approved" so we need them + # even though we know it's this book + book["lists"] = [] + list_items = ListItem.objects.filter(book=edition, user=user).distinct() + + for item in list_items: + list_info = item.book_list.to_activity() + list_info[ + "privacy" + ] = item.book_list.privacy # this isn't serialized so we add it + list_info["list_item"] = item.to_activity() + book["lists"].append(list_info) + + # Statuses + # Can't use select_subclasses here because + # we need to filter on the "book" value, + # which is not available on an ordinary Status + for status in ["comments", "quotations", "reviews"]: + book[status] = [] + + comments = Comment.objects.filter(user=user, book=edition).all() + for status in comments: + obj = status.to_activity() + obj["progress"] = status.progress + obj["progress_mode"] = status.progress_mode + book["comments"].append(obj) + + quotes = Quotation.objects.filter(user=user, book=edition).all() + for status in quotes: + obj = status.to_activity() + obj["position"] = status.position + obj["endposition"] = status.endposition + obj["position_mode"] = status.position_mode + book["quotations"].append(obj) + + reviews = Review.objects.filter(user=user, book=edition).all() + for status in reviews: + obj = status.to_activity() + book["reviews"].append(obj) + + # readthroughs can't be serialized to activity + book_readthroughs = ( + ReadThrough.objects.filter(user=user, book=edition).distinct().values() + ) + book["readthroughs"] = list(book_readthroughs) + + # append everything + exported_user["books"].append(book) + + # saved book lists - just the remote id + saved_lists = List.objects.filter(id__in=user.saved_lists.all()).distinct() + exported_user["saved_lists"] = [l.remote_id for l in saved_lists] + + # follows - just the remote id + follows = UserFollows.objects.filter(user_subject=user).distinct() + following = User.objects.filter(userfollows_user_object__in=follows).distinct() + exported_user["follows"] = [f.remote_id for f in following] + + # blocks - just the remote id + blocks = UserBlocks.objects.filter(user_subject=user).distinct() + blocking = User.objects.filter(userblocks_user_object__in=blocks).distinct() + + exported_user["blocks"] = [b.remote_id for b in blocking] + + return DjangoJSONEncoder().encode(exported_user) + + +def get_books_for_user(user): + """Get all the books and editions related to a user""" + + editions = ( + Edition.objects.select_related("parent_work") + .filter( + Q(shelves__user=user) + | Q(readthrough__user=user) + | Q(review__user=user) + | Q(list__user=user) + | Q(comment__user=user) + | Q(quotation__user=user) + ) + .distinct() + ) + + return editions diff --git a/bookwyrm/models/bookwyrm_import_job.py b/bookwyrm/models/bookwyrm_import_job.py new file mode 100644 index 000000000..9a11fd932 --- /dev/null +++ b/bookwyrm/models/bookwyrm_import_job.py @@ -0,0 +1,459 @@ +"""Import a user from another Bookwyrm instance""" + +import json +import logging + +from django.db.models import FileField, JSONField, CharField +from django.utils import timezone +from django.utils.html import strip_tags +from django.contrib.postgres.fields import ArrayField as DjangoArrayField + +from bookwyrm import activitypub +from bookwyrm import models +from bookwyrm.tasks import app, IMPORTS +from bookwyrm.models.job import ParentJob, ParentTask, SubTask +from bookwyrm.utils.tar import BookwyrmTarFile + +logger = logging.getLogger(__name__) + + +class BookwyrmImportJob(ParentJob): + """entry for a specific request for importing a bookwyrm user backup""" + + archive_file = FileField(null=True, blank=True) + import_data = JSONField(null=True) + required = DjangoArrayField(CharField(max_length=50, blank=True), blank=True) + + def start_job(self): + """Start the job""" + start_import_task.delay(job_id=self.id, no_children=True) + + +@app.task(queue=IMPORTS, base=ParentTask) +def start_import_task(**kwargs): + """trigger the child import tasks for each user data""" + job = BookwyrmImportJob.objects.get(id=kwargs["job_id"]) + archive_file = job.archive_file + + # don't start the job if it was stopped from the UI + if job.complete: + return + + try: + archive_file.open("rb") + with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: + job.import_data = json.loads(tar.read("archive.json").decode("utf-8")) + + if "include_user_profile" in job.required: + update_user_profile(job.user, tar, job.import_data) + if "include_user_settings" in job.required: + update_user_settings(job.user, job.import_data) + if "include_goals" in job.required: + update_goals(job.user, job.import_data.get("goals")) + if "include_saved_lists" in job.required: + upsert_saved_lists(job.user, job.import_data.get("saved_lists")) + if "include_follows" in job.required: + upsert_follows(job.user, job.import_data.get("follows")) + if "include_blocks" in job.required: + upsert_user_blocks(job.user, job.import_data.get("blocks")) + + process_books(job, tar) + + job.set_status("complete") + archive_file.close() + + except Exception as err: # pylint: disable=broad-except + logger.exception("User Import Job %s Failed with error: %s", job.id, err) + job.set_status("failed") + + +def process_books(job, tar): + """ + Process user import data related to books + We always import the books even if not assigning + them to shelves, lists etc + """ + + books = job.import_data.get("books") + + for data in books: + book = get_or_create_edition(data, tar) + + if "include_shelves" in job.required: + upsert_shelves(book, job.user, data) + + if "include_readthroughs" in job.required: + upsert_readthroughs(data.get("readthroughs"), job.user, book.id) + + if "include_comments" in job.required: + upsert_statuses( + job.user, models.Comment, data.get("comments"), book.remote_id + ) + if "include_quotations" in job.required: + upsert_statuses( + job.user, models.Quotation, data.get("quotations"), book.remote_id + ) + + if "include_reviews" in job.required: + upsert_statuses( + job.user, models.Review, data.get("reviews"), book.remote_id + ) + + if "include_lists" in job.required: + upsert_lists(job.user, data.get("lists"), book.id) + + +def get_or_create_edition(book_data, tar): + """Take a JSON string of work and edition data, + find or create the edition and work in the database and + return an edition instance""" + + edition = book_data.get("edition") + existing = models.Edition.find_existing(edition) + if existing: + return existing + + # make sure we have the authors in the local DB + # replace the old author ids in the edition JSON + edition["authors"] = [] + for author in book_data.get("authors"): + parsed_author = activitypub.parse(author) + instance = parsed_author.to_model( + model=models.Author, save=True, overwrite=True + ) + + edition["authors"].append(instance.remote_id) + + # we will add the cover later from the tar + # don't try to load it from the old server + cover = edition.get("cover", {}) + cover_path = cover.get("url", None) + edition["cover"] = {} + + # first we need the parent work to exist + work = book_data.get("work") + work["editions"] = [] + parsed_work = activitypub.parse(work) + work_instance = parsed_work.to_model(model=models.Work, save=True, overwrite=True) + + # now we have a work we can add it to the edition + # and create the edition model instance + edition["work"] = work_instance.remote_id + parsed_edition = activitypub.parse(edition) + book = parsed_edition.to_model(model=models.Edition, save=True, overwrite=True) + + # set the cover image from the tar + if cover_path: + tar.write_image_to_file(cover_path, book.cover) + + return book + + +def upsert_readthroughs(data, user, book_id): + """Take a JSON string of readthroughs and + find or create the instances in the database""" + + for read_through in data: + + obj = {} + keys = [ + "progress_mode", + "start_date", + "finish_date", + "stopped_date", + "is_active", + ] + for key in keys: + obj[key] = read_through[key] + obj["user_id"] = user.id + obj["book_id"] = book_id + + existing = models.ReadThrough.objects.filter(**obj).first() + if not existing: + models.ReadThrough.objects.create(**obj) + + +def upsert_statuses(user, cls, data, book_remote_id): + """Take a JSON string of a status and + find or create the instances in the database""" + + for status in data: + if is_alias( + user, status["attributedTo"] + ): # don't let l33t hax0rs steal other people's posts + # update ids and remove replies + status["attributedTo"] = user.remote_id + status["to"] = update_followers_address(user, status["to"]) + status["cc"] = update_followers_address(user, status["cc"]) + status[ + "replies" + ] = ( + {} + ) # this parses incorrectly but we can't set it without knowing the new id + status["inReplyToBook"] = book_remote_id + parsed = activitypub.parse(status) + if not status_already_exists( + user, parsed + ): # don't duplicate posts on multiple import + + instance = parsed.to_model(model=cls, save=True, overwrite=True) + + for val in [ + "progress", + "progress_mode", + "position", + "endposition", + "position_mode", + ]: + if status.get(val): + instance.val = status[val] + + instance.remote_id = instance.get_remote_id() # update the remote_id + instance.save() # save and broadcast + + else: + logger.info("User does not have permission to import statuses") + + +def upsert_lists(user, lists, book_id): + """Take a list of objects each containing + a list and list item as AP objects + + Because we are creating new IDs we can't assume the id + will exist or be accurate, so we only use to_model for + adding new items after checking whether they exist . + + """ + + book = models.Edition.objects.get(id=book_id) + + for blist in lists: + booklist = models.List.objects.filter(name=blist["name"], user=user).first() + if not booklist: + + blist["owner"] = user.remote_id + parsed = activitypub.parse(blist) + booklist = parsed.to_model(model=models.List, save=True, overwrite=True) + + booklist.privacy = blist["privacy"] + booklist.save() + + item = models.ListItem.objects.filter(book=book, book_list=booklist).exists() + if not item: + count = booklist.books.count() + models.ListItem.objects.create( + book=book, + book_list=booklist, + user=user, + notes=blist["list_item"]["notes"], + approved=blist["list_item"]["approved"], + order=count + 1, + ) + + +def upsert_shelves(book, user, book_data): + """Take shelf JSON objects and create + DB entries if they don't already exist""" + + shelves = book_data["shelves"] + for shelf in shelves: + + book_shelf = models.Shelf.objects.filter(name=shelf["name"], user=user).first() + + if not book_shelf: + book_shelf = models.Shelf.objects.create(name=shelf["name"], user=user) + + # add the book as a ShelfBook if needed + if not models.ShelfBook.objects.filter( + book=book, shelf=book_shelf, user=user + ).exists(): + models.ShelfBook.objects.create( + book=book, shelf=book_shelf, user=user, shelved_date=timezone.now() + ) + + +def update_user_profile(user, tar, data): + """update the user's profile from import data""" + name = data.get("name", None) + username = data.get("preferredUsername") + user.name = name if name else username + user.summary = strip_tags(data.get("summary", None)) + user.save(update_fields=["name", "summary"]) + if data["icon"].get("url"): + avatar_filename = next(filter(lambda n: n.startswith("avatar"), tar.getnames())) + tar.write_image_to_file(avatar_filename, user.avatar) + + +def update_user_settings(user, data): + """update the user's settings from import data""" + + update_fields = ["manually_approves_followers", "hide_follows", "discoverable"] + + ap_fields = [ + ("manuallyApprovesFollowers", "manually_approves_followers"), + ("hideFollows", "hide_follows"), + ("discoverable", "discoverable"), + ] + + for (ap_field, bw_field) in ap_fields: + setattr(user, bw_field, data[ap_field]) + + bw_fields = [ + "show_goal", + "show_suggested_users", + "default_post_privacy", + "preferred_timezone", + ] + + for field in bw_fields: + update_fields.append(field) + setattr(user, field, data["settings"][field]) + + user.save(update_fields=update_fields) + + +@app.task(queue=IMPORTS, base=SubTask) +def update_user_settings_task(job_id): + """wrapper task for user's settings import""" + parent_job = BookwyrmImportJob.objects.get(id=job_id) + + return update_user_settings(parent_job.user, parent_job.import_data.get("user")) + + +def update_goals(user, data): + """update the user's goals from import data""" + + for goal in data: + # edit the existing goal if there is one + existing = models.AnnualGoal.objects.filter( + year=goal["year"], user=user + ).first() + if existing: + for k in goal.keys(): + setattr(existing, k, goal[k]) + existing.save() + else: + goal["user"] = user + models.AnnualGoal.objects.create(**goal) + + +@app.task(queue=IMPORTS, base=SubTask) +def update_goals_task(job_id): + """wrapper task for user's goals import""" + parent_job = BookwyrmImportJob.objects.get(id=job_id) + + return update_goals(parent_job.user, parent_job.import_data.get("goals")) + + +def upsert_saved_lists(user, values): + """Take a list of remote ids and add as saved lists""" + + for remote_id in values: + book_list = activitypub.resolve_remote_id(remote_id, models.List) + if book_list: + user.saved_lists.add(book_list) + + +@app.task(queue=IMPORTS, base=SubTask) +def upsert_saved_lists_task(job_id): + """wrapper task for user's saved lists import""" + parent_job = BookwyrmImportJob.objects.get(id=job_id) + + return upsert_saved_lists( + parent_job.user, parent_job.import_data.get("saved_lists") + ) + + +def upsert_follows(user, values): + """Take a list of remote ids and add as follows""" + + for remote_id in values: + followee = activitypub.resolve_remote_id(remote_id, models.User) + if followee: + (follow_request, created,) = models.UserFollowRequest.objects.get_or_create( + user_subject=user, + user_object=followee, + ) + + if not created: + # this request probably failed to connect with the remote + # and should save to trigger a re-broadcast + follow_request.save() + + +@app.task(queue=IMPORTS, base=SubTask) +def upsert_follows_task(job_id): + """wrapper task for user's follows import""" + parent_job = BookwyrmImportJob.objects.get(id=job_id) + + return upsert_follows(parent_job.user, parent_job.import_data.get("follows")) + + +def upsert_user_blocks(user, user_ids): + """block users""" + + for user_id in user_ids: + user_object = activitypub.resolve_remote_id(user_id, models.User) + if user_object: + exists = models.UserBlocks.objects.filter( + user_subject=user, user_object=user_object + ).exists() + if not exists: + models.UserBlocks.objects.create( + user_subject=user, user_object=user_object + ) + # remove the blocked users's lists from the groups + models.List.remove_from_group(user, user_object) + # remove the blocked user from all blocker's owned groups + models.GroupMember.remove(user, user_object) + + +@app.task(queue=IMPORTS, base=SubTask) +def upsert_user_blocks_task(job_id): + """wrapper task for user's blocks import""" + parent_job = BookwyrmImportJob.objects.get(id=job_id) + + return upsert_user_blocks( + parent_job.user, parent_job.import_data.get("blocked_users") + ) + + +def update_followers_address(user, field): + """statuses to or cc followers need to have the followers + address updated to the new local user""" + + for i, audience in enumerate(field): + if audience.rsplit("/")[-1] == "followers": + field[i] = user.followers_url + + return field + + +def is_alias(user, remote_id): + """check that the user is listed as movedTo or also_known_as + in the remote user's profile""" + + remote_user = activitypub.resolve_remote_id( + remote_id=remote_id, model=models.User, save=False + ) + + if remote_user: + + if remote_user.moved_to: + return user.remote_id == remote_user.moved_to + + if remote_user.also_known_as: + return user in remote_user.also_known_as.all() + + return False + + +def status_already_exists(user, status): + """check whether this status has already been published + by this user. We can't rely on to_model() because it + only matches on remote_id, which we have to change + *after* saving because it needs the primary key (id)""" + + return models.Status.objects.filter( + user=user, content=status.content, published_date=status.published + ).exists() diff --git a/bookwyrm/models/job.py b/bookwyrm/models/job.py new file mode 100644 index 000000000..4f5cb2093 --- /dev/null +++ b/bookwyrm/models/job.py @@ -0,0 +1,308 @@ +"""Everything needed for Celery to multi-thread complex tasks.""" + +from django.db import models +from django.db import transaction +from django.utils.translation import gettext_lazy as _ +from django.utils import timezone +from bookwyrm.models.user import User + +from bookwyrm.tasks import app + + +class Job(models.Model): + """Abstract model to store the state of a Task.""" + + class Status(models.TextChoices): + """Possible job states.""" + + PENDING = "pending", _("Pending") + ACTIVE = "active", _("Active") + COMPLETE = "complete", _("Complete") + STOPPED = "stopped", _("Stopped") + FAILED = "failed", _("Failed") + + task_id = models.UUIDField(unique=True, null=True, blank=True) + + created_date = models.DateTimeField(default=timezone.now) + updated_date = models.DateTimeField(default=timezone.now) + complete = models.BooleanField(default=False) + status = models.CharField( + max_length=50, choices=Status.choices, default=Status.PENDING, null=True + ) + + class Meta: + """Make it abstract""" + + abstract = True + + def complete_job(self): + """Report that the job has completed""" + if self.complete: + return + + self.status = self.Status.COMPLETE + self.complete = True + self.updated_date = timezone.now() + + self.save(update_fields=["status", "complete", "updated_date"]) + + def stop_job(self, reason=None): + """Stop the job""" + if self.complete: + return + + self.__terminate_job() + + if reason and reason == "failed": + self.status = self.Status.FAILED + else: + self.status = self.Status.STOPPED + self.complete = True + self.updated_date = timezone.now() + + self.save(update_fields=["status", "complete", "updated_date"]) + + def set_status(self, status): + """Set job status""" + if self.complete: + return + + if self.status == status: + return + + if status == self.Status.COMPLETE: + self.complete_job() + return + + if status == self.Status.STOPPED: + self.stop_job() + return + + if status == self.Status.FAILED: + self.stop_job(reason="failed") + return + + self.updated_date = timezone.now() + self.status = status + + self.save(update_fields=["status", "updated_date"]) + + def __terminate_job(self): + """Tell workers to ignore and not execute this task.""" + app.control.revoke(self.task_id, terminate=True) + + +class ParentJob(Job): + """Store the state of a Task which can spawn many :model:`ChildJob`s to spread + resource load. + + Intended to be sub-classed if necessary via proxy or + multi-table inheritance. + Extends :model:`Job`. + """ + + user = models.ForeignKey(User, on_delete=models.CASCADE) + + def complete_job(self): + """Report that the job has completed and stop pending + children. Extend. + """ + super().complete_job() + self.__terminate_pending_child_jobs() + + def notify_child_job_complete(self): + """let the job know when the items get work done""" + if self.complete: + return + + self.updated_date = timezone.now() + self.save(update_fields=["updated_date"]) + + if not self.complete and self.has_completed: + self.complete_job() + + def __terminate_job(self): # pylint: disable=unused-private-member + """Tell workers to ignore and not execute this task + & pending child tasks. Extend. + """ + super().__terminate_job() + self.__terminate_pending_child_jobs() + + def __terminate_pending_child_jobs(self): + """Tell workers to ignore and not execute any pending child tasks.""" + tasks = self.pending_child_jobs.filter(task_id__isnull=False).values_list( + "task_id", flat=True + ) + app.control.revoke(list(tasks)) + + for task in self.pending_child_jobs: + task.update(status=self.Status.STOPPED) + + @property + def has_completed(self): + """has this job finished""" + return not self.pending_child_jobs.exists() + + @property + def pending_child_jobs(self): + """items that haven't been processed yet""" + return self.child_jobs.filter(complete=False) + + +class ChildJob(Job): + """Stores the state of a Task for the related :model:`ParentJob`. + + Intended to be sub-classed if necessary via proxy or + multi-table inheritance. + Extends :model:`Job`. + """ + + parent_job = models.ForeignKey( + ParentJob, on_delete=models.CASCADE, related_name="child_jobs" + ) + + def set_status(self, status): + """Set job and parent_job status. Extend.""" + super().set_status(status) + + if ( + status == self.Status.ACTIVE + and self.parent_job.status == self.Status.PENDING + ): + self.parent_job.set_status(self.Status.ACTIVE) + + def complete_job(self): + """Report to parent_job that the job has completed. Extend.""" + super().complete_job() + self.parent_job.notify_child_job_complete() + + +class ParentTask(app.Task): + """Used with ParentJob, Abstract Tasks execute code at specific points in + a Task's lifecycle, applying to all Tasks with the same 'base'. + + All status & ParentJob.task_id assignment is managed here for you. + Usage e.g. @app.task(base=ParentTask) + """ + + def before_start( + self, task_id, args, kwargs + ): # pylint: disable=no-self-use, unused-argument + """Handler called before the task starts. Override. + + Prepare ParentJob before the task starts. + + Arguments: + task_id (str): Unique id of the task to execute. + args (Tuple): Original arguments for the task to execute. + kwargs (Dict): Original keyword arguments for the task to execute. + + Keyword Arguments: + job_id (int): Unique 'id' of the ParentJob. + no_children (bool): If 'True' this is the only Task expected to run + for the given ParentJob. + + Returns: + None: The return value of this handler is ignored. + """ + job = ParentJob.objects.get(id=kwargs["job_id"]) + job.task_id = task_id + job.save(update_fields=["task_id"]) + + if kwargs["no_children"]: + job.set_status(ChildJob.Status.ACTIVE) + + def on_success( + self, retval, task_id, args, kwargs + ): # pylint: disable=no-self-use, unused-argument + """Run by the worker if the task executes successfully. Override. + + Update ParentJob on Task complete. + + Arguments: + retval (Any): The return value of the task. + task_id (str): Unique id of the executed task. + args (Tuple): Original arguments for the executed task. + kwargs (Dict): Original keyword arguments for the executed task. + + Keyword Arguments: + job_id (int): Unique 'id' of the ParentJob. + no_children (bool): If 'True' this is the only Task expected to run + for the given ParentJob. + + Returns: + None: The return value of this handler is ignored. + """ + + if kwargs["no_children"]: + job = ParentJob.objects.get(id=kwargs["job_id"]) + job.complete_job() + + +class SubTask(app.Task): + """Used with ChildJob, Abstract Tasks execute code at specific points in + a Task's lifecycle, applying to all Tasks with the same 'base'. + + All status & ChildJob.task_id assignment is managed here for you. + Usage e.g. @app.task(base=SubTask) + """ + + def before_start( + self, task_id, args, kwargs + ): # pylint: disable=no-self-use, unused-argument + """Handler called before the task starts. Override. + + Prepare ChildJob before the task starts. + + Arguments: + task_id (str): Unique id of the task to execute. + args (Tuple): Original arguments for the task to execute. + kwargs (Dict): Original keyword arguments for the task to execute. + + Keyword Arguments: + job_id (int): Unique 'id' of the ParentJob. + child_id (int): Unique 'id' of the ChildJob. + + Returns: + None: The return value of this handler is ignored. + """ + child_job = ChildJob.objects.get(id=kwargs["child_id"]) + child_job.task_id = task_id + child_job.save(update_fields=["task_id"]) + child_job.set_status(ChildJob.Status.ACTIVE) + + def on_success( + self, retval, task_id, args, kwargs + ): # pylint: disable=no-self-use, unused-argument + """Run by the worker if the task executes successfully. Override. + + Notify ChildJob of task completion. + + Arguments: + retval (Any): The return value of the task. + task_id (str): Unique id of the executed task. + args (Tuple): Original arguments for the executed task. + kwargs (Dict): Original keyword arguments for the executed task. + + Keyword Arguments: + job_id (int): Unique 'id' of the ParentJob. + child_id (int): Unique 'id' of the ChildJob. + + Returns: + None: The return value of this handler is ignored. + """ + subtask = ChildJob.objects.get(id=kwargs["child_id"]) + subtask.complete_job() + + +@transaction.atomic +def create_child_job(parent_job, task_callback): + """Utility method for creating a ChildJob + and running a task to avoid DB race conditions + """ + child_job = ChildJob.objects.create(parent_job=parent_job) + transaction.on_commit( + lambda: task_callback.delay(job_id=parent_job.id, child_id=child_job.id) + ) + + return child_job diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py index d056c05b3..ca1e2aeb0 100644 --- a/bookwyrm/models/notification.py +++ b/bookwyrm/models/notification.py @@ -1,8 +1,16 @@ """ alert a user to activity """ from django.db import models, transaction from django.dispatch import receiver +from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from .base_model import BookWyrmModel -from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain +from . import ( + Boost, + Favorite, + GroupMemberInvitation, + ImportJob, + BookwyrmImportJob, + LinkDomain, +) from . import ListItem, Report, Status, User, UserFollowRequest from .site import InviteRequest @@ -23,6 +31,8 @@ class NotificationType(models.TextChoices): # Imports IMPORT = "IMPORT" + USER_IMPORT = "USER_IMPORT" + USER_EXPORT = "USER_EXPORT" # List activity ADD = "ADD" @@ -63,6 +73,9 @@ class Notification(BookWyrmModel): ) related_status = models.ForeignKey("Status", on_delete=models.CASCADE, null=True) related_import = models.ForeignKey("ImportJob", on_delete=models.CASCADE, null=True) + related_user_export = models.ForeignKey( + "BookwyrmExportJob", on_delete=models.CASCADE, null=True + ) related_list_items = models.ManyToManyField( "ListItem", symmetrical=False, related_name="notifications" ) @@ -226,6 +239,36 @@ def notify_user_on_import_complete( ) +@receiver(models.signals.post_save, sender=BookwyrmImportJob) +# pylint: disable=unused-argument +def notify_user_on_user_import_complete( + sender, instance, *args, update_fields=None, **kwargs +): + """we imported your user details! aren't you proud of us""" + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: + return + Notification.objects.create( + user=instance.user, notification_type=NotificationType.USER_IMPORT + ) + + +@receiver(models.signals.post_save, sender=BookwyrmExportJob) +# pylint: disable=unused-argument +def notify_user_on_user_export_complete( + sender, instance, *args, update_fields=None, **kwargs +): + """we exported your user details! aren't you proud of us""" + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: + return + Notification.objects.create( + user=instance.user, + notification_type=NotificationType.USER_EXPORT, + related_user_export=instance, + ) + + @receiver(models.signals.post_save, sender=Report) @transaction.atomic # pylint: disable=unused-argument diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index b962d597b..bd53f1f07 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -96,6 +96,7 @@ class SiteSettings(SiteModel): imports_enabled = models.BooleanField(default=True) import_size_limit = models.IntegerField(default=0) import_limit_reset = models.IntegerField(default=0) + user_import_time_limit = models.IntegerField(default=48) field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"]) diff --git a/bookwyrm/templates/import/import.html b/bookwyrm/templates/import/import.html index 2c3be9e07..01014fa94 100644 --- a/bookwyrm/templates/import/import.html +++ b/bookwyrm/templates/import/import.html @@ -1,13 +1,12 @@ -{% extends 'layout.html' %} +{% extends 'preferences/layout.html' %} {% load i18n %} {% load humanize %} -{% block title %}{% trans "Import Books" %}{% endblock %} +{% block title %}{% trans "Import Book List" %}{% endblock %} +{% block header %}{% trans "Import Book List" %}{% endblock %} -{% block content %} +{% block panel %}
-

{% trans "Import Books" %}

- {% if invalid %}
{% trans "Not a valid CSV file" %} diff --git a/bookwyrm/templates/import/import_user.html b/bookwyrm/templates/import/import_user.html new file mode 100644 index 000000000..70b21673c --- /dev/null +++ b/bookwyrm/templates/import/import_user.html @@ -0,0 +1,222 @@ +{% extends 'preferences/layout.html' %} +{% load i18n %} +{% load humanize %} + +{% block title %}{% trans "Import BookWyrm Account" %}{% endblock %} +{% block header %}{% trans "Import BookWyrm Account" %}{% endblock %} + +{% block panel %} +
+ + {% if invalid %} +
+ {% trans "Not a valid import file" %} +
+ {% endif %} +

+ {% spaceless %} + {% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set this account as an alias of the one you are migrating from, or move that account to this one, before you import your user data." %} + {% endspaceless %} +

+ {% if not site.imports_enabled %} +
+

+ +

+

+ {% trans "Imports are temporarily disabled; thank you for your patience." %} +

+
+ {% elif next_available %} +
+

{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}

+

{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}

+
+ {% else %} +
+ {% csrf_token %} + +
+
+

{% trans "Step 1:" %}

+

+ {% blocktrans trimmed %} + Select an export file generated from another BookWyrm account. The file format should be .tar.gz. + {% endblocktrans %} +

+
+
+ + {{ import_form.archive_file }} +
+
+ + + +
+
+

{% trans "Step 2:" %}

+

+ {% blocktrans trimmed %} + Deselect any checkboxes for data you do not wish to include in your import. + {% endblocktrans %} +

+

Unless specified below, importing will not delete any data. Imported data will be added if it does not already exist. For example, if you have an existing list with the same name as an imported list, the existing list settings will not change, any new list items will be added, and no existing list items will be deleted.

+
+
+
+
+ +

+ {% trans "Overwrites display name, summary, and avatar" %} +

+
+
+ +
+ {% trans "Overwrites:" %} +
    +
  • + {% trans "Whether manual approval is required for other users to follow your account" %} +
    • +
  • + {% trans "Whether following/followers are shown on your profile" %} +
    • +
  • + {% trans "Whether your reading goal is shown on your profile" %} +
    • +
  • + {% trans "Whether you see user follow suggestions" %} +
    • +
  • + {% trans "Whether your account is suggested to others" %} +
    • +
  • + {% trans "Your timezone" %} +
    • +
  • + {% trans "Your default post privacy setting" %} +
    • +
+
+
+
+ +
+ +
+
+
+ +

+ {% trans "Overwrites reading goals for all years listed in the import file" %} +

+
+ + + + + + + +
+
+
+ + {% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %} + + {% else %} + +

{% trans "You've reached the import limit." %}

+ {% endif%} +
+ {% endif %} + +
+ +
+

{% trans "Recent Imports" %}

+
+ + + + + + + {% if not jobs %} + + + + {% endif %} + {% for job in jobs %} + + + + + + {% endfor %} +
+ {% trans "Date Created" %} + + {% trans "Last Updated" %} + + {% trans "Status" %} +
+ {% trans "No recent imports" %} +
+

{{ job.created_date }}

+ 		{{ job.updated_date }} + + {% if job.status %} + {{ job.status }} + {{ job.status_display }} + {% elif job.complete %} + {% trans "Complete" %} + {% else %} + {% trans "Active" %} + {% endif %} + +
+
+ + {% include 'snippets/pagination.html' with page=jobs path=request.path %} +
+{% endblock %} diff --git a/bookwyrm/templates/notifications/item.html b/bookwyrm/templates/notifications/item.html index a69790f52..a1329d31e 100644 --- a/bookwyrm/templates/notifications/item.html +++ b/bookwyrm/templates/notifications/item.html @@ -15,6 +15,10 @@ {% endif %} {% elif notification.notification_type == 'IMPORT' %} {% include 'notifications/items/import.html' %} +{% elif notification.notification_type == 'USER_IMPORT' %} + {% include 'notifications/items/user_import.html' %} +{% elif notification.notification_type == 'USER_EXPORT' %} + {% include 'notifications/items/user_export.html' %} {% elif notification.notification_type == 'ADD' %} {% include 'notifications/items/add.html' %} {% elif notification.notification_type == 'REPORT' %} diff --git a/bookwyrm/templates/notifications/items/user_export.html b/bookwyrm/templates/notifications/items/user_export.html new file mode 100644 index 000000000..1df40dbac --- /dev/null +++ b/bookwyrm/templates/notifications/items/user_export.html @@ -0,0 +1,15 @@ +{% extends 'notifications/items/layout.html' %} +{% load i18n %} + +{% block primary_link %}{% spaceless %} +{% url 'prefs-user-export' %} +{% endspaceless %}{% endblock %} + +{% block icon %} + +{% endblock %} + +{% block description %} + {% url 'prefs-user-export' as url %} + {% blocktrans %}Your user export is ready.{% endblocktrans %} +{% endblock %} diff --git a/bookwyrm/templates/notifications/items/user_import.html b/bookwyrm/templates/notifications/items/user_import.html new file mode 100644 index 000000000..27e3e975d --- /dev/null +++ b/bookwyrm/templates/notifications/items/user_import.html @@ -0,0 +1,16 @@ +{% extends 'notifications/items/layout.html' %} +{% load i18n %} + +{% block primary_link %}{% spaceless %} +{% url 'user-import' %} +{% endspaceless %}{% endblock %} + +{% block icon %} + +{% endblock %} + +{% block description %} +{% url 'user-import' as import_url %} +{% blocktrans %}Your user import is complete.{% endblocktrans %} + +{% endblock %} diff --git a/bookwyrm/templates/preferences/export-user.html b/bookwyrm/templates/preferences/export-user.html new file mode 100644 index 000000000..a468c3f74 --- /dev/null +++ b/bookwyrm/templates/preferences/export-user.html @@ -0,0 +1,138 @@ +{% extends 'preferences/layout.html' %} +{% load i18n %} +{% load utilities %} + +{% block title %}{% trans "Export BookWyrm Account" %}{% endblock %} + +{% block header %} +{% trans "Export BookWyrm Account" %} +{% endblock %} + +{% block panel %} +
+
+

{% trans "You can create an export file here. This will allow you to migrate your data to another BookWyrm account." %}

+
+
+ {% blocktrans trimmed %} +
+

Your file will include:

+
    +
  • User profile
    • +
  • Most user settings
    • +
  • Reading goals
    • +
  • Shelves
    • +
  • Reading history
    • +
  • Book reviews
    • +
  • Statuses
    • +
  • Your own lists and saved lists
    • +
  • Which users you follow and block
    • +
+
+
+

Your file will not include:

+
    +
  • Direct messages
    • +
  • Replies to your statuses
    • +
  • Groups
    • +
  • Favorites
    • +
+
+ {% endblocktrans %} +
+

{% trans "In your new BookWyrm account can choose what to import: you will not have to import everything that is exported." %}

+

+ {% spaceless %} + {% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set the account you are moving to as an alias of this one, or move this account to the new account, before you import your user data." %} + {% endspaceless %} +

+ {% if next_available %} +

+ {% blocktrans trimmed %} + You will be able to create a new export file at {{ next_available }} + {% endblocktrans %} +

+ {% else %} +
+ {% csrf_token %} + +
+ {% endif %} + +
+
+

{% trans "Recent Exports" %}

+

+ {% trans "User export files will show 'complete' once ready. This may take a little while. Click the link to download your file." %} +

+
+ + + + + + + {% if not jobs %} + + + + {% endif %} + {% for job in jobs %} + + + + + + + {% endfor %} +
+ {% trans "Date" %} + + {% trans "Status" %} + + {% trans "Size" %} +
+ {% trans "No recent imports" %} +
{{ job.updated_date }} + + {% if job.status %} + {{ job.status }} + {{ job.status_display }} + {% elif job.complete %} + {% trans "Complete" %} + {% else %} + {% trans "Active" %} + {% endif %} + + + {{ job.export_data|get_file_size }} + + {% if job.complete and not job.status == "stopped" and not job.status == "failed" %} +

+ + + + {% trans "Download your export" %} + + +

+ {% endif %} +
+
+ + {% include 'snippets/pagination.html' with page=jobs path=request.path %} +
+{% endblock %} diff --git a/bookwyrm/templates/preferences/export.html b/bookwyrm/templates/preferences/export.html index 61933be3e..e301eb5cc 100644 --- a/bookwyrm/templates/preferences/export.html +++ b/bookwyrm/templates/preferences/export.html @@ -1,16 +1,16 @@ {% extends 'preferences/layout.html' %} {% load i18n %} -{% block title %}{% trans "CSV Export" %}{% endblock %} +{% block title %}{% trans "Export Book List" %}{% endblock %} {% block header %} -{% trans "CSV Export" %} +{% trans "Export Book List" %} {% endblock %} {% block panel %}

- {% trans "Your export will include all the books on your shelves, books you have reviewed, and books with reading activity." %} + {% trans "Your CSV export file will include all the books on your shelves, books you have reviewed, and books with reading activity.
Use this to import into a service like Goodreads." %}

diff --git a/bookwyrm/templates/preferences/layout.html b/bookwyrm/templates/preferences/layout.html index fb0b6fba6..56151233f 100644 --- a/bookwyrm/templates/preferences/layout.html +++ b/bookwyrm/templates/preferences/layout.html @@ -40,11 +40,19 @@

{% trans "Relationships" %}

diff --git a/bookwyrm/templates/settings/imports/complete_user_import_modal.html b/bookwyrm/templates/settings/imports/complete_user_import_modal.html new file mode 100644 index 000000000..74004b7a2 --- /dev/null +++ b/bookwyrm/templates/settings/imports/complete_user_import_modal.html @@ -0,0 +1,23 @@ +{% extends 'components/modal.html' %} +{% load i18n %} + +{% block modal-title %}{% trans "Stop import?" %}{% endblock %} + +{% block modal-body %} +{% trans "This action will stop the user import before it is complete and cannot be un-done" %} +{% endblock %} + +{% block modal-footer %} + + {% csrf_token %} + +
+ + +
+
+{% endblock %} diff --git a/bookwyrm/templates/settings/imports/imports.html b/bookwyrm/templates/settings/imports/imports.html index 8819220fb..8898aab71 100644 --- a/bookwyrm/templates/settings/imports/imports.html +++ b/bookwyrm/templates/settings/imports/imports.html @@ -29,6 +29,7 @@
{% trans "This is only intended to be used when things have gone very wrong with imports and you need to pause the feature while addressing issues." %} {% trans "While imports are disabled, users will not be allowed to start new imports, but existing imports will not be affected." %} + {% trans "This setting prevents both book imports and user imports." %}
{% csrf_token %}
@@ -89,91 +90,215 @@
+
+ + + {% trans "Limit how often users can import and export" %} + + + +
+
+ {% trans "Some users might try to run user imports or exports very frequently, which you want to limit." %} + {% trans "Set the value to 0 to not enforce any limit." %} +
+
+ + + + {% csrf_token %} +
+ +
+
+
+
-
- +

{% trans "Book Imports" %}

+
+
+ +
+ +
+ + + {% url 'settings-imports' status as url %} + + + + {% if status != "active" %} + + {% endif %} + + + + + {% if status == "active" %} + + {% endif %} + + {% for import in imports %} + + + + + {% if status != "active" %} + + {% endif %} + + + + + {% if status == "active" %} + + {% endif %} + + {% endfor %} + {% if not imports %} + + + + {% endif %} +
+ {% trans "ID" %} + + {% trans "User" as text %} + {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %} + + {% trans "Date Created" as text %} + {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %} + + {% trans "Date Updated" %} + + {% trans "Items" %} + + {% trans "Pending items" %} + + {% trans "Successful items" %} + + {% trans "Failed items" %} + {% trans "Actions" %}
{{ import.id }} + {{ import.user|username }} + {{ import.created_date }}{{ import.updated_date }}{{ import.item_count|intcomma }}{{ import.pending_item_count|intcomma }}{{ import.successful_item_count|intcomma }}{{ import.failed_item_count|intcomma }} + {% join "complete" import.id as modal_id %} + + {% include "settings/imports/complete_import_modal.html" with id=modal_id %} +
+ {% trans "No matching imports found." %} +
+
+ + {% include 'snippets/pagination.html' with page=imports path=request.path %} +
-
- - - {% url 'settings-imports' status as url %} - - - - {% if status != "active" %} - - {% endif %} - - - - - {% if status == "active" %} - - {% endif %} - - {% for import in imports %} - - - - - {% if status != "active" %} - - {% endif %} - - - - - {% if status == "active" %} - - {% endif %} - - {% endfor %} - {% if not imports %} - - - - {% endif %} -
- {% trans "ID" %} - - {% trans "User" as text %} - {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %} - - {% trans "Date Created" as text %} - {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %} - - {% trans "Date Updated" %} - - {% trans "Items" %} - - {% trans "Pending items" %} - - {% trans "Successful items" %} - - {% trans "Failed items" %} - {% trans "Actions" %}
{{ import.id }} - {{ import.user|username }} - {{ import.created_date }}{{ import.updated_date }}{{ import.item_count|intcomma }}{{ import.pending_item_count|intcomma }}{{ import.successful_item_count|intcomma }}{{ import.failed_item_count|intcomma }} - {% join "complete" import.id as modal_id %} - - {% include "settings/imports/complete_import_modal.html" with id=modal_id %} -
- {% trans "No matching imports found." %} -
-
+
+

{% trans "User Imports" %}

+
+
+ +
+
-{% include 'snippets/pagination.html' with page=imports path=request.path %} +
+ + + {% url 'settings-imports' status as url %} + + + + {% if status != "active" %} + + {% endif %} + + {% if status == "active" %} + + {% else %} + + {% endif %} + + {% for import in user_imports %} + + + + + {% if status != "active" %} + + {% endif %} + {% if status == "active" %} + + {% else %} + + {% endif %} + + {% endfor %} + {% if not user_imports %} + + + + {% endif %} +
+ {% trans "ID" %} + + {% trans "User" as text %} + {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %} + + {% trans "Date Created" as text %} + {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %} + + {% trans "Date Updated" %} + {% trans "Actions" %}{% trans "Status" %}
{{ import.id }} + {{ import.user|username }} + {{ import.created_date }}{{ import.updated_date }} + {% join "complete" import.id as modal_id %} + + {% include "settings/imports/complete_user_import_modal.html" with id=modal_id %} + + {{ import.status }} + +
+ {% trans "No matching imports found." %} +
+
+ + {% include 'snippets/pagination.html' with page=user_imports path=request.path %} +
{% endblock %} - diff --git a/bookwyrm/templatetags/utilities.py b/bookwyrm/templatetags/utilities.py index 76e5d988f..fca66688a 100644 --- a/bookwyrm/templatetags/utilities.py +++ b/bookwyrm/templatetags/utilities.py @@ -128,6 +128,23 @@ def id_to_username(user_id): return value +@register.filter(name="get_file_size") +def get_file_size(file): + """display the size of a file in human readable terms""" + + try: + raw_size = os.stat(file.path).st_size + if raw_size < 1024: + return f"{raw_size} bytes" + if raw_size < 1024**2: + return f"{raw_size/1024:.2f} KB" + if raw_size < 1024**3: + return f"{raw_size/1024**2:.2f} MB" + return f"{raw_size/1024**3:.2f} GB" + except Exception: # pylint: disable=broad-except + return "" + + @register.filter(name="get_user_permission") def get_user_permission(user): """given a user, return their permission level""" diff --git a/bookwyrm/tests/data/bookwyrm_account_export.tar.gz b/bookwyrm/tests/data/bookwyrm_account_export.tar.gz new file mode 100644 index 000000000..34cee6bc0 Binary files /dev/null and b/bookwyrm/tests/data/bookwyrm_account_export.tar.gz differ diff --git a/bookwyrm/tests/data/user_import.json b/bookwyrm/tests/data/user_import.json new file mode 100644 index 000000000..0318ddfeb --- /dev/null +++ b/bookwyrm/tests/data/user_import.json @@ -0,0 +1,399 @@ +{ + "id": "https://www.example.com/user/rat", + "type": "Person", + "preferredUsername": "rat", + "inbox": "https://www.example.com/user/rat/inbox", + "publicKey": { + "id": "https://www.example.com/user/rat/#main-key", + "owner": "https://www.example.com/user/rat", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nzzzz\n-----END PUBLIC KEY-----" + }, + "followers": "https://www.example.com/user/rat/followers", + "following": "https://www.example.com/user/rat/following", + "outbox": "https://www.example.com/user/rat/outbox", + "endpoints": { + "sharedInbox": "https://www.example.com/inbox" + }, + "name": "Rat", + "summary": "

I love to make soup in Paris and eat pizza in New York

", + "icon": { + "type": "Document", + "url": "avatar.png", + "name": "avatar for rat", + "@context": "https://www.w3.org/ns/activitystreams" + }, + "bookwyrmUser": true, + "manuallyApprovesFollowers": true, + "discoverable": false, + "hideFollows": true, + "alsoKnownAs": [], + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "schema": "http://schema.org#", + "PropertyValue": "schema:PropertyValue", + "value": "schema:value", + "alsoKnownAs": { + "@id": "as:alsoKnownAs", + "@type": "@id" + }, + "movedTo": { + "@id": "as:movedTo", + "@type": "@id" + } + } + ], + "settings": { + "show_goal": false, + "preferred_timezone": "Australia/Adelaide", + "default_post_privacy": "followers", + "show_suggested_users": false + }, + "goals": [ + { + "goal": 12, + "year": 2023, + "privacy": "followers" + } + ], + "books": [ + { + "work": { + "id": "https://www.example.com/book/1", + "type": "Work", + "title": "Seeing Like a State", + "description": "

Examines how (sometimes quasi-) authoritarian high-modernist planning fails to deliver the goods, be they increased resources for the state or a better life for the people.

", + "languages": [ "English" ], + "series": "", + "seriesNumber": "", + "subjects": [], + "subjectPlaces": [], + "authors": [ + "https://www.example.com/author/1" + ], + "firstPublishedDate": "", + "publishedDate": "1998-03-30T00:00:00Z", + "fileLinks": [], + "lccn": "", + "editions": [ + "https://www.example.com/book/2" + ], + "@context": "https://www.w3.org/ns/activitystreams" + }, + "edition": { + "id": "https://www.example.com/book/2", + "type": "Edition", + "openlibraryKey": "OL680025M", + "title": "Seeking Like A State", + "sortTitle": "seeing like a state", + "subtitle": "", + "description": "

Examines how (sometimes quasi-) authoritarian high-modernist planning fails to deliver the goods, be they increased resources for the state or a better life for the people.

", + "languages": ["English"], + "series": "", + "seriesNumber": "", + "subjects": [], + "subjectPlaces": [], + "authors": [ + "https://www.example.com/author/1" + ], + "firstPublishedDate": "", + "publishedDate": "", + "fileLinks": [], + "cover": { + "type": "Document", + "url": "covers/d273d638-191d-4ebf-b213-3c60dbf010fe.jpeg", + "name": "James C. Scott: Seeing like a state", + "@context": "https://www.w3.org/ns/activitystreams" + }, + "work": "https://www.example.com/book/1", + "isbn10": "", + "isbn13": "9780300070163", + "oclcNumber": "", + "physicalFormat": "", + "physicalFormatDetail": "", + "publishers": [], + "editionRank": 4, + "@context": "https://www.w3.org/ns/activitystreams" + }, + "authors": [ + { + "id": "https://www.example.com/author/1", + "type": "Author", + "name": "James C. Scott", + "aliases": [ + "James Campbell Scott", + "\u30b8\u30a7\u30fc\u30e0\u30ba\u30fbC. \u30b9\u30b3\u30c3\u30c8", + "\u30b8\u30a7\u30fc\u30e0\u30ba\u30fbC\u30fb\u30b9\u30b3\u30c3\u30c8", + "\u062c\u06cc\u0645\u0632 \u0633\u06cc. \u0627\u0633\u06a9\u0627\u062a", + "Jim Scott", + "\u062c\u064a\u0645\u0633 \u0633\u0643\u0648\u062a", + "James C. Scott", + "\u0414\u0436\u0435\u0439\u043c\u0441 \u0421\u043a\u043e\u0442\u0442", + "\u30b8\u30a7\u30fc\u30e0\u30b9\u30fbC \u30b9\u30b3\u30c3\u30c8", + "James Cameron Scott" + ], + "bio": "

American political scientist and anthropologist

", + "wikipediaLink": "https://en.wikipedia.org/wiki/James_C._Scott", + "website": "", + "@context": "https://www.w3.org/ns/activitystreams" + } + ], + "shelves": [ + { + "id": "https://www.example.com/user/rat/books/read", + "type": "Shelf", + "totalItems": 1, + "first": "https://www.example.com/user/rat/books/read?page=1", + "last": "https://www.example.com/user/rat/books/read?page=1", + "name": "Read", + "owner": "https://www.example.com/user/rat", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://www.example.com/user/rat/followers" + ], + "@context": "https://www.w3.org/ns/activitystreams" + }, + { + "id": "https://www.example.com/user/rat/books/to-read", + "type": "Shelf", + "totalItems": 1, + "first": "https://www.example.com/user/rat/books/to-read?page=1", + "last": "https://www.example.com/user/rat/books/to-read?page=1", + "name": "To Read", + "owner": "https://www.example.com/user/rat", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://www.example.com/user/rat/followers" + ], + "@context": "https://www.w3.org/ns/activitystreams" + } + ], + "lists": [ + { + "id": "https://www.example.com/list/2", + "type": "BookList", + "totalItems": 1, + "first": "https://www.example.com/list/2?page=1", + "last": "https://www.example.com/list/2?page=1", + "name": "my list of books", + "owner": "https://www.example.com/user/rat", + "to": [ + "https://www.example.com/user/rat/followers" + ], + "cc": [], + "summary": "Here is a description of my list", + "curation": "closed", + "@context": "https://www.w3.org/ns/activitystreams", + "privacy": "followers", + "list_item": { + "id": "https://www.example.com/user/rat/listitem/3", + "type": "ListItem", + "actor": "https://www.example.com/user/rat", + "book": "https://www.example.com/book/2", + "notes": "

It's fun.

", + "approved": true, + "order": 1, + "@context": "https://www.w3.org/ns/activitystreams" + } + } + ], + "comments": [], + "quotations": [], + "reviews": [ + { + "id": "https://www.example.com/user/rat/review/7", + "type": "Review", + "published": "2023-08-14T04:09:18.343+00:00", + "attributedTo": "https://www.example.com//user/rat", + "content": "

I like it

", + "to": [ + "https://your.domain.here/user/rat/followers" + ], + "cc": [], + "replies": { + "id": "https://www.example.com/user/rat/review/7/replies", + "type": "OrderedCollection", + "totalItems": 0, + "first": "https://www.example.com/user/rat/review/7/replies?page=1", + "last": "https://www.example.com/user/rat/review/7/replies?page=1", + "@context": "https://www.w3.org/ns/activitystreams" + }, + "summary": "Here's a spoiler alert", + "tag": [], + "attachment": [], + "sensitive": true, + "inReplyToBook": "https://www.example.com/book/6", + "name": "great book", + "rating": 5.0, + "@context": "https://www.w3.org/ns/activitystreams", + "progress": 23, + "progress_mode": "PG" + } + ], + "readthroughs": [ + { + "id": 1, + "created_date": "2023-08-14T04:00:27.544Z", + "updated_date": "2023-08-14T04:00:27.546Z", + "remote_id": "https://www.example.com/user/rat/readthrough/1", + "user_id": 1, + "book_id": 4880, + "progress": null, + "progress_mode": "PG", + "start_date": "2018-01-01T00:00:00Z", + "finish_date": "2023-08-13T00:00:00Z", + "stopped_date": null, + "is_active": false + } + ] + }, + { + "work": { + "id": "https://www.example.com/book/3", + "type": "Work", + "title": "Sand Talk: How Indigenous Thinking Can Save the World", + "description": "", + "languages": [], + "series": "", + "seriesNumber": "", + "subjects": [], + "subjectPlaces": [], + "authors": [ + "https://www.example.com/author/2" + ], + "firstPublishedDate": "", + "publishedDate": "", + "fileLinks": [], + "lccn": "", + "openlibraryKey": "OL28216445M", + "editions": [ + "https://www.example.com/book/4" + ], + "@context": "https://www.w3.org/ns/activitystreams" + }, + "edition": { + "id": "https://www.example.com/book/4", + "type": "Edition", + "title": "Sand Talk", + "sortTitle": "sand talk", + "subtitle": "How Indigenous Thinking Can Save the World", + "description": "", + "languages": [], + "series": "", + "seriesNumber": "", + "subjects": [], + "subjectPlaces": [], + "authors": [ + "https://www.example.com/author/2" + ], + "firstPublishedDate": "", + "publishedDate": "", + "fileLinks": [], + "cover": { + "type": "Document", + "url": "covers/6a553a08-2641-42a1-baa4-960df9edbbfc.jpeg", + "name": "Tyson Yunkaporta - Sand Talk", + "@context": "https://www.w3.org/ns/activitystreams" + }, + "work": "https://www.example.com/book/3", + "isbn10": "", + "isbn13": "9780062975645", + "oclcNumber": "", + "inventaireId": "isbn:9780062975645", + "physicalFormat": "paperback", + "physicalFormatDetail": "", + "publishers": [], + "editionRank": 5, + "@context": "https://www.w3.org/ns/activitystreams" + }, + "authors": [ + { + "id": "https://www.example.com/author/2", + "type": "Author", + "name": "Tyson Yunkaporta", + "aliases": [], + "bio": "", + "wikipediaLink": "", + "website": "", + "@context": "https://www.w3.org/ns/activitystreams" + } + ], + "shelves": [], + "lists": [], + "comments": [ + { + "id": "https://www.example.com/user/rat/comment/4", + "type": "Comment", + "published": "2023-08-14T04:48:18.746+00:00", + "attributedTo": "https://www.example.com/user/rat", + "content": "

this is a comment about an amazing book

", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://www.example.com/user/rat/followers" + ], + "replies": { + "id": "https://www.example.com/user/rat/comment/4/replies", + "type": "OrderedCollection", + "totalItems": 0, + "first": "https://www.example.com/user/rat/comment/4/replies?page=1", + "last": "https://www.example.com/user/rat/comment/4/replies?page=1", + "@context": "https://www.w3.org/ns/activitystreams" + }, + "tag": [], + "attachment": [], + "sensitive": false, + "inReplyToBook": "https://www.example.com/book/4", + "readingStatus": null, + "@context": "https://www.w3.org/ns/activitystreams" + } + ], + "quotations": [ + { + "id": "https://www.example.com/user/rat/quotation/2", + "type": "Quotation", + "published": "2023-11-12T04:29:38.370305+00:00", + "attributedTo": "https://www.example.com/user/rat", + "content": "

not actually from this book lol

", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://www.example.com/user/rat/followers" + ], + "replies": { + "id": "https://www.example.com/user/rat/quotation/2/replies", + "type": "OrderedCollection", + "totalItems": 0, + "first": "https://www.example.com/user/rat/quotation/2/replies?page=1", + "last": "https://www.example.com/user/rat/quotation/2/replies?page=1", + "@context": "https://www.w3.org/ns/activitystreams" + }, + "tag": [], + "attachment": [], + "sensitive": false, + "summary": "spoiler ahead!", + "inReplyToBook": "https://www.example.com/book/2", + "quote": "

To be or not to be

", + "@context": "https://www.w3.org/ns/activitystreams" + } + ], + "reviews": [], + "readthroughs": [] + } + ], + "saved_lists": [ + "https://local.lists/9999" + ], + "follows": [ + "https://your.domain.here/user/rat" + ], + "blocks": ["https://your.domain.here/user/badger"] +} \ No newline at end of file diff --git a/bookwyrm/tests/migrations/test_0184.py b/bookwyrm/tests/migrations/test_0184.py deleted file mode 100644 index 4bf1b66c9..000000000 --- a/bookwyrm/tests/migrations/test_0184.py +++ /dev/null @@ -1,121 +0,0 @@ -""" testing migrations """ -from unittest.mock import patch - -from django.test import TestCase -from django.db.migrations.executor import MigrationExecutor -from django.db import connection - -from bookwyrm import models -from bookwyrm.management.commands import initdb -from bookwyrm.settings import DOMAIN - -# pylint: disable=missing-class-docstring -# pylint: disable=missing-function-docstring -class EraseDeletedUserDataMigration(TestCase): - - migrate_from = "0183_auto_20231105_1607" - migrate_to = "0184_auto_20231106_0421" - - # pylint: disable=invalid-name - def setUp(self): - with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( - "bookwyrm.activitystreams.populate_stream_task.delay" - ), patch("bookwyrm.lists_stream.populate_lists_task.delay"): - self.active_user = models.User.objects.create_user( - f"activeuser@{DOMAIN}", - "activeuser@activeuser.activeuser", - "activeuserword", - local=True, - localname="active", - name="a name", - ) - self.inactive_user = models.User.objects.create_user( - f"inactiveuser@{DOMAIN}", - "inactiveuser@inactiveuser.inactiveuser", - "inactiveuserword", - local=True, - localname="inactive", - is_active=False, - deactivation_reason="self_deactivation", - name="name name", - ) - self.deleted_user = models.User.objects.create_user( - f"deleteduser@{DOMAIN}", - "deleteduser@deleteduser.deleteduser", - "deleteduserword", - local=True, - localname="deleted", - is_active=False, - deactivation_reason="self_deletion", - name="cool name", - ) - with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" - ), patch("bookwyrm.activitystreams.add_status_task.delay"): - self.active_status = models.Status.objects.create( - user=self.active_user, content="don't delete me" - ) - self.inactive_status = models.Status.objects.create( - user=self.inactive_user, content="also don't delete me" - ) - self.deleted_status = models.Status.objects.create( - user=self.deleted_user, content="yes, delete me" - ) - - initdb.init_groups() - initdb.init_permissions() - - self.migrate_from = [("bookwyrm", self.migrate_from)] - self.migrate_to = [("bookwyrm", self.migrate_to)] - executor = MigrationExecutor(connection) - old_apps = executor.loader.project_state(self.migrate_from).apps - - # Reverse to the original migration - executor.migrate(self.migrate_from) - - self.setUpBeforeMigration(old_apps) - - # Run the migration to test - executor = MigrationExecutor(connection) - executor.loader.build_graph() # reload. - with patch("bookwyrm.activitystreams.remove_status_task.delay"): - executor.migrate(self.migrate_to) - - self.apps = executor.loader.project_state(self.migrate_to).apps - - def setUpBeforeMigration(self, apps): - pass - - def test_user_data_deleted(self): - """Make sure that only the right data was deleted""" - self.active_user.refresh_from_db() - self.inactive_user.refresh_from_db() - self.deleted_user.refresh_from_db() - self.active_status.refresh_from_db() - self.inactive_status.refresh_from_db() - self.deleted_status.refresh_from_db() - - self.assertTrue(self.active_user.is_active) - self.assertFalse(self.active_user.is_deleted) - self.assertEqual(self.active_user.name, "a name") - self.assertNotEqual(self.deleted_user.email, "activeuser@activeuser.activeuser") - self.assertFalse(self.active_status.deleted) - self.assertEqual(self.active_status.content, "don't delete me") - - self.assertFalse(self.inactive_user.is_active) - self.assertFalse(self.inactive_user.is_deleted) - self.assertEqual(self.inactive_user.name, "name name") - self.assertNotEqual( - self.deleted_user.email, "inactiveuser@inactiveuser.inactiveuser" - ) - self.assertFalse(self.inactive_status.deleted) - self.assertEqual(self.inactive_status.content, "also don't delete me") - - self.assertFalse(self.deleted_user.is_active) - self.assertTrue(self.deleted_user.is_deleted) - self.assertIsNone(self.deleted_user.name) - self.assertNotEqual( - self.deleted_user.email, "deleteduser@deleteduser.deleteduser" - ) - self.assertTrue(self.deleted_status.deleted) - self.assertIsNone(self.deleted_status.content) diff --git a/bookwyrm/tests/models/test_bookwyrm_export_job.py b/bookwyrm/tests/models/test_bookwyrm_export_job.py new file mode 100644 index 000000000..b5f2520a9 --- /dev/null +++ b/bookwyrm/tests/models/test_bookwyrm_export_job.py @@ -0,0 +1,231 @@ +"""test bookwyrm user export functions""" +import datetime +import json +from unittest.mock import patch + +from django.core.serializers.json import DjangoJSONEncoder +from django.test import TestCase +from django.utils import timezone + +from bookwyrm import models +import bookwyrm.models.bookwyrm_export_job as export_job + + +class BookwyrmExport(TestCase): + """testing user export functions""" + + def setUp(self): + """lots of stuff to set up for a user export""" + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ), patch("bookwyrm.lists_stream.populate_lists_task.delay"), patch( + "bookwyrm.suggested_users.rerank_user_task.delay" + ), patch( + "bookwyrm.lists_stream.remove_list_task.delay" + ), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch( + "bookwyrm.activitystreams.add_book_statuses_task" + ): + + self.local_user = models.User.objects.create_user( + "mouse", + "mouse@mouse.mouse", + "password", + local=True, + localname="mouse", + name="Mouse", + summary="I'm a real bookmouse", + manually_approves_followers=False, + hide_follows=False, + show_goal=False, + show_suggested_users=False, + discoverable=True, + preferred_timezone="America/Los Angeles", + default_post_privacy="followers", + ) + + self.rat_user = models.User.objects.create_user( + "rat", "rat@rat.rat", "ratword", local=True, localname="rat" + ) + + self.badger_user = models.User.objects.create_user( + "badger", + "badger@badger.badger", + "badgerword", + local=True, + localname="badger", + ) + + models.AnnualGoal.objects.create( + user=self.local_user, + year=timezone.now().year, + goal=128937123, + privacy="followers", + ) + + self.list = models.List.objects.create( + name="My excellent list", + user=self.local_user, + remote_id="https://local.lists/1111", + ) + + self.saved_list = models.List.objects.create( + name="My cool list", + user=self.rat_user, + remote_id="https://local.lists/9999", + ) + + self.local_user.saved_lists.add(self.saved_list) + self.local_user.blocks.add(self.badger_user) + self.rat_user.followers.add(self.local_user) + + # book, edition, author + self.author = models.Author.objects.create(name="Sam Zhu") + self.work = models.Work.objects.create( + title="Example Work", remote_id="https://example.com/book/1" + ) + self.edition = models.Edition.objects.create( + title="Example Edition", parent_work=self.work + ) + + self.edition.authors.add(self.author) + + # readthrough + self.readthrough_start = timezone.now() + finish = self.readthrough_start + datetime.timedelta(days=1) + models.ReadThrough.objects.create( + user=self.local_user, + book=self.edition, + start_date=self.readthrough_start, + finish_date=finish, + ) + + # shelve + read_shelf = models.Shelf.objects.get( + user=self.local_user, identifier="read" + ) + models.ShelfBook.objects.create( + book=self.edition, shelf=read_shelf, user=self.local_user + ) + + # add to list + models.ListItem.objects.create( + book_list=self.list, + user=self.local_user, + book=self.edition, + approved=True, + order=1, + ) + + # review + models.Review.objects.create( + content="awesome", + name="my review", + rating=5, + user=self.local_user, + book=self.edition, + ) + # comment + models.Comment.objects.create( + content="ok so far", + user=self.local_user, + book=self.edition, + progress=15, + ) + # quote + models.Quotation.objects.create( + content="check this out", + quote="A rose by any other name", + user=self.local_user, + book=self.edition, + ) + + def test_json_export_user_settings(self): + """Test the json export function for basic user info""" + data = export_job.json_export(self.local_user) + user_data = json.loads(data) + self.assertEqual(user_data["preferredUsername"], "mouse") + self.assertEqual(user_data["name"], "Mouse") + self.assertEqual(user_data["summary"], "

I'm a real bookmouse

") + self.assertEqual(user_data["manuallyApprovesFollowers"], False) + self.assertEqual(user_data["hideFollows"], False) + self.assertEqual(user_data["discoverable"], True) + self.assertEqual(user_data["settings"]["show_goal"], False) + self.assertEqual(user_data["settings"]["show_suggested_users"], False) + self.assertEqual( + user_data["settings"]["preferred_timezone"], "America/Los Angeles" + ) + self.assertEqual(user_data["settings"]["default_post_privacy"], "followers") + + def test_json_export_extended_user_data(self): + """Test the json export function for other non-book user info""" + data = export_job.json_export(self.local_user) + json_data = json.loads(data) + + # goal + self.assertEqual(len(json_data["goals"]), 1) + self.assertEqual(json_data["goals"][0]["goal"], 128937123) + self.assertEqual(json_data["goals"][0]["year"], timezone.now().year) + self.assertEqual(json_data["goals"][0]["privacy"], "followers") + + # saved lists + self.assertEqual(len(json_data["saved_lists"]), 1) + self.assertEqual(json_data["saved_lists"][0], "https://local.lists/9999") + + # follows + self.assertEqual(len(json_data["follows"]), 1) + self.assertEqual(json_data["follows"][0], "https://your.domain.here/user/rat") + # blocked users + self.assertEqual(len(json_data["blocks"]), 1) + self.assertEqual(json_data["blocks"][0], "https://your.domain.here/user/badger") + + def test_json_export_books(self): + """Test the json export function for extended user info""" + + data = export_job.json_export(self.local_user) + json_data = json.loads(data) + start_date = json_data["books"][0]["readthroughs"][0]["start_date"] + + self.assertEqual(len(json_data["books"]), 1) + self.assertEqual(json_data["books"][0]["edition"]["title"], "Example Edition") + self.assertEqual(len(json_data["books"][0]["authors"]), 1) + self.assertEqual(json_data["books"][0]["authors"][0]["name"], "Sam Zhu") + + self.assertEqual( + f'"{start_date}"', DjangoJSONEncoder().encode(self.readthrough_start) + ) + + self.assertEqual(json_data["books"][0]["shelves"][0]["name"], "Read") + + self.assertEqual(len(json_data["books"][0]["lists"]), 1) + self.assertEqual(json_data["books"][0]["lists"][0]["name"], "My excellent list") + self.assertEqual( + json_data["books"][0]["lists"][0]["list_item"]["book"], + self.edition.remote_id, + self.edition.id, + ) + + self.assertEqual(len(json_data["books"][0]["reviews"]), 1) + self.assertEqual(len(json_data["books"][0]["comments"]), 1) + self.assertEqual(len(json_data["books"][0]["quotations"]), 1) + + self.assertEqual(json_data["books"][0]["reviews"][0]["name"], "my review") + self.assertEqual( + json_data["books"][0]["reviews"][0]["content"], "

awesome

" + ) + self.assertEqual(json_data["books"][0]["reviews"][0]["rating"], 5.0) + + self.assertEqual( + json_data["books"][0]["comments"][0]["content"], "

ok so far

" + ) + self.assertEqual(json_data["books"][0]["comments"][0]["progress"], 15) + self.assertEqual(json_data["books"][0]["comments"][0]["progress_mode"], "PG") + + self.assertEqual( + json_data["books"][0]["quotations"][0]["content"], "

check this out

" + ) + self.assertEqual( + json_data["books"][0]["quotations"][0]["quote"], + "

A rose by any other name

", + ) diff --git a/bookwyrm/tests/models/test_bookwyrm_import_job.py b/bookwyrm/tests/models/test_bookwyrm_import_job.py new file mode 100644 index 000000000..adc04706c --- /dev/null +++ b/bookwyrm/tests/models/test_bookwyrm_import_job.py @@ -0,0 +1,545 @@ +""" testing models """ + +import json +import pathlib +from unittest.mock import patch + +from django.db.models import Q +from django.utils.dateparse import parse_datetime +from django.test import TestCase + +from bookwyrm import models +from bookwyrm.utils.tar import BookwyrmTarFile +from bookwyrm.models import bookwyrm_import_job + + +class BookwyrmImport(TestCase): # pylint: disable=too-many-public-methods + """testing user import functions""" + + def setUp(self): + """setting stuff up""" + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ), patch("bookwyrm.lists_stream.populate_lists_task.delay"), patch( + "bookwyrm.suggested_users.rerank_user_task.delay" + ): + + self.local_user = models.User.objects.create_user( + "mouse", + "mouse@mouse.mouse", + "password", + local=True, + localname="mouse", + name="Mouse", + summary="I'm a real bookmouse", + manually_approves_followers=False, + hide_follows=False, + show_goal=True, + show_suggested_users=True, + discoverable=True, + preferred_timezone="America/Los Angeles", + default_post_privacy="public", + ) + + self.rat_user = models.User.objects.create_user( + "rat", "rat@rat.rat", "password", local=True, localname="rat" + ) + + self.badger_user = models.User.objects.create_user( + "badger", + "badger@badger.badger", + "password", + local=True, + localname="badger", + ) + + self.work = models.Work.objects.create(title="Sand Talk") + + self.book = models.Edition.objects.create( + title="Sand Talk", + remote_id="https://example.com/book/1234", + openlibrary_key="OL28216445M", + inventaire_id="isbn:9780062975645", + isbn_13="9780062975645", + parent_work=self.work, + ) + + self.json_file = pathlib.Path(__file__).parent.joinpath( + "../data/user_import.json" + ) + + with open(self.json_file, "r", encoding="utf-8") as jsonfile: + self.json_data = json.loads(jsonfile.read()) + + self.archive_file = pathlib.Path(__file__).parent.joinpath( + "../data/bookwyrm_account_export.tar.gz" + ) + + def test_update_user_profile(self): + """Test update the user's profile from import data""" + + with patch("bookwyrm.suggested_users.remove_user_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch("bookwyrm.suggested_users.rerank_user_task.delay"): + + with open(self.archive_file, "rb") as fileobj: + with BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile: + + models.bookwyrm_import_job.update_user_profile( + self.local_user, tarfile, self.json_data + ) + + self.local_user.refresh_from_db() + + self.assertEqual( + self.local_user.username, "mouse" + ) # username should not change + self.assertEqual(self.local_user.name, "Rat") + self.assertEqual( + self.local_user.summary, + "I love to make soup in Paris and eat pizza in New York", + ) + + def test_update_user_settings(self): + """Test updating the user's settings from import data""" + + with patch("bookwyrm.suggested_users.remove_user_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch("bookwyrm.suggested_users.rerank_user_task.delay"): + + models.bookwyrm_import_job.update_user_settings( + self.local_user, self.json_data + ) + self.local_user.refresh_from_db() + + self.assertEqual(self.local_user.manually_approves_followers, True) + self.assertEqual(self.local_user.hide_follows, True) + self.assertEqual(self.local_user.show_goal, False) + self.assertEqual(self.local_user.show_suggested_users, False) + self.assertEqual(self.local_user.discoverable, False) + self.assertEqual(self.local_user.preferred_timezone, "Australia/Adelaide") + self.assertEqual(self.local_user.default_post_privacy, "followers") + + def test_update_goals(self): + """Test update the user's goals from import data""" + + models.AnnualGoal.objects.create( + user=self.local_user, + year=2023, + goal=999, + privacy="public", + ) + + goals = [{"goal": 12, "year": 2023, "privacy": "followers"}] + + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + + models.bookwyrm_import_job.update_goals(self.local_user, goals) + + self.local_user.refresh_from_db() + goal = models.AnnualGoal.objects.get() + self.assertEqual(goal.year, 2023) + self.assertEqual(goal.goal, 12) + self.assertEqual(goal.privacy, "followers") + + def test_upsert_saved_lists_existing(self): + """Test upserting an existing saved list""" + + with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + book_list = models.List.objects.create( + name="My cool list", + user=self.rat_user, + remote_id="https://local.lists/9999", + ) + + self.assertFalse(self.local_user.saved_lists.filter(id=book_list.id).exists()) + + self.local_user.saved_lists.add(book_list) + + self.assertTrue(self.local_user.saved_lists.filter(id=book_list.id).exists()) + + with patch("bookwyrm.activitypub.base_activity.resolve_remote_id"): + models.bookwyrm_import_job.upsert_saved_lists( + self.local_user, ["https://local.lists/9999"] + ) + saved_lists = self.local_user.saved_lists.filter( + remote_id="https://local.lists/9999" + ).all() + self.assertEqual(len(saved_lists), 1) + + def test_upsert_saved_lists_not_existing(self): + """Test upserting a new saved list""" + + with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + book_list = models.List.objects.create( + name="My cool list", + user=self.rat_user, + remote_id="https://local.lists/9999", + ) + + self.assertFalse(self.local_user.saved_lists.filter(id=book_list.id).exists()) + + with patch("bookwyrm.activitypub.base_activity.resolve_remote_id"): + models.bookwyrm_import_job.upsert_saved_lists( + self.local_user, ["https://local.lists/9999"] + ) + + self.assertTrue(self.local_user.saved_lists.filter(id=book_list.id).exists()) + + def test_upsert_follows(self): + """Test take a list of remote ids and add as follows""" + + before_follow = models.UserFollows.objects.filter( + user_subject=self.local_user, user_object=self.rat_user + ).exists() + + self.assertFalse(before_follow) + + with patch("bookwyrm.activitystreams.add_user_statuses_task.delay"), patch( + "bookwyrm.lists_stream.add_user_lists_task.delay" + ), patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + models.bookwyrm_import_job.upsert_follows( + self.local_user, self.json_data.get("follows") + ) + + after_follow = models.UserFollows.objects.filter( + user_subject=self.local_user, user_object=self.rat_user + ).exists() + self.assertTrue(after_follow) + + def test_upsert_user_blocks(self): + """test adding blocked users""" + + blocked_before = models.UserBlocks.objects.filter( + Q( + user_subject=self.local_user, + user_object=self.badger_user, + ) + ).exists() + self.assertFalse(blocked_before) + + with patch("bookwyrm.suggested_users.remove_suggestion_task.delay"), patch( + "bookwyrm.activitystreams.remove_user_statuses_task.delay" + ), patch("bookwyrm.lists_stream.remove_user_lists_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + models.bookwyrm_import_job.upsert_user_blocks( + self.local_user, self.json_data.get("blocks") + ) + + blocked_after = models.UserBlocks.objects.filter( + Q( + user_subject=self.local_user, + user_object=self.badger_user, + ) + ).exists() + self.assertTrue(blocked_after) + + def test_get_or_create_edition_existing(self): + """Test take a JSON string of books and editions, + find or create the editions in the database and + return a list of edition instances""" + + self.assertEqual(models.Edition.objects.count(), 1) + + with open(self.archive_file, "rb") as fileobj: + with BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile: + + bookwyrm_import_job.get_or_create_edition( + self.json_data["books"][1], tarfile + ) # Sand Talk + + self.assertEqual(models.Edition.objects.count(), 1) + + def test_get_or_create_edition_not_existing(self): + """Test take a JSON string of books and editions, + find or create the editions in the database and + return a list of edition instances""" + + self.assertEqual(models.Edition.objects.count(), 1) + + with open(self.archive_file, "rb") as fileobj: + with BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile: + + bookwyrm_import_job.get_or_create_edition( + self.json_data["books"][0], tarfile + ) # Seeing like a state + + self.assertTrue(models.Edition.objects.filter(isbn_13="9780300070163").exists()) + self.assertEqual(models.Edition.objects.count(), 2) + + def test_upsert_readthroughs(self): + """Test take a JSON string of readthroughs, find or create the + instances in the database and return a list of saved instances""" + + readthroughs = [ + { + "id": 1, + "created_date": "2023-08-24T10:18:45.923Z", + "updated_date": "2023-08-24T10:18:45.928Z", + "remote_id": "https://example.com/mouse/readthrough/1", + "user_id": 1, + "book_id": 1234, + "progress": 23, + "progress_mode": "PG", + "start_date": "2022-12-31T13:30:00Z", + "finish_date": "2023-08-23T14:30:00Z", + "stopped_date": None, + "is_active": False, + } + ] + + self.assertEqual(models.ReadThrough.objects.count(), 0) + bookwyrm_import_job.upsert_readthroughs( + readthroughs, self.local_user, self.book.id + ) + + self.assertEqual(models.ReadThrough.objects.count(), 1) + self.assertEqual(models.ReadThrough.objects.first().progress_mode, "PG") + self.assertEqual( + models.ReadThrough.objects.first().start_date, + parse_datetime("2022-12-31T13:30:00Z"), + ) + self.assertEqual(models.ReadThrough.objects.first().book_id, self.book.id) + self.assertEqual(models.ReadThrough.objects.first().user, self.local_user) + + def test_get_or_create_review(self): + """Test get_or_create_review_status with a review""" + + self.assertEqual(models.Review.objects.filter(user=self.local_user).count(), 0) + reviews = self.json_data["books"][0]["reviews"] + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True): + + bookwyrm_import_job.upsert_statuses( + self.local_user, models.Review, reviews, self.book.remote_id + ) + + self.assertEqual(models.Review.objects.filter(user=self.local_user).count(), 1) + self.assertEqual( + models.Review.objects.filter(book=self.book).first().content, + "

I like it

", + ) + self.assertEqual( + models.Review.objects.filter(book=self.book).first().content_warning, + "Here's a spoiler alert", + ) + self.assertEqual( + models.Review.objects.filter(book=self.book).first().sensitive, True + ) + self.assertEqual( + models.Review.objects.filter(book=self.book).first().name, "great book" + ) + self.assertAlmostEqual( + models.Review.objects.filter(book=self.book).first().rating, 5.00 + ) + + self.assertEqual( + models.Review.objects.filter(book=self.book).first().privacy, "followers" + ) + + def test_get_or_create_comment(self): + """Test get_or_create_review_status with a comment""" + + self.assertEqual(models.Comment.objects.filter(user=self.local_user).count(), 0) + comments = self.json_data["books"][1]["comments"] + + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True): + + bookwyrm_import_job.upsert_statuses( + self.local_user, models.Comment, comments, self.book.remote_id + ) + self.assertEqual(models.Comment.objects.filter(user=self.local_user).count(), 1) + self.assertEqual( + models.Comment.objects.filter(book=self.book).first().content, + "

this is a comment about an amazing book

", + ) + self.assertEqual( + models.Comment.objects.filter(book=self.book).first().content_warning, None + ) + self.assertEqual( + models.Comment.objects.filter(book=self.book).first().sensitive, False + ) + self.assertEqual( + models.Comment.objects.filter(book=self.book).first().progress_mode, "PG" + ) + + def test_get_or_create_quote(self): + """Test get_or_create_review_status with a quote""" + + self.assertEqual( + models.Quotation.objects.filter(user=self.local_user).count(), 0 + ) + quotes = self.json_data["books"][1]["quotations"] + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True): + + bookwyrm_import_job.upsert_statuses( + self.local_user, models.Quotation, quotes, self.book.remote_id + ) + self.assertEqual( + models.Quotation.objects.filter(user=self.local_user).count(), 1 + ) + self.assertEqual( + models.Quotation.objects.filter(book=self.book).first().content, + "

not actually from this book lol

", + ) + self.assertEqual( + models.Quotation.objects.filter(book=self.book).first().content_warning, + "spoiler ahead!", + ) + self.assertEqual( + models.Quotation.objects.filter(book=self.book).first().quote, + "

To be or not to be

", + ) + self.assertEqual( + models.Quotation.objects.filter(book=self.book).first().position_mode, "PG" + ) + + def test_get_or_create_quote_unauthorized(self): + """Test get_or_create_review_status with a quote but not authorized""" + + self.assertEqual( + models.Quotation.objects.filter(user=self.local_user).count(), 0 + ) + quotes = self.json_data["books"][1]["quotations"] + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=False): + + bookwyrm_import_job.upsert_statuses( + self.local_user, models.Quotation, quotes, self.book.remote_id + ) + self.assertEqual( + models.Quotation.objects.filter(user=self.local_user).count(), 0 + ) + + def test_upsert_list_existing(self): + """Take a list and ListItems as JSON and create DB entries + if they don't already exist""" + + book_data = self.json_data["books"][0] + + other_book = models.Edition.objects.create( + title="Another Book", remote_id="https://example.com/book/9876" + ) + + with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + book_list = models.List.objects.create( + name="my list of books", user=self.local_user + ) + + models.ListItem.objects.create( + book=self.book, book_list=book_list, user=self.local_user, order=1 + ) + + self.assertTrue(models.List.objects.filter(id=book_list.id).exists()) + self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 1) + self.assertEqual( + models.ListItem.objects.filter( + user=self.local_user, book_list=book_list + ).count(), + 1, + ) + + with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + bookwyrm_import_job.upsert_lists( + self.local_user, + book_data["lists"], + other_book.id, + ) + + self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 1) + self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 1) + self.assertEqual( + models.ListItem.objects.filter( + user=self.local_user, book_list=book_list + ).count(), + 2, + ) + + def test_upsert_list_not_existing(self): + """Take a list and ListItems as JSON and create DB entries + if they don't already exist""" + + book_data = self.json_data["books"][0] + + self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 0) + self.assertFalse(models.ListItem.objects.filter(book=self.book.id).exists()) + + with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + bookwyrm_import_job.upsert_lists( + self.local_user, + book_data["lists"], + self.book.id, + ) + + self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 1) + self.assertEqual( + models.ListItem.objects.filter(user=self.local_user).count(), 1 + ) + + def test_upsert_shelves_existing(self): + """Take shelf and ShelfBooks JSON objects and create + DB entries if they don't already exist""" + + self.assertEqual( + models.ShelfBook.objects.filter(user=self.local_user.id).count(), 0 + ) + + shelf = models.Shelf.objects.get(name="Read", user=self.local_user) + + with patch("bookwyrm.activitystreams.add_book_statuses_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + models.ShelfBook.objects.create( + book=self.book, shelf=shelf, user=self.local_user + ) + + book_data = self.json_data["books"][0] + with patch("bookwyrm.activitystreams.add_book_statuses_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + bookwyrm_import_job.upsert_shelves(self.book, self.local_user, book_data) + + self.assertEqual( + models.ShelfBook.objects.filter(user=self.local_user.id).count(), 2 + ) + + def test_upsert_shelves_not_existing(self): + """Take shelf and ShelfBooks JSON objects and create + DB entries if they don't already exist""" + + self.assertEqual( + models.ShelfBook.objects.filter(user=self.local_user.id).count(), 0 + ) + + book_data = self.json_data["books"][0] + + with patch("bookwyrm.activitystreams.add_book_statuses_task.delay"), patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ): + bookwyrm_import_job.upsert_shelves(self.book, self.local_user, book_data) + + self.assertEqual( + models.ShelfBook.objects.filter(user=self.local_user.id).count(), 2 + ) + + # check we didn't create an extra shelf + self.assertEqual( + models.Shelf.objects.filter(user=self.local_user.id).count(), 4 + ) diff --git a/bookwyrm/tests/utils/test_tar.py b/bookwyrm/tests/utils/test_tar.py new file mode 100644 index 000000000..be5257542 --- /dev/null +++ b/bookwyrm/tests/utils/test_tar.py @@ -0,0 +1,25 @@ +import os +import pytest +from bookwyrm.utils.tar import BookwyrmTarFile + + +@pytest.fixture +def read_tar(): + archive_path = "../data/bookwyrm_account_export.tar.gz" + with open(archive_path, "rb") as archive_file: + with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: + yield tar + + +@pytest.fixture +def write_tar(): + archive_path = "/tmp/test.tar.gz" + with open(archive_path, "wb") as archive_file: + with BookwyrmTarFile.open(mode="w:gz", fileobj=archive_file) as tar: + yield tar + + os.remove(archive_path) + + +def test_write_bytes(write_tar): + write_tar.write_bytes(b"ABCDEF") diff --git a/bookwyrm/tests/views/imports/test_user_import.py b/bookwyrm/tests/views/imports/test_user_import.py new file mode 100644 index 000000000..db5837101 --- /dev/null +++ b/bookwyrm/tests/views/imports/test_user_import.py @@ -0,0 +1,68 @@ +""" test for app action functionality """ +import pathlib +from unittest.mock import patch + +from django.core.files.uploadedfile import SimpleUploadedFile +from django.template.response import TemplateResponse +from django.test import TestCase +from django.test.client import RequestFactory + +from bookwyrm import forms, models, views +from bookwyrm.tests.validate_html import validate_html + + +class ImportUserViews(TestCase): + """user import views""" + + # pylint: disable=invalid-name + def setUp(self): + """we need basic test data and mocks""" + self.factory = RequestFactory() + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ), patch("bookwyrm.lists_stream.populate_lists_task.delay"): + self.local_user = models.User.objects.create_user( + "mouse@local.com", + "mouse@mouse.mouse", + "password", + local=True, + localname="mouse", + ) + models.SiteSettings.objects.create() + + def test_get_user_import_page(self): + """there are so many views, this just makes sure it LOADS""" + view = views.UserImport.as_view() + request = self.factory.get("") + request.user = self.local_user + result = view(request) + self.assertIsInstance(result, TemplateResponse) + validate_html(result.render()) + self.assertEqual(result.status_code, 200) + + def test_user_import_post(self): + """does the import job start?""" + + view = views.UserImport.as_view() + form = forms.ImportUserForm() + archive_file = pathlib.Path(__file__).parent.joinpath( + "../../data/bookwyrm_account_export.tar.gz" + ) + + form.data["archive_file"] = SimpleUploadedFile( + # pylint: disable=consider-using-with + archive_file, + open(archive_file, "rb").read(), + content_type="application/gzip", + ) + + form.data["include_user_settings"] = "" + form.data["include_goals"] = "on" + + request = self.factory.post("", form.data) + request.user = self.local_user + + with patch("bookwyrm.models.bookwyrm_import_job.BookwyrmImportJob.start_job"): + view(request) + job = models.BookwyrmImportJob.objects.get() + self.assertEqual(job.required, ["include_goals"]) diff --git a/bookwyrm/tests/views/preferences/test_export_user.py b/bookwyrm/tests/views/preferences/test_export_user.py new file mode 100644 index 000000000..654ed2a05 --- /dev/null +++ b/bookwyrm/tests/views/preferences/test_export_user.py @@ -0,0 +1,50 @@ +""" test for user export app functionality """ +from unittest.mock import patch + +from django.http import HttpResponse +from django.test import TestCase +from django.test.client import RequestFactory + +from bookwyrm import models, views +from bookwyrm.tests.validate_html import validate_html + + +class ExportUserViews(TestCase): + """exporting user data""" + + def setUp(self): + self.factory = RequestFactory() + models.SiteSettings.objects.create() + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ): + self.local_user = models.User.objects.create_user( + "hugh@example.com", + "hugh@example.com", + "password", + local=True, + localname="Hugh", + summary="just a test account", + remote_id="https://example.com/users/hugh", + preferred_timezone="Australia/Broken_Hill", + ) + + def test_export_user_get(self, *_): + """request export""" + request = self.factory.get("") + request.user = self.local_user + result = views.ExportUser.as_view()(request) + validate_html(result.render()) + + def test_trigger_export_user_file(self, *_): + """simple user export""" + + request = self.factory.post("") + request.user = self.local_user + with patch("bookwyrm.models.bookwyrm_export_job.start_export_task.delay"): + export = views.ExportUser.as_view()(request) + self.assertIsInstance(export, HttpResponse) + self.assertEqual(export.status_code, 302) + + jobs = models.bookwyrm_export_job.BookwyrmExportJob.objects.count() + self.assertEqual(jobs, 1) diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index a059436ff..76e60245b 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -328,6 +328,11 @@ urlpatterns = [ views.ImportList.as_view(), name="settings-imports-complete", ), + re_path( + r"^settings/user-imports/(?P\d+)/complete/?$", + views.set_user_import_completed, + name="settings-user-import-complete", + ), re_path( r"^settings/imports/disable/?$", views.disable_imports, @@ -343,6 +348,11 @@ urlpatterns = [ views.set_import_size_limit, name="settings-imports-set-limit", ), + re_path( + r"^settings/user-imports/set-limit/?$", + views.set_user_import_limit, + name="settings-user-imports-set-limit", + ), re_path( r"^settings/celery/?$", views.CeleryStatus.as_view(), name="settings-celery" ), @@ -408,6 +418,7 @@ urlpatterns = [ re_path(r"^search/?$", views.Search.as_view(), name="search"), # imports re_path(r"^import/?$", views.Import.as_view(), name="import"), + re_path(r"^user-import/?$", views.UserImport.as_view(), name="user-import"), re_path( r"^import/(?P\d+)/?$", views.ImportStatus.as_view(), @@ -605,6 +616,16 @@ urlpatterns = [ name="prompt-2fa", ), re_path(r"^preferences/export/?$", views.Export.as_view(), name="prefs-export"), + re_path( + r"^preferences/user-export/?$", + views.ExportUser.as_view(), + name="prefs-user-export", + ), + path( + "preferences/user-export/", + views.ExportArchive.as_view(), + name="prefs-export-file", + ), re_path(r"^preferences/move/?$", views.MoveUser.as_view(), name="prefs-move"), re_path(r"^preferences/alias/?$", views.AliasUser.as_view(), name="prefs-alias"), re_path( diff --git a/bookwyrm/utils/tar.py b/bookwyrm/utils/tar.py new file mode 100644 index 000000000..bae3f7628 --- /dev/null +++ b/bookwyrm/utils/tar.py @@ -0,0 +1,49 @@ +"""manage tar files for user exports""" +import io +import tarfile +from typing import Any, Optional +from uuid import uuid4 +from django.core.files import File + + +class BookwyrmTarFile(tarfile.TarFile): + """Create tar files for user exports""" + + def write_bytes(self, data: bytes) -> None: + """Add a file containing bytes to the archive""" + buffer = io.BytesIO(data) + info = tarfile.TarInfo("archive.json") + info.size = len(data) + self.addfile(info, fileobj=buffer) + + def add_image( + self, image: Any, filename: Optional[str] = None, directory: Any = "" + ) -> None: + """ + Add an image to the tar archive + :param str filename: overrides the file name set by image + :param str directory: the directory in the archive to put the image + """ + if filename is not None: + file_type = image.name.rsplit(".", maxsplit=1)[-1] + filename = f"{directory}{filename}.{file_type}" + else: + filename = f"{directory}{image.name}" + + info = tarfile.TarInfo(name=filename) + info.size = image.size + + self.addfile(info, fileobj=image) + + def read(self, filename: str) -> Any: + """read data from the tar""" + if reader := self.extractfile(filename): + return reader.read() + return None + + def write_image_to_file(self, filename: str, file_field: Any) -> None: + """add an image to the tar""" + extension = filename.rsplit(".")[-1] + if buf := self.extractfile(filename): + filename = f"{str(uuid4())}.{extension}" + file_field.save(filename, File(buf)) diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index 64060a5c2..3be813208 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -16,6 +16,8 @@ from .admin.imports import ( disable_imports, enable_imports, set_import_size_limit, + set_user_import_completed, + set_user_import_limit, ) from .admin.ip_blocklist import IPBlocklist from .admin.invite import ManageInvites, Invite, InviteRequest @@ -36,7 +38,7 @@ from .admin.user_admin import UserAdmin, UserAdminList, ActivateUserAdmin # user preferences from .preferences.change_password import ChangePassword from .preferences.edit_user import EditUser -from .preferences.export import Export +from .preferences.export import Export, ExportUser, ExportArchive from .preferences.move_user import MoveUser, AliasUser, remove_alias, unmove from .preferences.delete_user import DeleteUser, DeactivateUser, ReactivateUser from .preferences.block import Block, unblock @@ -81,7 +83,7 @@ from .shelf.shelf_actions import create_shelf, delete_shelf from .shelf.shelf_actions import shelve, unshelve # csv import -from .imports.import_data import Import +from .imports.import_data import Import, UserImport from .imports.import_status import ImportStatus, retry_item, stop_import from .imports.troubleshoot import ImportTroubleshoot from .imports.manually_review import ( diff --git a/bookwyrm/views/admin/imports.py b/bookwyrm/views/admin/imports.py index 7ae190ce8..a85d6c79e 100644 --- a/bookwyrm/views/admin/imports.py +++ b/bookwyrm/views/admin/imports.py @@ -40,9 +40,17 @@ class ImportList(View): paginated = Paginator(imports, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) + user_imports = models.BookwyrmImportJob.objects.filter( + complete=complete + ).order_by("created_date") + + user_paginated = Paginator(user_imports, PAGE_LENGTH) + user_page = user_paginated.get_page(request.GET.get("page")) + site_settings = models.SiteSettings.objects.get() data = { "imports": page, + "user_imports": user_page, "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), @@ -50,6 +58,7 @@ class ImportList(View): "sort": sort, "import_size_limit": site_settings.import_size_limit, "import_limit_reset": site_settings.import_limit_reset, + "user_import_time_limit": site_settings.user_import_time_limit, } return TemplateResponse(request, "settings/imports/imports.html", data) @@ -95,3 +104,25 @@ def set_import_size_limit(request): site.import_limit_reset = import_limit_reset site.save(update_fields=["import_size_limit", "import_limit_reset"]) return redirect("settings-imports") + + +@require_POST +@login_required +@permission_required("bookwyrm.moderate_user", raise_exception=True) +# pylint: disable=unused-argument +def set_user_import_completed(request, import_id): + """Mark a user import as complete""" + import_job = get_object_or_404(models.BookwyrmImportJob, id=import_id) + import_job.stop_job() + return redirect("settings-imports") + + +@require_POST +@permission_required("bookwyrm.edit_instance_settings", raise_exception=True) +# pylint: disable=unused-argument +def set_user_import_limit(request): + """Limit how ofter users can import and export their account""" + site = models.SiteSettings.objects.get() + site.user_import_time_limit = int(request.POST.get("limit")) + site.save(update_fields=["user_import_time_limit"]) + return redirect("settings-imports") diff --git a/bookwyrm/views/imports/import_data.py b/bookwyrm/views/imports/import_data.py index 01812e1d5..1a9085ce1 100644 --- a/bookwyrm/views/imports/import_data.py +++ b/bookwyrm/views/imports/import_data.py @@ -15,12 +15,14 @@ from django.views import View from bookwyrm import forms, models from bookwyrm.importers import ( + BookwyrmImporter, CalibreImporter, LibrarythingImporter, GoodreadsImporter, StorygraphImporter, OpenLibraryImporter, ) +from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob from bookwyrm.settings import PAGE_LENGTH from bookwyrm.utils.cache import get_or_set @@ -127,3 +129,61 @@ def get_average_import_time() -> float: if recent_avg: return recent_avg.total_seconds() return None + + +# pylint: disable= no-self-use +@method_decorator(login_required, name="dispatch") +class UserImport(View): + """import user view""" + + def get(self, request, invalid=False): + """load user import page""" + + jobs = BookwyrmImportJob.objects.filter(user=request.user).order_by( + "-created_date" + ) + site = models.SiteSettings.objects.get() + hours = site.user_import_time_limit + allowed = ( + jobs.first().created_date < timezone.now() - datetime.timedelta(hours=hours) + if jobs.first() + else True + ) + next_available = ( + jobs.first().created_date + datetime.timedelta(hours=hours) + if not allowed + else False + ) + paginated = Paginator(jobs, PAGE_LENGTH) + page = paginated.get_page(request.GET.get("page")) + data = { + "import_form": forms.ImportUserForm(), + "jobs": page, + "user_import_hours": hours, + "next_available": next_available, + "page_range": paginated.get_elided_page_range( + page.number, on_each_side=2, on_ends=1 + ), + "invalid": invalid, + } + + return TemplateResponse(request, "import/import_user.html", data) + + def post(self, request): + """ingest a Bookwyrm json file""" + + importer = BookwyrmImporter() + + form = forms.ImportUserForm(request.POST, request.FILES) + if not form.is_valid(): + return HttpResponseBadRequest() + + job = importer.process_import( + user=request.user, + archive_file=request.FILES["archive_file"], + settings=request.POST, + ) + + job.start_job() + + return redirect("user-import") diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index 6880318bc..f54d97ccb 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -1,15 +1,21 @@ """ Let users export their book data """ +from datetime import timedelta import csv import io from django.contrib.auth.decorators import login_required +from django.core.paginator import Paginator from django.db.models import Q from django.http import HttpResponse from django.template.response import TemplateResponse +from django.utils import timezone from django.views import View from django.utils.decorators import method_decorator +from django.shortcuts import redirect from bookwyrm import models +from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob +from bookwyrm.settings import PAGE_LENGTH # pylint: disable=no-self-use @method_decorator(login_required, name="dispatch") @@ -84,3 +90,61 @@ class Export(View): "Content-Disposition": 'attachment; filename="bookwyrm-export.csv"' }, ) + + +# pylint: disable=no-self-use +@method_decorator(login_required, name="dispatch") +class ExportUser(View): + """Let users export user data to import into another Bookwyrm instance""" + + def get(self, request): + """Request tar file""" + + jobs = BookwyrmExportJob.objects.filter(user=request.user).order_by( + "-created_date" + ) + site = models.SiteSettings.objects.get() + hours = site.user_import_time_limit + allowed = ( + jobs.first().created_date < timezone.now() - timedelta(hours=hours) + if jobs.first() + else True + ) + next_available = ( + jobs.first().created_date + timedelta(hours=hours) if not allowed else False + ) + paginated = Paginator(jobs, PAGE_LENGTH) + page = paginated.get_page(request.GET.get("page")) + data = { + "jobs": page, + "next_available": next_available, + "page_range": paginated.get_elided_page_range( + page.number, on_each_side=2, on_ends=1 + ), + } + + return TemplateResponse(request, "preferences/export-user.html", data) + + def post(self, request): + """Download the json file of a user's data""" + + job = BookwyrmExportJob.objects.create(user=request.user) + job.start_job() + + return redirect("prefs-user-export") + + +@method_decorator(login_required, name="dispatch") +class ExportArchive(View): + """Serve the archive file""" + + def get(self, request, archive_id): + """download user export file""" + export = BookwyrmExportJob.objects.get(task_id=archive_id, user=request.user) + return HttpResponse( + export.export_data, + content_type="application/gzip", + headers={ + "Content-Disposition": 'attachment; filename="bookwyrm-account-export.tar.gz"' # pylint: disable=line-too-long + }, + )