Merge pull request #3054 from bookwyrm-social/user-migration

User migration via export file

bookwyrm/forms/forms.py

@@ -25,6 +25,10 @@ class ImportForm(forms.Form):
     csv_file = forms.FileField()
 
 
+class ImportUserForm(forms.Form):
+    archive_file = forms.FileField()
+
+
 class ShelfForm(CustomForm):
     class Meta:
         model = models.Shelf

bookwyrm/importers/__init__.py

@@ -1,6 +1,7 @@
 """ import classes """
 
 from .importer import Importer
+from .bookwyrm_import import BookwyrmImporter
 from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter

bookwyrm/importers/bookwyrm_import.py

@@ -0,0 +1,24 @@
+"""Import data from Bookwyrm export files"""
+from django.http import QueryDict
+
+from bookwyrm.models import User
+from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
+
+
+class BookwyrmImporter:
+    """Import a Bookwyrm User export file.
+    This is kind of a combination of an importer and a connector.
+    """
+
+    # pylint: disable=no-self-use
+    def process_import(
+        self, user: User, archive_file: bytes, settings: QueryDict
+    ) -> BookwyrmImportJob:
+        """import user data from a Bookwyrm export file"""
+
+        required = [k for k in settings if settings.get(k) == "on"]
+
+        job = BookwyrmImportJob.objects.create(
+            user=user, archive_file=archive_file, required=required
+        )
+        return job

bookwyrm/migrations/0186_auto_20231116_0048.py

@@ -0,0 +1,212 @@
+# Generated by Django 3.2.20 on 2023-11-16 00:48
+
+from django.conf import settings
+import django.contrib.postgres.fields
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0185_alter_notification_notification_type"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ParentJob",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("task_id", models.UUIDField(blank=True, null=True, unique=True)),
+                (
+                    "created_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                (
+                    "updated_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("complete", models.BooleanField(default=False)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("pending", "Pending"),
+                            ("active", "Active"),
+                            ("complete", "Complete"),
+                            ("stopped", "Stopped"),
+                            ("failed", "Failed"),
+                        ],
+                        default="pending",
+                        max_length=50,
+                        null=True,
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="user_import_time_limit",
+            field=models.IntegerField(default=48),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("USER_IMPORT", "User Import"),
+                    ("USER_EXPORT", "User Export"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+        migrations.CreateModel(
+            name="BookwyrmExportJob",
+            fields=[
+                (
+                    "parentjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+                ("export_data", models.FileField(null=True, upload_to="")),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.parentjob",),
+        ),
+        migrations.CreateModel(
+            name="BookwyrmImportJob",
+            fields=[
+                (
+                    "parentjob_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+                ("archive_file", models.FileField(blank=True, null=True, upload_to="")),
+                ("import_data", models.JSONField(null=True)),
+                (
+                    "required",
+                    django.contrib.postgres.fields.ArrayField(
+                        base_field=models.CharField(blank=True, max_length=50),
+                        blank=True,
+                        size=None,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("bookwyrm.parentjob",),
+        ),
+        migrations.CreateModel(
+            name="ChildJob",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("task_id", models.UUIDField(blank=True, null=True, unique=True)),
+                (
+                    "created_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                (
+                    "updated_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("complete", models.BooleanField(default=False)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("pending", "Pending"),
+                            ("active", "Active"),
+                            ("complete", "Complete"),
+                            ("stopped", "Stopped"),
+                            ("failed", "Failed"),
+                        ],
+                        default="pending",
+                        max_length=50,
+                        null=True,
+                    ),
+                ),
+                (
+                    "parent_job",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="child_jobs",
+                        to="bookwyrm.parentjob",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="notification",
+            name="related_user_export",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="bookwyrm.bookwyrmexportjob",
+            ),
+        ),
+    ]

bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.23 on 2023-11-22 10:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0186_auto_20231116_0048"),
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = []

bookwyrm/migrations/0190_alter_notification_notification_type.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.23 on 2023-11-23 19:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0189_merge_0186_auto_20231116_0048_0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("BOOST", "Boost"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("IMPORT", "Import"),
+                    ("USER_IMPORT", "User Import"),
+                    ("USER_EXPORT", "User Export"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE_REQUEST", "Invite Request"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                    ("MOVE", "Move"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/models/__init__.py

@@ -26,6 +26,8 @@ from .federated_server import FederatedServer
 from .group import Group, GroupMember, GroupMemberInvitation
 
 from .import_job import ImportJob, ImportItem
+from .bookwyrm_import_job import BookwyrmImportJob
+from .bookwyrm_export_job import BookwyrmExportJob
 
 from .move import MoveUser
 

bookwyrm/models/bookwyrm_export_job.py

@@ -0,0 +1,232 @@
+"""Export user account to tar.gz file for import into another Bookwyrm instance"""
+
+import dataclasses
+import logging
+from uuid import uuid4
+
+from django.db.models import FileField
+from django.db.models import Q
+from django.core.serializers.json import DjangoJSONEncoder
+from django.core.files.base import ContentFile
+
+from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, List, ListItem
+from bookwyrm.models import Review, Comment, Quotation
+from bookwyrm.models import Edition
+from bookwyrm.models import UserFollows, User, UserBlocks
+from bookwyrm.models.job import ParentJob, ParentTask
+from bookwyrm.tasks import app, IMPORTS
+from bookwyrm.utils.tar import BookwyrmTarFile
+
+logger = logging.getLogger(__name__)
+
+
+class BookwyrmExportJob(ParentJob):
+    """entry for a specific request to export a bookwyrm user"""
+
+    export_data = FileField(null=True)
+
+    def start_job(self):
+        """Start the job"""
+        start_export_task.delay(job_id=self.id, no_children=True)
+
+        return self
+
+
+@app.task(queue=IMPORTS, base=ParentTask)
+def start_export_task(**kwargs):
+    """trigger the child tasks for each row"""
+    job = BookwyrmExportJob.objects.get(id=kwargs["job_id"])
+
+    # don't start the job if it was stopped from the UI
+    if job.complete:
+        return
+    try:
+        # This is where ChildJobs get made
+        job.export_data = ContentFile(b"", str(uuid4()))
+        json_data = json_export(job.user)
+        tar_export(json_data, job.user, job.export_data)
+        job.save(update_fields=["export_data"])
+    except Exception as err:  # pylint: disable=broad-except
+        logger.exception("User Export Job %s Failed with error: %s", job.id, err)
+        job.set_status("failed")
+
+    job.set_status("complete")
+
+
+def tar_export(json_data: str, user, file):
+    """wrap the export information in a tar file"""
+    file.open("wb")
+    with BookwyrmTarFile.open(mode="w:gz", fileobj=file) as tar:
+        tar.write_bytes(json_data.encode("utf-8"))
+
+        # Add avatar image if present
+        if getattr(user, "avatar", False):
+            tar.add_image(user.avatar, filename="avatar")
+
+        editions = get_books_for_user(user)
+        for book in editions:
+            if getattr(book, "cover", False):
+                tar.add_image(book.cover)
+
+    file.close()
+
+
+def json_export(
+    user,
+):  # pylint: disable=too-many-locals, too-many-statements, too-many-branches
+    """Generate an export for a user"""
+
+    # User as AP object
+    exported_user = user.to_activity()
+    # I don't love this but it prevents a JSON encoding error
+    # when there is no user image
+    if isinstance(
+        exported_user["icon"],
+        dataclasses._MISSING_TYPE,  # pylint: disable=protected-access
+    ):
+        exported_user["icon"] = {}
+    else:
+        # change the URL to be relative to the JSON file
+        file_type = exported_user["icon"]["url"].rsplit(".", maxsplit=1)[-1]
+        filename = f"avatar.{file_type}"
+        exported_user["icon"]["url"] = filename
+
+    # Additional settings - can't be serialized as AP
+    vals = [
+        "show_goal",
+        "preferred_timezone",
+        "default_post_privacy",
+        "show_suggested_users",
+    ]
+    exported_user["settings"] = {}
+    for k in vals:
+        exported_user["settings"][k] = getattr(user, k)
+
+    # Reading goals - can't be serialized as AP
+    reading_goals = AnnualGoal.objects.filter(user=user).distinct()
+    exported_user["goals"] = []
+    for goal in reading_goals:
+        exported_user["goals"].append(
+            {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy}
+        )
+
+    # Reading history - can't be serialized as AP
+    readthroughs = ReadThrough.objects.filter(user=user).distinct().values()
+    readthroughs = list(readthroughs)
+
+    # Books
+    editions = get_books_for_user(user)
+    exported_user["books"] = []
+
+    for edition in editions:
+        book = {}
+        book["work"] = edition.parent_work.to_activity()
+        book["edition"] = edition.to_activity()
+
+        if book["edition"].get("cover"):
+            # change the URL to be relative to the JSON file
+            filename = book["edition"]["cover"]["url"].rsplit("/", maxsplit=1)[-1]
+            book["edition"]["cover"]["url"] = f"covers/{filename}"
+
+        # authors
+        book["authors"] = []
+        for author in edition.authors.all():
+            book["authors"].append(author.to_activity())
+
+        # Shelves this book is on
+        # Every ShelfItem is this book so we don't other serializing
+        book["shelves"] = []
+        shelf_books = (
+            ShelfBook.objects.select_related("shelf")
+            .filter(user=user, book=edition)
+            .distinct()
+        )
+
+        for shelfbook in shelf_books:
+            book["shelves"].append(shelfbook.shelf.to_activity())
+
+        # Lists and ListItems
+        # ListItems include "notes" and "approved" so we need them
+        # even though we know it's this book
+        book["lists"] = []
+        list_items = ListItem.objects.filter(book=edition, user=user).distinct()
+
+        for item in list_items:
+            list_info = item.book_list.to_activity()
+            list_info[
+                "privacy"
+            ] = item.book_list.privacy  # this isn't serialized so we add it
+            list_info["list_item"] = item.to_activity()
+            book["lists"].append(list_info)
+
+        # Statuses
+        # Can't use select_subclasses here because
+        # we need to filter on the "book" value,
+        # which is not available on an ordinary Status
+        for status in ["comments", "quotations", "reviews"]:
+            book[status] = []
+
+        comments = Comment.objects.filter(user=user, book=edition).all()
+        for status in comments:
+            obj = status.to_activity()
+            obj["progress"] = status.progress
+            obj["progress_mode"] = status.progress_mode
+            book["comments"].append(obj)
+
+        quotes = Quotation.objects.filter(user=user, book=edition).all()
+        for status in quotes:
+            obj = status.to_activity()
+            obj["position"] = status.position
+            obj["endposition"] = status.endposition
+            obj["position_mode"] = status.position_mode
+            book["quotations"].append(obj)
+
+        reviews = Review.objects.filter(user=user, book=edition).all()
+        for status in reviews:
+            obj = status.to_activity()
+            book["reviews"].append(obj)
+
+        # readthroughs can't be serialized to activity
+        book_readthroughs = (
+            ReadThrough.objects.filter(user=user, book=edition).distinct().values()
+        )
+        book["readthroughs"] = list(book_readthroughs)
+
+        # append everything
+        exported_user["books"].append(book)
+
+    # saved book lists - just the remote id
+    saved_lists = List.objects.filter(id__in=user.saved_lists.all()).distinct()
+    exported_user["saved_lists"] = [l.remote_id for l in saved_lists]
+
+    # follows - just the remote id
+    follows = UserFollows.objects.filter(user_subject=user).distinct()
+    following = User.objects.filter(userfollows_user_object__in=follows).distinct()
+    exported_user["follows"] = [f.remote_id for f in following]
+
+    # blocks - just the remote id
+    blocks = UserBlocks.objects.filter(user_subject=user).distinct()
+    blocking = User.objects.filter(userblocks_user_object__in=blocks).distinct()
+
+    exported_user["blocks"] = [b.remote_id for b in blocking]
+
+    return DjangoJSONEncoder().encode(exported_user)
+
+
+def get_books_for_user(user):
+    """Get all the books and editions related to a user"""
+
+    editions = (
+        Edition.objects.select_related("parent_work")
+        .filter(
+            Q(shelves__user=user)
+            | Q(readthrough__user=user)
+            | Q(review__user=user)
+            | Q(list__user=user)
+            | Q(comment__user=user)
+            | Q(quotation__user=user)
+        )
+        .distinct()
+    )
+
+    return editions

bookwyrm/models/bookwyrm_import_job.py

@@ -0,0 +1,459 @@
+"""Import a user from another Bookwyrm instance"""
+
+import json
+import logging
+
+from django.db.models import FileField, JSONField, CharField
+from django.utils import timezone
+from django.utils.html import strip_tags
+from django.contrib.postgres.fields import ArrayField as DjangoArrayField
+
+from bookwyrm import activitypub
+from bookwyrm import models
+from bookwyrm.tasks import app, IMPORTS
+from bookwyrm.models.job import ParentJob, ParentTask, SubTask
+from bookwyrm.utils.tar import BookwyrmTarFile
+
+logger = logging.getLogger(__name__)
+
+
+class BookwyrmImportJob(ParentJob):
+    """entry for a specific request for importing a bookwyrm user backup"""
+
+    archive_file = FileField(null=True, blank=True)
+    import_data = JSONField(null=True)
+    required = DjangoArrayField(CharField(max_length=50, blank=True), blank=True)
+
+    def start_job(self):
+        """Start the job"""
+        start_import_task.delay(job_id=self.id, no_children=True)
+
+
+@app.task(queue=IMPORTS, base=ParentTask)
+def start_import_task(**kwargs):
+    """trigger the child import tasks for each user data"""
+    job = BookwyrmImportJob.objects.get(id=kwargs["job_id"])
+    archive_file = job.archive_file
+
+    # don't start the job if it was stopped from the UI
+    if job.complete:
+        return
+
+    try:
+        archive_file.open("rb")
+        with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar:
+            job.import_data = json.loads(tar.read("archive.json").decode("utf-8"))
+
+            if "include_user_profile" in job.required:
+                update_user_profile(job.user, tar, job.import_data)
+            if "include_user_settings" in job.required:
+                update_user_settings(job.user, job.import_data)
+            if "include_goals" in job.required:
+                update_goals(job.user, job.import_data.get("goals"))
+            if "include_saved_lists" in job.required:
+                upsert_saved_lists(job.user, job.import_data.get("saved_lists"))
+            if "include_follows" in job.required:
+                upsert_follows(job.user, job.import_data.get("follows"))
+            if "include_blocks" in job.required:
+                upsert_user_blocks(job.user, job.import_data.get("blocks"))
+
+            process_books(job, tar)
+
+            job.set_status("complete")
+        archive_file.close()
+
+    except Exception as err:  # pylint: disable=broad-except
+        logger.exception("User Import Job %s Failed with error: %s", job.id, err)
+        job.set_status("failed")
+
+
+def process_books(job, tar):
+    """
+    Process user import data related to books
+    We always import the books even if not assigning
+    them to shelves, lists etc
+    """
+
+    books = job.import_data.get("books")
+
+    for data in books:
+        book = get_or_create_edition(data, tar)
+
+        if "include_shelves" in job.required:
+            upsert_shelves(book, job.user, data)
+
+        if "include_readthroughs" in job.required:
+            upsert_readthroughs(data.get("readthroughs"), job.user, book.id)
+
+        if "include_comments" in job.required:
+            upsert_statuses(
+                job.user, models.Comment, data.get("comments"), book.remote_id
+            )
+        if "include_quotations" in job.required:
+            upsert_statuses(
+                job.user, models.Quotation, data.get("quotations"), book.remote_id
+            )
+
+        if "include_reviews" in job.required:
+            upsert_statuses(
+                job.user, models.Review, data.get("reviews"), book.remote_id
+            )
+
+        if "include_lists" in job.required:
+            upsert_lists(job.user, data.get("lists"), book.id)
+
+
+def get_or_create_edition(book_data, tar):
+    """Take a JSON string of work and edition data,
+    find or create the edition and work in the database and
+    return an edition instance"""
+
+    edition = book_data.get("edition")
+    existing = models.Edition.find_existing(edition)
+    if existing:
+        return existing
+
+    # make sure we have the authors in the local DB
+    # replace the old author ids in the edition JSON
+    edition["authors"] = []
+    for author in book_data.get("authors"):
+        parsed_author = activitypub.parse(author)
+        instance = parsed_author.to_model(
+            model=models.Author, save=True, overwrite=True
+        )
+
+        edition["authors"].append(instance.remote_id)
+
+    # we will add the cover later from the tar
+    # don't try to load it from the old server
+    cover = edition.get("cover", {})
+    cover_path = cover.get("url", None)
+    edition["cover"] = {}
+
+    # first we need the parent work to exist
+    work = book_data.get("work")
+    work["editions"] = []
+    parsed_work = activitypub.parse(work)
+    work_instance = parsed_work.to_model(model=models.Work, save=True, overwrite=True)
+
+    # now we have a work we can add it to the edition
+    # and create the edition model instance
+    edition["work"] = work_instance.remote_id
+    parsed_edition = activitypub.parse(edition)
+    book = parsed_edition.to_model(model=models.Edition, save=True, overwrite=True)
+
+    # set the cover image from the tar
+    if cover_path:
+        tar.write_image_to_file(cover_path, book.cover)
+
+    return book
+
+
+def upsert_readthroughs(data, user, book_id):
+    """Take a JSON string of readthroughs and
+    find or create the instances in the database"""
+
+    for read_through in data:
+
+        obj = {}
+        keys = [
+            "progress_mode",
+            "start_date",
+            "finish_date",
+            "stopped_date",
+            "is_active",
+        ]
+        for key in keys:
+            obj[key] = read_through[key]
+        obj["user_id"] = user.id
+        obj["book_id"] = book_id
+
+        existing = models.ReadThrough.objects.filter(**obj).first()
+        if not existing:
+            models.ReadThrough.objects.create(**obj)
+
+
+def upsert_statuses(user, cls, data, book_remote_id):
+    """Take a JSON string of a status and
+    find or create the instances in the database"""
+
+    for status in data:
+        if is_alias(
+            user, status["attributedTo"]
+        ):  # don't let l33t hax0rs steal other people's posts
+            # update ids and remove replies
+            status["attributedTo"] = user.remote_id
+            status["to"] = update_followers_address(user, status["to"])
+            status["cc"] = update_followers_address(user, status["cc"])
+            status[
+                "replies"
+            ] = (
+                {}
+            )  # this parses incorrectly but we can't set it without knowing the new id
+            status["inReplyToBook"] = book_remote_id
+            parsed = activitypub.parse(status)
+            if not status_already_exists(
+                user, parsed
+            ):  # don't duplicate posts on multiple import
+
+                instance = parsed.to_model(model=cls, save=True, overwrite=True)
+
+                for val in [
+                    "progress",
+                    "progress_mode",
+                    "position",
+                    "endposition",
+                    "position_mode",
+                ]:
+                    if status.get(val):
+                        instance.val = status[val]
+
+                instance.remote_id = instance.get_remote_id()  # update the remote_id
+                instance.save()  # save and broadcast
+
+        else:
+            logger.info("User does not have permission to import statuses")
+
+
+def upsert_lists(user, lists, book_id):
+    """Take a list of objects each containing
+    a list and list item as AP objects
+
+    Because we are creating new IDs we can't assume the id
+    will exist or be accurate, so we only use to_model for
+    adding new items after checking whether they exist  .
+
+    """
+
+    book = models.Edition.objects.get(id=book_id)
+
+    for blist in lists:
+        booklist = models.List.objects.filter(name=blist["name"], user=user).first()
+        if not booklist:
+
+            blist["owner"] = user.remote_id
+            parsed = activitypub.parse(blist)
+            booklist = parsed.to_model(model=models.List, save=True, overwrite=True)
+
+            booklist.privacy = blist["privacy"]
+            booklist.save()
+
+        item = models.ListItem.objects.filter(book=book, book_list=booklist).exists()
+        if not item:
+            count = booklist.books.count()
+            models.ListItem.objects.create(
+                book=book,
+                book_list=booklist,
+                user=user,
+                notes=blist["list_item"]["notes"],
+                approved=blist["list_item"]["approved"],
+                order=count + 1,
+            )
+
+
+def upsert_shelves(book, user, book_data):
+    """Take shelf JSON objects and create
+    DB entries if they don't already exist"""
+
+    shelves = book_data["shelves"]
+    for shelf in shelves:
+
+        book_shelf = models.Shelf.objects.filter(name=shelf["name"], user=user).first()
+
+        if not book_shelf:
+            book_shelf = models.Shelf.objects.create(name=shelf["name"], user=user)
+
+        # add the book as a ShelfBook if needed
+        if not models.ShelfBook.objects.filter(
+            book=book, shelf=book_shelf, user=user
+        ).exists():
+            models.ShelfBook.objects.create(
+                book=book, shelf=book_shelf, user=user, shelved_date=timezone.now()
+            )
+
+
+def update_user_profile(user, tar, data):
+    """update the user's profile from import data"""
+    name = data.get("name", None)
+    username = data.get("preferredUsername")
+    user.name = name if name else username
+    user.summary = strip_tags(data.get("summary", None))
+    user.save(update_fields=["name", "summary"])
+    if data["icon"].get("url"):
+        avatar_filename = next(filter(lambda n: n.startswith("avatar"), tar.getnames()))
+        tar.write_image_to_file(avatar_filename, user.avatar)
+
+
+def update_user_settings(user, data):
+    """update the user's settings from import data"""
+
+    update_fields = ["manually_approves_followers", "hide_follows", "discoverable"]
+
+    ap_fields = [
+        ("manuallyApprovesFollowers", "manually_approves_followers"),
+        ("hideFollows", "hide_follows"),
+        ("discoverable", "discoverable"),
+    ]
+
+    for (ap_field, bw_field) in ap_fields:
+        setattr(user, bw_field, data[ap_field])
+
+    bw_fields = [
+        "show_goal",
+        "show_suggested_users",
+        "default_post_privacy",
+        "preferred_timezone",
+    ]
+
+    for field in bw_fields:
+        update_fields.append(field)
+        setattr(user, field, data["settings"][field])
+
+    user.save(update_fields=update_fields)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def update_user_settings_task(job_id):
+    """wrapper task for user's settings import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return update_user_settings(parent_job.user, parent_job.import_data.get("user"))
+
+
+def update_goals(user, data):
+    """update the user's goals from import data"""
+
+    for goal in data:
+        # edit the existing goal if there is one
+        existing = models.AnnualGoal.objects.filter(
+            year=goal["year"], user=user
+        ).first()
+        if existing:
+            for k in goal.keys():
+                setattr(existing, k, goal[k])
+            existing.save()
+        else:
+            goal["user"] = user
+            models.AnnualGoal.objects.create(**goal)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def update_goals_task(job_id):
+    """wrapper task for user's goals import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return update_goals(parent_job.user, parent_job.import_data.get("goals"))
+
+
+def upsert_saved_lists(user, values):
+    """Take a list of remote ids and add as saved lists"""
+
+    for remote_id in values:
+        book_list = activitypub.resolve_remote_id(remote_id, models.List)
+        if book_list:
+            user.saved_lists.add(book_list)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def upsert_saved_lists_task(job_id):
+    """wrapper task for user's saved lists import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return upsert_saved_lists(
+        parent_job.user, parent_job.import_data.get("saved_lists")
+    )
+
+
+def upsert_follows(user, values):
+    """Take a list of remote ids and add as follows"""
+
+    for remote_id in values:
+        followee = activitypub.resolve_remote_id(remote_id, models.User)
+        if followee:
+            (follow_request, created,) = models.UserFollowRequest.objects.get_or_create(
+                user_subject=user,
+                user_object=followee,
+            )
+
+            if not created:
+                # this request probably failed to connect with the remote
+                # and should save to trigger a re-broadcast
+                follow_request.save()
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def upsert_follows_task(job_id):
+    """wrapper task for user's follows import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return upsert_follows(parent_job.user, parent_job.import_data.get("follows"))
+
+
+def upsert_user_blocks(user, user_ids):
+    """block users"""
+
+    for user_id in user_ids:
+        user_object = activitypub.resolve_remote_id(user_id, models.User)
+        if user_object:
+            exists = models.UserBlocks.objects.filter(
+                user_subject=user, user_object=user_object
+            ).exists()
+            if not exists:
+                models.UserBlocks.objects.create(
+                    user_subject=user, user_object=user_object
+                )
+                # remove the blocked users's lists from the groups
+                models.List.remove_from_group(user, user_object)
+                # remove the blocked user from all blocker's owned groups
+                models.GroupMember.remove(user, user_object)
+
+
+@app.task(queue=IMPORTS, base=SubTask)
+def upsert_user_blocks_task(job_id):
+    """wrapper task for user's blocks import"""
+    parent_job = BookwyrmImportJob.objects.get(id=job_id)
+
+    return upsert_user_blocks(
+        parent_job.user, parent_job.import_data.get("blocked_users")
+    )
+
+
+def update_followers_address(user, field):
+    """statuses to or cc followers need to have the followers
+    address updated to the new local user"""
+
+    for i, audience in enumerate(field):
+        if audience.rsplit("/")[-1] == "followers":
+            field[i] = user.followers_url
+
+    return field
+
+
+def is_alias(user, remote_id):
+    """check that the user is listed as movedTo or also_known_as
+    in the remote user's profile"""
+
+    remote_user = activitypub.resolve_remote_id(
+        remote_id=remote_id, model=models.User, save=False
+    )
+
+    if remote_user:
+
+        if remote_user.moved_to:
+            return user.remote_id == remote_user.moved_to
+
+        if remote_user.also_known_as:
+            return user in remote_user.also_known_as.all()
+
+    return False
+
+
+def status_already_exists(user, status):
+    """check whether this status has already been published
+    by this user. We can't rely on to_model() because it
+    only matches on remote_id, which we have to change
+    *after* saving because it needs the primary key (id)"""
+
+    return models.Status.objects.filter(
+        user=user, content=status.content, published_date=status.published
+    ).exists()

bookwyrm/models/job.py

@@ -0,0 +1,308 @@
+"""Everything needed for Celery to multi-thread complex tasks."""
+
+from django.db import models
+from django.db import transaction
+from django.utils.translation import gettext_lazy as _
+from django.utils import timezone
+from bookwyrm.models.user import User
+
+from bookwyrm.tasks import app
+
+
+class Job(models.Model):
+    """Abstract model to store the state of a Task."""
+
+    class Status(models.TextChoices):
+        """Possible job states."""
+
+        PENDING = "pending", _("Pending")
+        ACTIVE = "active", _("Active")
+        COMPLETE = "complete", _("Complete")
+        STOPPED = "stopped", _("Stopped")
+        FAILED = "failed", _("Failed")
+
+    task_id = models.UUIDField(unique=True, null=True, blank=True)
+
+    created_date = models.DateTimeField(default=timezone.now)
+    updated_date = models.DateTimeField(default=timezone.now)
+    complete = models.BooleanField(default=False)
+    status = models.CharField(
+        max_length=50, choices=Status.choices, default=Status.PENDING, null=True
+    )
+
+    class Meta:
+        """Make it abstract"""
+
+        abstract = True
+
+    def complete_job(self):
+        """Report that the job has completed"""
+        if self.complete:
+            return
+
+        self.status = self.Status.COMPLETE
+        self.complete = True
+        self.updated_date = timezone.now()
+
+        self.save(update_fields=["status", "complete", "updated_date"])
+
+    def stop_job(self, reason=None):
+        """Stop the job"""
+        if self.complete:
+            return
+
+        self.__terminate_job()
+
+        if reason and reason == "failed":
+            self.status = self.Status.FAILED
+        else:
+            self.status = self.Status.STOPPED
+        self.complete = True
+        self.updated_date = timezone.now()
+
+        self.save(update_fields=["status", "complete", "updated_date"])
+
+    def set_status(self, status):
+        """Set job status"""
+        if self.complete:
+            return
+
+        if self.status == status:
+            return
+
+        if status == self.Status.COMPLETE:
+            self.complete_job()
+            return
+
+        if status == self.Status.STOPPED:
+            self.stop_job()
+            return
+
+        if status == self.Status.FAILED:
+            self.stop_job(reason="failed")
+            return
+
+        self.updated_date = timezone.now()
+        self.status = status
+
+        self.save(update_fields=["status", "updated_date"])
+
+    def __terminate_job(self):
+        """Tell workers to ignore and not execute this task."""
+        app.control.revoke(self.task_id, terminate=True)
+
+
+class ParentJob(Job):
+    """Store the state of a Task which can spawn many :model:`ChildJob`s to spread
+    resource load.
+
+    Intended to be sub-classed if necessary via proxy or
+    multi-table inheritance.
+    Extends :model:`Job`.
+    """
+
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+
+    def complete_job(self):
+        """Report that the job has completed and stop pending
+        children. Extend.
+        """
+        super().complete_job()
+        self.__terminate_pending_child_jobs()
+
+    def notify_child_job_complete(self):
+        """let the job know when the items get work done"""
+        if self.complete:
+            return
+
+        self.updated_date = timezone.now()
+        self.save(update_fields=["updated_date"])
+
+        if not self.complete and self.has_completed:
+            self.complete_job()
+
+    def __terminate_job(self):  # pylint: disable=unused-private-member
+        """Tell workers to ignore and not execute this task
+        & pending child tasks. Extend.
+        """
+        super().__terminate_job()
+        self.__terminate_pending_child_jobs()
+
+    def __terminate_pending_child_jobs(self):
+        """Tell workers to ignore and not execute any pending child tasks."""
+        tasks = self.pending_child_jobs.filter(task_id__isnull=False).values_list(
+            "task_id", flat=True
+        )
+        app.control.revoke(list(tasks))
+
+        for task in self.pending_child_jobs:
+            task.update(status=self.Status.STOPPED)
+
+    @property
+    def has_completed(self):
+        """has this job finished"""
+        return not self.pending_child_jobs.exists()
+
+    @property
+    def pending_child_jobs(self):
+        """items that haven't been processed yet"""
+        return self.child_jobs.filter(complete=False)
+
+
+class ChildJob(Job):
+    """Stores the state of a Task for the related :model:`ParentJob`.
+
+    Intended to be sub-classed if necessary via proxy or
+    multi-table inheritance.
+    Extends :model:`Job`.
+    """
+
+    parent_job = models.ForeignKey(
+        ParentJob, on_delete=models.CASCADE, related_name="child_jobs"
+    )
+
+    def set_status(self, status):
+        """Set job and parent_job status. Extend."""
+        super().set_status(status)
+
+        if (
+            status == self.Status.ACTIVE
+            and self.parent_job.status == self.Status.PENDING
+        ):
+            self.parent_job.set_status(self.Status.ACTIVE)
+
+    def complete_job(self):
+        """Report to parent_job that the job has completed. Extend."""
+        super().complete_job()
+        self.parent_job.notify_child_job_complete()
+
+
+class ParentTask(app.Task):
+    """Used with ParentJob, Abstract Tasks execute code at specific points in
+    a Task's lifecycle, applying to all Tasks with the same 'base'.
+
+    All status & ParentJob.task_id assignment is managed here for you.
+    Usage e.g. @app.task(base=ParentTask)
+    """
+
+    def before_start(
+        self, task_id, args, kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Handler called before the task starts. Override.
+
+        Prepare ParentJob before the task starts.
+
+        Arguments:
+            task_id (str): Unique id of the task to execute.
+            args (Tuple): Original arguments for the task to execute.
+            kwargs (Dict): Original keyword arguments for the task to execute.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            no_children (bool): If 'True' this is the only Task expected to run
+                for the given ParentJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+        job = ParentJob.objects.get(id=kwargs["job_id"])
+        job.task_id = task_id
+        job.save(update_fields=["task_id"])
+
+        if kwargs["no_children"]:
+            job.set_status(ChildJob.Status.ACTIVE)
+
+    def on_success(
+        self, retval, task_id, args, kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Run by the worker if the task executes successfully. Override.
+
+        Update ParentJob on Task complete.
+
+        Arguments:
+            retval (Any): The return value of the task.
+            task_id (str): Unique id of the executed task.
+            args (Tuple): Original arguments for the executed task.
+            kwargs (Dict): Original keyword arguments for the executed task.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            no_children (bool): If 'True' this is the only Task expected to run
+                for the given ParentJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+
+        if kwargs["no_children"]:
+            job = ParentJob.objects.get(id=kwargs["job_id"])
+            job.complete_job()
+
+
+class SubTask(app.Task):
+    """Used with ChildJob, Abstract Tasks execute code at specific points in
+    a Task's lifecycle, applying to all Tasks with the same 'base'.
+
+    All status & ChildJob.task_id assignment is managed here for you.
+    Usage e.g. @app.task(base=SubTask)
+    """
+
+    def before_start(
+        self, task_id, args, kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Handler called before the task starts. Override.
+
+        Prepare ChildJob before the task starts.
+
+        Arguments:
+            task_id (str): Unique id of the task to execute.
+            args (Tuple): Original arguments for the task to execute.
+            kwargs (Dict): Original keyword arguments for the task to execute.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            child_id (int): Unique 'id' of the ChildJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+        child_job = ChildJob.objects.get(id=kwargs["child_id"])
+        child_job.task_id = task_id
+        child_job.save(update_fields=["task_id"])
+        child_job.set_status(ChildJob.Status.ACTIVE)
+
+    def on_success(
+        self, retval, task_id, args, kwargs
+    ):  # pylint: disable=no-self-use, unused-argument
+        """Run by the worker if the task executes successfully. Override.
+
+        Notify ChildJob of task completion.
+
+        Arguments:
+            retval (Any): The return value of the task.
+            task_id (str): Unique id of the executed task.
+            args (Tuple): Original arguments for the executed task.
+            kwargs (Dict): Original keyword arguments for the executed task.
+
+        Keyword Arguments:
+            job_id (int): Unique 'id' of the ParentJob.
+            child_id (int): Unique 'id' of the ChildJob.
+
+        Returns:
+            None: The return value of this handler is ignored.
+        """
+        subtask = ChildJob.objects.get(id=kwargs["child_id"])
+        subtask.complete_job()
+
+
+@transaction.atomic
+def create_child_job(parent_job, task_callback):
+    """Utility method for creating a ChildJob
+    and running a task to avoid DB race conditions
+    """
+    child_job = ChildJob.objects.create(parent_job=parent_job)
+    transaction.on_commit(
+        lambda: task_callback.delay(job_id=parent_job.id, child_id=child_job.id)
+    )
+
+    return child_job

bookwyrm/models/notification.py

@@ -1,8 +1,16 @@
 """ alert a user to activity """
 from django.db import models, transaction
 from django.dispatch import receiver
+from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob
 from .base_model import BookWyrmModel
-from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain
+from . import (
+    Boost,
+    Favorite,
+    GroupMemberInvitation,
+    ImportJob,
+    BookwyrmImportJob,
+    LinkDomain,
+)
 from . import ListItem, Report, Status, User, UserFollowRequest
 from .site import InviteRequest
 
@@ -23,6 +31,8 @@ class NotificationType(models.TextChoices):
 
     # Imports
     IMPORT = "IMPORT"
+    USER_IMPORT = "USER_IMPORT"
+    USER_EXPORT = "USER_EXPORT"
 
     # List activity
     ADD = "ADD"
@@ -63,6 +73,9 @@ class Notification(BookWyrmModel):
     )
     related_status = models.ForeignKey("Status", on_delete=models.CASCADE, null=True)
     related_import = models.ForeignKey("ImportJob", on_delete=models.CASCADE, null=True)
+    related_user_export = models.ForeignKey(
+        "BookwyrmExportJob", on_delete=models.CASCADE, null=True
+    )
     related_list_items = models.ManyToManyField(
         "ListItem", symmetrical=False, related_name="notifications"
     )
@@ -226,6 +239,36 @@ def notify_user_on_import_complete(
     )
 
 
+@receiver(models.signals.post_save, sender=BookwyrmImportJob)
+# pylint: disable=unused-argument
+def notify_user_on_user_import_complete(
+    sender, instance, *args, update_fields=None, **kwargs
+):
+    """we imported your user details! aren't you proud of us"""
+    update_fields = update_fields or []
+    if not instance.complete or "complete" not in update_fields:
+        return
+    Notification.objects.create(
+        user=instance.user, notification_type=NotificationType.USER_IMPORT
+    )
+
+
+@receiver(models.signals.post_save, sender=BookwyrmExportJob)
+# pylint: disable=unused-argument
+def notify_user_on_user_export_complete(
+    sender, instance, *args, update_fields=None, **kwargs
+):
+    """we exported your user details! aren't you proud of us"""
+    update_fields = update_fields or []
+    if not instance.complete or "complete" not in update_fields:
+        return
+    Notification.objects.create(
+        user=instance.user,
+        notification_type=NotificationType.USER_EXPORT,
+        related_user_export=instance,
+    )
+
+
 @receiver(models.signals.post_save, sender=Report)
 @transaction.atomic
 # pylint: disable=unused-argument

bookwyrm/models/site.py

@@ -96,6 +96,7 @@ class SiteSettings(SiteModel):
     imports_enabled = models.BooleanField(default=True)
     import_size_limit = models.IntegerField(default=0)
     import_limit_reset = models.IntegerField(default=0)
+    user_import_time_limit = models.IntegerField(default=48)
 
     field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"])
 

bookwyrm/templates/import/import.html

@@ -1,13 +1,12 @@
-{% extends 'layout.html' %}
+{% extends 'preferences/layout.html' %}
 {% load i18n %}
 {% load humanize %}
 
-{% block title %}{% trans "Import Books" %}{% endblock %}
+{% block title %}{% trans "Import Book List" %}{% endblock %}
+{% block header %}{% trans "Import Book List" %}{% endblock %}
 
-{% block content %}
+{% block panel %}
 <div class="block">
-    <h1 class="title">{% trans "Import Books" %}</h1>
-
     {% if invalid %}
     <div class="notification is-danger">
         {% trans "Not a valid CSV file" %}

bookwyrm/templates/import/import_user.html

@@ -0,0 +1,222 @@
+{% extends 'preferences/layout.html' %}
+{% load i18n %}
+{% load humanize %}
+
+{% block title %}{% trans "Import BookWyrm Account" %}{% endblock %}
+{% block header %}{% trans "Import BookWyrm Account" %}{% endblock %}
+
+{% block panel %}
+<div class="block">
+
+    {% if invalid %}
+    <div class="notification is-danger">
+        {% trans "Not a valid import file" %}
+    </div>
+    {% endif %}
+    <p class="notification is-warning">
+        {% spaceless %}
+        {% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set this account as an <strong>alias</strong> of the one you are migrating from, or <strong>move</strong> that account to this one, before you import your user data." %}
+        {% endspaceless %}
+    </p>
+    {% if not site.imports_enabled %}
+    <div class="box notification has-text-centered is-warning m-6 content">
+        <p class="mt-5">
+            <span class="icon icon-warning is-size-2" aria-hidden="true"></span>
+        </p>
+        <p class="mb-5">
+            {% trans "Imports are temporarily disabled; thank you for your patience." %}
+        </p>
+    </div>
+    {% elif next_available %}
+        <div class="notification is-warning">
+            <p>{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}</p>
+            <p>{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}</p>
+        </div>
+    {% else %}
+    <form class="box content" name="import-user" action="/user-import" method="post" enctype="multipart/form-data">
+        {% csrf_token %}
+
+        <div class="block">
+            <div class="notification">
+                <h2 class="is-size-5">{% trans "Step 1:" %}</h2>
+                <p>
+                    {% blocktrans trimmed %}
+                    Select an export file generated from another BookWyrm account. The file format should be <code>.tar.gz</code>.
+                    {% endblocktrans %}
+                </p>
+            </div>
+            <div class="block m-5">
+                <label class="label" for="id_archive_file">{% trans "Data file:" %}</label>
+                {{ import_form.archive_file }}
+            </div>
+        </div>
+
+        <hr aria-hidden="true">
+
+        <div class="block">
+            <div class="notification">
+                <h2 class="is-size-5">{% trans "Step 2:" %}</h2>
+                <p>
+                    {% blocktrans trimmed %}
+                    Deselect any checkboxes for data you do not wish to include in your import.
+                    {% endblocktrans %}
+                </p>
+                <p class="block">Unless specified below, importing will not delete any data. Imported data will be <strong>added if it does not already exist</strong>. For example, if you have an existing list with the same name as an imported list, the existing list settings will not change, any new list items will be added, and no existing list items will be deleted.</p>
+            </div>
+            <div class="block m-5 columns">
+                <div class="column is-half">
+                    <div class="field">
+                        <label class="label mb-0">
+                            <input type="checkbox" name="include_user_profile" checked aria-describedby="desc_include_user_profile">
+                            {% trans "User profile" %}
+                        </label>
+                        <p id="desc_include_user_profile">
+                        {% trans "Overwrites display name, summary, and avatar" %}
+                        </p>
+                    </div>
+                    <div class="field">
+                        <label class="label mb-0">
+                            <input type="checkbox" name="include_user_settings" checked aria-describedby="desc_include_user_settings">
+                            {% trans "User settings" %}
+                        </label>
+                        <div id="desc_include_user_settings">
+                            {% trans "Overwrites:" %}
+                            <ul class="mt-0">
+                                <li>
+                                    {% trans "Whether manual approval is required for other users to follow your account" %}
+                                </li>
+                                <li>
+                                    {% trans "Whether following/followers are shown on your profile" %}
+                                </li>
+                                <li>
+                                    {% trans "Whether your reading goal is shown on your profile" %}
+                                </li>
+                                <li>
+                                    {% trans "Whether you see user follow suggestions" %}
+                                </li>
+                                <li>
+                                    {% trans "Whether your account is suggested to others" %}
+                                </li>
+                                <li>
+                                    {% trans "Your timezone" %}
+                                </li>
+                                <li>
+                                    {% trans "Your default post privacy setting" %}
+                                </li>
+                            </ul>
+                        </div>
+                    </div>
+                    <div class="field">
+                        <label class="label">
+                            <input type="checkbox" name="include_follows" checked>
+                            {% trans "Followers and following" %}
+                        </label>
+                    </div>
+                    <label class="label">
+                        <input type="checkbox" name="include_blocks" checked> {% trans "User blocks" %}
+                    </label>
+                </div>
+                <div class="column is-half">
+                    <div class="field">
+                        <label class="label mb-0">
+                            <input type="checkbox" name="include_goals" checked aria-describedby="desc_include_goals">
+                            {% trans "Reading goals" %}
+                        </label>
+                        <p id="desc_include_goals">
+                            {% trans "Overwrites reading goals for all years listed in the import file" %}
+                        </p>
+                    </div>
+                    <label class="label">
+                        <input type="checkbox" name="include_shelves" checked> {% trans "Shelves" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_readthroughs" checked> {% trans "Reading history" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_reviews" checked> {% trans "Book reviews" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_quotations" checked> {% trans "Quotations" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_comments" checked> {% trans "Comments about books" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_lists" checked> {% trans "Book lists" %}
+                    </label>
+                    <label class="label">
+                        <input type="checkbox" name="include_saved_lists" checked> {% trans "Saved lists" %}
+                    </label>
+                </div>
+            </div>
+        </div>
+
+        {% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %}
+            <button class="button is-primary" type="submit">{% trans "Import" %}</button>
+        {% else %}
+            <button class="button is-primary is-disabled" type="submit">{% trans "Import" %}</button>
+            <p>{% trans "You've reached the import limit." %}</p>
+        {% endif%}
+    </form>
+    {% endif %}
+
+</div>
+
+<div class="content block">
+    <h2 class="title">{% trans "Recent Imports" %}</h2>
+    <div class="table-container">
+        <table class="table is-striped is-fullwidth">
+            <tr>
+                <th>
+                    {% trans "Date Created" %}
+                </th>
+                <th>
+                    {% trans "Last Updated" %}
+                </th>
+                <th>
+                    {% trans "Status" %}
+                </th>
+            </tr>
+            {% if not jobs %}
+            <tr>
+                <td colspan="4">
+                    <em>{% trans "No recent imports" %}</em>
+                </td>
+            </tr>
+            {% endif %}
+            {% for job in jobs %}
+            <tr>
+                <td>
+                    <p>{{ job.created_date }}</p>
+                </td>
+                <td>{{ job.updated_date }}</td>
+                <td>
+                    <span
+                        {% if job.status == "stopped" or job.status == "failed" %}
+                        class="tag is-danger"
+                        {% elif job.status == "pending" %}
+                        class="tag is-warning"
+                        {% elif job.complete %}
+                        class="tag"
+                        {% else %}
+                        class="tag is-success"
+                        {% endif %}
+                    >
+                        {% if job.status %}
+                        {{ job.status }}
+                            {{ job.status_display }}
+                        {% elif job.complete %}
+                            {% trans "Complete" %}
+                        {% else %}
+                            {% trans "Active" %}
+                        {% endif %}
+                    </span>
+                </td>
+            </tr>
+            {% endfor %}
+        </table>
+    </div>
+
+    {% include 'snippets/pagination.html' with page=jobs path=request.path %}
+</div>
+{% endblock %}

bookwyrm/templates/notifications/item.html

@@ -15,6 +15,10 @@
     {% endif %}
 {% elif notification.notification_type == 'IMPORT' %}
     {% include 'notifications/items/import.html' %}
+{% elif notification.notification_type == 'USER_IMPORT' %}
+    {% include 'notifications/items/user_import.html' %}
+{% elif notification.notification_type == 'USER_EXPORT' %}
+    {% include 'notifications/items/user_export.html' %}
 {% elif notification.notification_type == 'ADD' %}
     {% include 'notifications/items/add.html' %}
 {% elif notification.notification_type == 'REPORT' %}

bookwyrm/templates/notifications/items/user_export.html

@@ -0,0 +1,15 @@
+{% extends 'notifications/items/layout.html' %}
+{% load i18n %}
+
+{% block primary_link %}{% spaceless %}
+{% url 'prefs-user-export' %}
+{% endspaceless %}{% endblock %}
+
+{% block icon %}
+<span class="icon icon-list"></span>
+{% endblock %}
+
+{% block description %}
+    {% url 'prefs-user-export'  as url %}
+    {% blocktrans %}Your <a href="{{ url }}">user export</a> is ready.{% endblocktrans %}
+{% endblock %}

bookwyrm/templates/notifications/items/user_import.html

@@ -0,0 +1,16 @@
+{% extends 'notifications/items/layout.html' %}
+{% load i18n %}
+
+{% block primary_link %}{% spaceless %}
+{% url 'user-import' %}
+{% endspaceless %}{% endblock %}
+
+{% block icon %}
+<span class="icon icon-list"></span>
+{% endblock %}
+
+{% block description %}
+{% url 'user-import' as import_url %}
+{% blocktrans %}Your <a href="{{ import_url }}">user import</a> is complete.{% endblocktrans %}
+
+{% endblock %}

bookwyrm/templates/preferences/export-user.html

@@ -0,0 +1,138 @@
+{% extends 'preferences/layout.html' %}
+{% load i18n %}
+{% load utilities %}
+
+{% block title %}{% trans "Export BookWyrm Account" %}{% endblock %}
+
+{% block header %}
+{% trans "Export BookWyrm Account" %}
+{% endblock %}
+
+{% block panel %}
+<div class="block content">
+    <div class="block content">
+        <p> {% trans "You can create an export file here. This will allow you to migrate your data to another BookWyrm account." %}</p>
+    </div>
+        <div class="block mx-5 columns">
+            {% blocktrans trimmed %}
+            <div class="column is-half">
+                <h2 class="is-size-5">Your file will include:</h2>
+                <ul>
+                    <li>User profile</li>
+                    <li>Most user settings</li>
+                    <li>Reading goals</li>
+                    <li>Shelves</li>
+                    <li>Reading history</li>
+                    <li>Book reviews</li>
+                    <li>Statuses</li>
+                    <li>Your own lists and saved lists</li>
+                    <li>Which users you follow and block</li>
+                </ul>
+            </div>
+            <div class="column is-half">
+                <h2 class="is-size-5">Your file will not include:</h2>
+                <ul>
+                    <li>Direct messages</li>
+                    <li>Replies to your statuses</li>
+                    <li>Groups</li>
+                    <li>Favorites</li>
+                </ul>
+            </div>
+            {% endblocktrans %}
+        </div>
+        <p class="block">{% trans "In your new BookWyrm account can choose what to import: you will not have to import everything that is exported." %}</p>
+        <p class="notification is-warning">
+            {% spaceless %}
+            {% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set the account you are moving to as an <strong>alias</strong> of this one, or <strong>move</strong> this account to the new account, before you import your user data." %}
+            {% endspaceless %}
+        </p>
+    {% if next_available %}
+    <p class="notification is-warning">
+        {% blocktrans trimmed %}
+        You will be able to create a new export file at {{ next_available }}
+        {% endblocktrans %}
+    </p>
+    {% else %}
+    <form name="export" method="POST" href="{% url 'prefs-user-export' %}">
+        {% csrf_token %}
+        <button type="submit" class="button">
+            <span class="icon icon-download" aria-hidden="true"></span>
+            <span>{% trans "Create user export file" %}</span>
+        </button>
+    </form>
+    {% endif %}
+
+</div>
+<div class="content block">
+    <h2 class="title">{% trans "Recent Exports" %}</h2>
+    <p class="content">
+    {% trans "User export files will show 'complete' once ready. This may take a little while. Click the link to download your file." %}
+    </p>
+    <div class="table-container">
+        <table class="table is-striped is-fullwidth">
+            <tr>
+                <th>
+                    {% trans "Date" %}
+                </th>
+                <th>
+                    {% trans "Status" %}
+                </th>
+                <th colspan="2">
+                    {% trans "Size" %}
+                </th>
+            </tr>
+            {% if not jobs %}
+            <tr>
+                <td colspan="4">
+                    <em>{% trans "No recent imports" %}</em>
+                </td>
+            </tr>
+            {% endif %}
+            {% for job in jobs %}
+            <tr>
+                <td>{{ job.updated_date }}</td>
+                <td>
+                    <span
+                        {% if job.status == "stopped" or job.status == "failed" %}
+                        class="tag is-danger"
+                        {% elif job.status == "pending" %}
+                        class="tag is-warning"
+                        {% elif job.complete %}
+                        class="tag"
+                        {% else %}
+                        class="tag is-success"
+                        {% endif %}
+                    >
+                        {% if job.status %}
+                        {{ job.status }}
+                            {{ job.status_display }}
+                        {% elif job.complete %}
+                            {% trans "Complete" %}
+                        {% else %}
+                            {% trans "Active" %}
+                        {% endif %}
+                    </span>
+                </td>
+                <td>
+                    <span>{{ job.export_data|get_file_size }}</span>
+                </td>
+                <td>
+                    {% if job.complete and not job.status == "stopped" and not job.status == "failed" %}
+                    <p>
+                        <a download="" href="/preferences/user-export/{{ job.task_id }}">
+                            <span class="icon icon-download" aria-hidden="true"></span>
+                            <span class="is-hidden-mobile">
+                                {% trans "Download your export" %}
+                            </span>
+                        </a>
+                    </p>
+                    {% endif %}
+                </td>
+            </tr>
+            {% endfor %}
+        </table>
+    </div>
+
+    {% include 'snippets/pagination.html' with page=jobs path=request.path %}
+</div>
+{% endblock %}

bookwyrm/templates/preferences/export.html

@@ -1,16 +1,16 @@
 {% extends 'preferences/layout.html' %}
 {% load i18n %}
 
-{% block title %}{% trans "CSV Export" %}{% endblock %}
+{% block title %}{% trans "Export Book List" %}{% endblock %}
 
 {% block header %}
-{% trans "CSV Export" %}
+{% trans "Export Book List" %}
 {% endblock %}
 
 {% block panel %}
 <div class="block content">
     <p class="notification">
-        {% trans "Your export will include all the books on your shelves, books you have reviewed, and books with reading activity." %}
+        {% trans "Your CSV export file will include all the books on your shelves, books you have reviewed, and books with reading activity. <br/>Use this to import into a service like Goodreads." %}
     </p>
     <p>
         <form name="export" method="POST" href="{% url 'prefs-export' %}">

bookwyrm/templates/preferences/layout.html

@@ -40,11 +40,19 @@
         <ul class="menu-list">
             <li>
                 {% url 'import' as url %}
-                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Import" %}</a>
+                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Import Book List" %}</a>
             </li>
             <li>
                 {% url 'prefs-export' as url %}
-                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "CSV export" %}</a>
+                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Export Book List" %}</a>
+            </li>
+            <li>
+                {% url 'user-import' as url %}
+                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Import BookWyrm Account" %}</a>
+            </li>
+            <li>
+                {% url 'prefs-user-export' as url %}
+                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Export BookWyrm Account" %}</a>
             </li>
         </ul>
         <h2 class="menu-label">{% trans "Relationships" %}</h2>

bookwyrm/templates/settings/imports/complete_user_import_modal.html

@@ -0,0 +1,23 @@
+{% extends 'components/modal.html' %}
+{% load i18n %}
+
+{% block modal-title %}{% trans "Stop import?" %}{% endblock %}
+
+{% block modal-body %}
+{% trans "This action will stop the user import before it is complete and cannot be un-done" %}
+{% endblock %}
+
+{% block modal-footer %}
+<form name="complete-import-{{ import.id }}" action="{% url 'settings-user-import-complete' import.id %}" method="POST" class="is-flex-grow-1">
+    {% csrf_token %}
+    <input type="hidden" name="id" value="{{ import.id }}">
+    <div class="buttons is-right is-flex-grow-1">
+        <button type="button" class="button" data-modal-close>
+            {% trans "Cancel" %}
+        </button>
+        <button class="button is-danger" type="submit">
+            {% trans "Confirm" %}
+        </button>
+    </div>
+</form>
+{% endblock %}

bookwyrm/templates/settings/imports/imports.html

@@ -29,6 +29,7 @@
             <div class="notification">
                 {% trans "This is only intended to be used when things have gone very wrong with imports and you need to pause the feature while addressing issues." %}
                 {% trans "While imports are disabled, users will not be allowed to start new imports, but existing imports will not be affected." %}
+                {% trans "This setting prevents both book imports and user imports." %}
             </div>
             {% csrf_token %}
             <div class="control">
@@ -89,91 +90,215 @@
             </div>
         </form>
     </details>
+    <details class="details-panel box">
+        <summary>
+            <span role="heading" aria-level="2" class="title is-6">
+                {% trans "Limit how often users can import and export" %}
+            </span>
+            <span class="details-close icon icon-x" aria-hidden="true"></span>
+        </summary>
+        <form
+            name="user-imports-set-limit"
+            id="user-imports-set-limit"
+            method="POST"
+            action="{% url 'settings-user-imports-set-limit' %}"
+        >
+            <div class="notification">
+                {% trans "Some users might try to run user imports or exports very frequently, which you want to limit." %}
+                {% trans "Set the value to 0 to not enforce any limit." %}
+            </div>
+            <div class="align.to-t">
+                <label for="limit">{% trans "Restrict user imports and exports to once every " %}</label>
+                <input name="limit" class="input is-w-xs is-h-em" type="text" placeholder="0" value="{{ user_import_time_limit }}">
+                <label>{% trans "hours" %}</label>
+                {% csrf_token %}
+                <div class="control">
+                    <button type="submit" class="button is-warning">
+                        {% trans "Change limit" %}
+                    </button>
+                </div>
+            </div>
+        </form>
+    </details>
 </div>
 <div class="block">
-    <div class="tabs">
-        <ul>
-            {% url 'settings-imports' as url %}
-            <li {% if request.path in url %}class="is-active" aria-current="page"{% endif %}>
-                <a href="{{ url }}">{% trans "Active" %}</a>
-            </li>
-            {% url 'settings-imports' status="complete" as url %}
-            <li {% if url in request.path %}class="is-active" aria-current="page"{% endif %}>
-                <a href="{{ url }}">{% trans "Completed" %}</a>
-            </li>
-        </ul>
+    <h4 class="title is-4">{% trans "Book Imports" %}</h4>
+    <div class="block">
+        <div class="tabs">
+            <ul>
+                {% url 'settings-imports' as url %}
+                <li {% if request.path in url %}class="is-active" aria-current="page"{% endif %}>
+                    <a href="{{ url }}">{% trans "Active" %}</a>
+                </li>
+                {% url 'settings-imports' status="complete" as url %}
+                <li {% if url in request.path %}class="is-active" aria-current="page"{% endif %}>
+                    <a href="{{ url }}">{% trans "Completed" %}</a>
+                </li>
+            </ul>
+        </div>
     </div>
+
+    <div class="table-container block content">
+        <table class="table is-striped is-fullwidth">
+            <tr>
+                {% url 'settings-imports' status as url %}
+                <th>
+                    {% trans "ID" %}
+                </th>
+                <th>
+                    {% trans "User" as text %}
+                    {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %}
+                </th>
+                <th>
+                    {% trans "Date Created" as text %}
+                    {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %}
+                </th>
+                {% if status != "active" %}
+                <th>
+                    {% trans "Date Updated" %}
+                </th>
+                {% endif %}
+                <th>
+                    {% trans "Items" %}
+                </th>
+                <th>
+                    {% trans "Pending items" %}
+                </th>
+                <th>
+                    {% trans "Successful items" %}
+                </th>
+                <th>
+                    {% trans "Failed items" %}
+                </th>
+                {% if status == "active" %}
+                <th>{% trans "Actions" %}</th>
+                {% endif %}
+            </tr>
+            {% for import in imports %}
+            <tr>
+                <td>{{ import.id }}</td>
+                <td class="overflow-wrap-anywhere">
+                    <a href="{% url 'settings-user' import.user.id %}">{{ import.user|username }}</a>
+                </td>
+                <td>{{ import.created_date }}</td>
+                {% if status != "active" %}
+                <td>{{ import.updated_date }}</td>
+                {% endif %}
+                <td>{{ import.item_count|intcomma }}</td>
+                <td>{{ import.pending_item_count|intcomma }}</td>
+                <td>{{ import.successful_item_count|intcomma }}</td>
+                <td>{{ import.failed_item_count|intcomma }}</td>
+                {% if status == "active" %}
+                <td>
+                    {% join "complete" import.id as modal_id %}
+                    <button type="button" data-modal-open="{{ modal_id }}" class="button is-danger">{% trans "Stop import" %}</button>
+                    {% include "settings/imports/complete_import_modal.html" with id=modal_id %}
+                </td>
+                {% endif %}
+            </tr>
+            {% endfor %}
+            {% if not imports %}
+            <tr>
+                <td colspan="6">
+                    <em>{% trans "No matching imports found." %} </em>
+                </td>
+            </tr>
+            {% endif %}
+        </table>
+    </div>
+
+    {% include 'snippets/pagination.html' with page=imports path=request.path %}
+
 </div>
 
-<div class="table-container block content">
-    <table class="table is-striped is-fullwidth">
-        <tr>
-            {% url 'settings-imports' status  as url %}
-            <th>
-                {% trans "ID" %}
-            </th>
-            <th>
-                {% trans "User" as text %}
-                {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %}
-            </th>
-            <th>
-                {% trans "Date Created" as text %}
-                {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %}
-            </th>
-            {% if status != "active" %}
-            <th>
-                {% trans "Date Updated" %}
-            </th>
-            {% endif %}
-            <th>
-                {% trans "Items" %}
-            </th>
-            <th>
-                {% trans "Pending items" %}
-            </th>
-            <th>
-                {% trans "Successful items" %}
-            </th>
-            <th>
-                {% trans "Failed items" %}
-            </th>
-            {% if status == "active" %}
-            <th>{% trans "Actions" %}</th>
-            {% endif %}
-        </tr>
-        {% for import in imports %}
-        <tr>
-            <td>{{ import.id }}</td>
-            <td class="overflow-wrap-anywhere">
-                <a href="{% url 'settings-user' import.user.id %}">{{ import.user|username }}</a>
-            </td>
-            <td>{{ import.created_date }}</td>
-            {% if status != "active" %}
-            <td>{{ import.updated_date }}</td>
-            {% endif %}
-            <td>{{ import.item_count|intcomma }}</td>
-            <td>{{ import.pending_item_count|intcomma }}</td>
-            <td>{{ import.successful_item_count|intcomma }}</td>
-            <td>{{ import.failed_item_count|intcomma }}</td>
-            {% if status == "active" %}
-            <td>
-                {% join "complete" import.id as modal_id %}
-                <button type="button" data-modal-open="{{ modal_id }}" class="button is-danger">{% trans "Stop import" %}</button>
-                {% include "settings/imports/complete_import_modal.html" with id=modal_id %}
-            </td>
-            {% endif %}
-        </tr>
-        {% endfor %}
-        {% if not imports %}
-        <tr>
-            <td colspan="6">
-                <em>{% trans "No matching imports found." %} </em>
-            </td>
-        </tr>
-        {% endif %}
-    </table>
-</div>
+<div class="block">
+    <h4 class="title is-4">{% trans "User Imports" %}</h4>
+    <div class="block">
+        <div class="tabs">
+            <ul>
+                {% url 'settings-imports' as url %}
+                <li {% if request.path in url %}class="is-active" aria-current="page"{% endif %}>
+                    <a href="{{ url }}">{% trans "Active" %}</a>
+                </li>
+                {% url 'settings-imports' status="complete" as url %}
+                <li {% if url in request.path %}class="is-active" aria-current="page"{% endif %}>
+                    <a href="{{ url }}">{% trans "Completed" %}</a>
+                </li>
+            </ul>
+        </div>
+    </div>
 
-{% include 'snippets/pagination.html' with page=imports path=request.path %}
+    <div class="table-container block content">
+        <table class="table is-striped is-fullwidth">
+            <tr>
+                {% url 'settings-imports' status as url %}
+                <th>
+                    {% trans "ID" %}
+                </th>
+                <th>
+                    {% trans "User" as text %}
+                    {% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %}
+                </th>
+                <th>
+                    {% trans "Date Created" as text %}
+                    {% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %}
+                </th>
+                {% if status != "active" %}
+                <th>
+                    {% trans "Date Updated" %}
+                </th>
+                {% endif %}
+
+                {% if status == "active" %}
+                <th>{% trans "Actions" %}</th>
+                {% else %}
+                <th>{% trans "Status" %}</th>
+                {% endif %}
+            </tr>
+            {% for import in user_imports %}
+            <tr>
+                <td>{{ import.id }}</td>
+                <td class="overflow-wrap-anywhere">
+                    <a href="{% url 'settings-user' import.user.id %}">{{ import.user|username }}</a>
+                </td>
+                <td>{{ import.created_date }}</td>
+                {% if status != "active" %}
+                <td>{{ import.updated_date }}</td>
+                {% endif %}
+                {% if status == "active" %}
+                <td>
+                    {% join "complete" import.id as modal_id %}
+                    <button type="button" data-modal-open="{{ modal_id }}" class="button is-danger">{% trans "Stop import" %}</button>
+                    {% include "settings/imports/complete_user_import_modal.html" with id=modal_id %}
+                </td>
+                {% else %}
+                <td>
+                    <span
+                        {% if import.status == "stopped" or import.status == "failed" %}
+                        class="tag is-danger"
+                        {% elif import.status == "pending" %}
+                        class="tag is-warning"
+                        {% elif import.complete %}
+                        class="tag"
+                        {% else %}
+                        class="tag is-success"
+                        {% endif %}
+                        >{{ import.status }}
+                    </span>
+                </td>
+                {% endif %}
+            </tr>
+            {% endfor %}
+            {% if not user_imports %}
+            <tr>
+                <td colspan="6">
+                    <em>{% trans "No matching imports found." %} </em>
+                </td>
+            </tr>
+            {% endif %}
+        </table>
+    </div>
+
+    {% include 'snippets/pagination.html' with page=user_imports path=request.path %}
+</div>
 {% endblock %}
-

bookwyrm/templatetags/utilities.py

@@ -128,6 +128,23 @@ def id_to_username(user_id):
     return value
 
 
+@register.filter(name="get_file_size")
+def get_file_size(file):
+    """display the size of a file in human readable terms"""
+
+    try:
+        raw_size = os.stat(file.path).st_size
+        if raw_size < 1024:
+            return f"{raw_size} bytes"
+        if raw_size < 1024**2:
+            return f"{raw_size/1024:.2f} KB"
+        if raw_size < 1024**3:
+            return f"{raw_size/1024**2:.2f} MB"
+        return f"{raw_size/1024**3:.2f} GB"
+    except Exception:  # pylint: disable=broad-except
+        return ""
+
+
 @register.filter(name="get_user_permission")
 def get_user_permission(user):
     """given a user, return their permission level"""

bookwyrm/tests/data/user_import.json

@@ -0,0 +1,399 @@
+{
+    "id": "https://www.example.com/user/rat",
+    "type": "Person",
+    "preferredUsername": "rat",
+    "inbox": "https://www.example.com/user/rat/inbox",
+    "publicKey": {
+        "id": "https://www.example.com/user/rat/#main-key",
+        "owner": "https://www.example.com/user/rat",
+        "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nzzzz\n-----END PUBLIC KEY-----"
+    },
+    "followers": "https://www.example.com/user/rat/followers",
+    "following": "https://www.example.com/user/rat/following",
+    "outbox": "https://www.example.com/user/rat/outbox",
+    "endpoints": {
+        "sharedInbox": "https://www.example.com/inbox"
+    },
+    "name": "Rat",
+    "summary": "<p>I love to make soup in Paris and eat pizza in New York</p>",
+    "icon": {
+        "type": "Document",
+        "url": "avatar.png",
+        "name": "avatar for rat",
+        "@context": "https://www.w3.org/ns/activitystreams"
+    },
+    "bookwyrmUser": true,
+    "manuallyApprovesFollowers": true,
+    "discoverable": false,
+    "hideFollows": true,
+    "alsoKnownAs": [],
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {
+            "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+            "schema": "http://schema.org#",
+            "PropertyValue": "schema:PropertyValue",
+            "value": "schema:value",
+            "alsoKnownAs": {
+                "@id": "as:alsoKnownAs",
+                "@type": "@id"
+            },
+            "movedTo": {
+                "@id": "as:movedTo",
+                "@type": "@id"
+            }
+        }
+    ],
+    "settings": {
+        "show_goal": false,
+        "preferred_timezone": "Australia/Adelaide",
+        "default_post_privacy": "followers",
+        "show_suggested_users": false
+    },
+    "goals": [
+        {
+            "goal": 12,
+            "year": 2023,
+            "privacy": "followers"
+        }
+    ],
+    "books": [
+        {
+            "work": {
+                "id": "https://www.example.com/book/1",
+                "type": "Work",
+                "title": "Seeing Like a State",
+                "description": "<p>Examines how (sometimes quasi-) authoritarian high-modernist planning  fails to deliver the goods, be they increased resources for the state or a better life for the people.</p>",
+                "languages": [ "English" ],
+                "series": "",
+                "seriesNumber": "",
+                "subjects": [],
+                "subjectPlaces": [],
+                "authors": [
+                    "https://www.example.com/author/1"
+                ],
+                "firstPublishedDate": "",
+                "publishedDate": "1998-03-30T00:00:00Z",
+                "fileLinks": [],
+                "lccn": "",
+                "editions": [
+                    "https://www.example.com/book/2"
+                ],
+                "@context": "https://www.w3.org/ns/activitystreams"
+            },
+            "edition": {
+                "id": "https://www.example.com/book/2",
+                "type": "Edition",
+                "openlibraryKey": "OL680025M",
+                "title": "Seeking Like A State",
+                "sortTitle": "seeing like a state",
+                "subtitle": "",
+                "description": "<p>Examines how (sometimes quasi-) authoritarian high-modernist planning  fails to deliver the goods, be they increased resources for the state or a better life for the people.</p>",
+                "languages": ["English"],
+                "series": "",
+                "seriesNumber": "",
+                "subjects": [],
+                "subjectPlaces": [],
+                "authors": [
+                    "https://www.example.com/author/1"
+                ],
+                "firstPublishedDate": "",
+                "publishedDate": "",
+                "fileLinks": [],
+                "cover": {
+                    "type": "Document",
+                    "url": "covers/d273d638-191d-4ebf-b213-3c60dbf010fe.jpeg",
+                    "name": "James C. Scott: Seeing like a state",
+                    "@context": "https://www.w3.org/ns/activitystreams"
+                },
+                "work": "https://www.example.com/book/1",
+                "isbn10": "",
+                "isbn13": "9780300070163",
+                "oclcNumber": "",
+                "physicalFormat": "",
+                "physicalFormatDetail": "",
+                "publishers": [],
+                "editionRank": 4,
+                "@context": "https://www.w3.org/ns/activitystreams"
+            },
+            "authors": [
+                {
+                    "id": "https://www.example.com/author/1",
+                    "type": "Author",
+                    "name": "James C. Scott",
+                    "aliases": [
+                        "James Campbell Scott",
+                        "\u30b8\u30a7\u30fc\u30e0\u30ba\u30fbC. \u30b9\u30b3\u30c3\u30c8",
+                        "\u30b8\u30a7\u30fc\u30e0\u30ba\u30fbC\u30fb\u30b9\u30b3\u30c3\u30c8",
+                        "\u062c\u06cc\u0645\u0632 \u0633\u06cc. \u0627\u0633\u06a9\u0627\u062a",
+                        "Jim Scott",
+                        "\u062c\u064a\u0645\u0633 \u0633\u0643\u0648\u062a",
+                        "James C. Scott",
+                        "\u0414\u0436\u0435\u0439\u043c\u0441 \u0421\u043a\u043e\u0442\u0442",
+                        "\u30b8\u30a7\u30fc\u30e0\u30b9\u30fbC \u30b9\u30b3\u30c3\u30c8",
+                        "James Cameron Scott"
+                    ],
+                    "bio": "<p>American political scientist and anthropologist</p>",
+                    "wikipediaLink": "https://en.wikipedia.org/wiki/James_C._Scott",
+                    "website": "",
+                    "@context": "https://www.w3.org/ns/activitystreams"
+                }
+            ],
+            "shelves": [
+                {
+                    "id": "https://www.example.com/user/rat/books/read",
+                    "type": "Shelf",
+                    "totalItems": 1,
+                    "first": "https://www.example.com/user/rat/books/read?page=1",
+                    "last": "https://www.example.com/user/rat/books/read?page=1",
+                    "name": "Read",
+                    "owner": "https://www.example.com/user/rat",
+                    "to": [
+                        "https://www.w3.org/ns/activitystreams#Public"
+                    ],
+                    "cc": [
+                        "https://www.example.com/user/rat/followers"
+                    ],
+                    "@context": "https://www.w3.org/ns/activitystreams"
+                },
+                {
+                    "id": "https://www.example.com/user/rat/books/to-read",
+                    "type": "Shelf",
+                    "totalItems": 1,
+                    "first": "https://www.example.com/user/rat/books/to-read?page=1",
+                    "last": "https://www.example.com/user/rat/books/to-read?page=1",
+                    "name": "To Read",
+                    "owner": "https://www.example.com/user/rat",
+                    "to": [
+                        "https://www.w3.org/ns/activitystreams#Public"
+                    ],
+                    "cc": [
+                        "https://www.example.com/user/rat/followers"
+                    ],
+                    "@context": "https://www.w3.org/ns/activitystreams"
+                }
+            ],
+            "lists": [
+                {
+                    "id": "https://www.example.com/list/2",
+                    "type": "BookList",
+                    "totalItems": 1,
+                    "first": "https://www.example.com/list/2?page=1",
+                    "last": "https://www.example.com/list/2?page=1",
+                    "name": "my list of books",
+                    "owner": "https://www.example.com/user/rat",
+                    "to": [
+                        "https://www.example.com/user/rat/followers"
+                    ],
+                    "cc": [],
+                    "summary": "Here is a description of my list",
+                    "curation": "closed",
+                    "@context": "https://www.w3.org/ns/activitystreams",
+                    "privacy": "followers",
+                    "list_item": {
+                        "id": "https://www.example.com/user/rat/listitem/3",
+                        "type": "ListItem",
+                        "actor": "https://www.example.com/user/rat",
+                        "book": "https://www.example.com/book/2",
+                        "notes": "<p>It's fun.</p>",
+                        "approved": true,
+                        "order": 1,
+                        "@context": "https://www.w3.org/ns/activitystreams"
+                    }
+                }
+            ],
+            "comments": [],
+            "quotations": [],
+            "reviews": [
+                {
+                    "id": "https://www.example.com/user/rat/review/7",
+                    "type": "Review",
+                    "published": "2023-08-14T04:09:18.343+00:00",
+                    "attributedTo": "https://www.example.com//user/rat",
+                    "content": "<p>I like it</p>",
+                    "to": [
+                        "https://your.domain.here/user/rat/followers"
+                    ],
+                    "cc": [],
+                    "replies": {
+                        "id": "https://www.example.com/user/rat/review/7/replies",
+                        "type": "OrderedCollection",
+                        "totalItems": 0,
+                        "first": "https://www.example.com/user/rat/review/7/replies?page=1",
+                        "last": "https://www.example.com/user/rat/review/7/replies?page=1",
+                        "@context": "https://www.w3.org/ns/activitystreams"
+                    },
+                    "summary": "Here's a spoiler alert",
+                    "tag": [],
+                    "attachment": [],
+                    "sensitive": true,
+                    "inReplyToBook": "https://www.example.com/book/6",
+                    "name": "great book",
+                    "rating": 5.0,
+                    "@context": "https://www.w3.org/ns/activitystreams",
+                    "progress": 23,
+                    "progress_mode": "PG"
+                }
+            ],
+            "readthroughs": [
+                {
+                    "id": 1,
+                    "created_date": "2023-08-14T04:00:27.544Z",
+                    "updated_date": "2023-08-14T04:00:27.546Z",
+                    "remote_id": "https://www.example.com/user/rat/readthrough/1",
+                    "user_id": 1,
+                    "book_id": 4880,
+                    "progress": null,
+                    "progress_mode": "PG",
+                    "start_date": "2018-01-01T00:00:00Z",
+                    "finish_date": "2023-08-13T00:00:00Z",
+                    "stopped_date": null,
+                    "is_active": false
+                }
+            ]
+        },
+        {
+            "work": {
+                "id": "https://www.example.com/book/3",
+                "type": "Work",
+                "title": "Sand Talk: How Indigenous Thinking Can Save the World",
+                "description": "",
+                "languages": [],
+                "series": "",
+                "seriesNumber": "",
+                "subjects": [],
+                "subjectPlaces": [],
+                "authors": [
+                    "https://www.example.com/author/2"
+                ],
+                "firstPublishedDate": "",
+                "publishedDate": "",
+                "fileLinks": [],
+                "lccn": "",
+                "openlibraryKey": "OL28216445M",
+                "editions": [
+                    "https://www.example.com/book/4"
+                ],
+                "@context": "https://www.w3.org/ns/activitystreams"
+            },
+            "edition": {
+                "id": "https://www.example.com/book/4",
+                "type": "Edition",
+                "title": "Sand Talk",
+                "sortTitle": "sand talk",
+                "subtitle": "How Indigenous Thinking Can Save the World",
+                "description": "",
+                "languages": [],
+                "series": "",
+                "seriesNumber": "",
+                "subjects": [],
+                "subjectPlaces": [],
+                "authors": [
+                    "https://www.example.com/author/2"
+                ],
+                "firstPublishedDate": "",
+                "publishedDate": "",
+                "fileLinks": [],
+                "cover": {
+                    "type": "Document",
+                    "url": "covers/6a553a08-2641-42a1-baa4-960df9edbbfc.jpeg",
+                    "name": "Tyson Yunkaporta - Sand Talk",
+                    "@context": "https://www.w3.org/ns/activitystreams"
+                },
+                "work": "https://www.example.com/book/3",
+                "isbn10": "",
+                "isbn13": "9780062975645",
+                "oclcNumber": "",
+                "inventaireId": "isbn:9780062975645",
+                "physicalFormat": "paperback",
+                "physicalFormatDetail": "",
+                "publishers": [],
+                "editionRank": 5,
+                "@context": "https://www.w3.org/ns/activitystreams"
+            },
+            "authors": [
+                {
+                    "id": "https://www.example.com/author/2",
+                    "type": "Author",
+                    "name": "Tyson Yunkaporta",
+                    "aliases": [],
+                    "bio": "",
+                    "wikipediaLink": "",
+                    "website": "",
+                    "@context": "https://www.w3.org/ns/activitystreams"
+                }
+            ],
+            "shelves": [],
+            "lists": [],
+            "comments": [
+                {
+                    "id": "https://www.example.com/user/rat/comment/4",
+                    "type": "Comment",
+                    "published": "2023-08-14T04:48:18.746+00:00",
+                    "attributedTo": "https://www.example.com/user/rat",
+                    "content": "<p>this is a comment about an amazing book</p>",
+                    "to": [
+                        "https://www.w3.org/ns/activitystreams#Public"
+                    ],
+                    "cc": [
+                        "https://www.example.com/user/rat/followers"
+                    ],
+                    "replies": {
+                        "id": "https://www.example.com/user/rat/comment/4/replies",
+                        "type": "OrderedCollection",
+                        "totalItems": 0,
+                        "first": "https://www.example.com/user/rat/comment/4/replies?page=1",
+                        "last": "https://www.example.com/user/rat/comment/4/replies?page=1",
+                        "@context": "https://www.w3.org/ns/activitystreams"
+                    },
+                    "tag": [],
+                    "attachment": [],
+                    "sensitive": false,
+                    "inReplyToBook": "https://www.example.com/book/4",
+                    "readingStatus": null,
+                    "@context": "https://www.w3.org/ns/activitystreams"
+                }
+            ],
+            "quotations": [
+                {
+                    "id": "https://www.example.com/user/rat/quotation/2",
+                    "type": "Quotation",
+                    "published": "2023-11-12T04:29:38.370305+00:00",
+                    "attributedTo": "https://www.example.com/user/rat",
+                    "content": "<p>not actually from this book lol</p>",
+                    "to": [
+                        "https://www.w3.org/ns/activitystreams#Public"
+                    ],
+                    "cc": [
+                        "https://www.example.com/user/rat/followers"
+                    ],
+                    "replies": {
+                        "id": "https://www.example.com/user/rat/quotation/2/replies",
+                        "type": "OrderedCollection",
+                        "totalItems": 0,
+                        "first": "https://www.example.com/user/rat/quotation/2/replies?page=1",
+                        "last": "https://www.example.com/user/rat/quotation/2/replies?page=1",
+                        "@context": "https://www.w3.org/ns/activitystreams"
+                    },
+                    "tag": [],
+                    "attachment": [],
+                    "sensitive": false,
+                    "summary": "spoiler ahead!",
+                    "inReplyToBook": "https://www.example.com/book/2",
+                    "quote": "<p>To be or not to be</p>",
+                    "@context": "https://www.w3.org/ns/activitystreams"
+                }
+            ],
+            "reviews": [],
+            "readthroughs": []
+        }
+    ],
+    "saved_lists": [
+        "https://local.lists/9999"
+    ],
+    "follows": [
+        "https://your.domain.here/user/rat"
+    ],
+    "blocks": ["https://your.domain.here/user/badger"]
+}

bookwyrm/tests/migrations/test_0184.py

@@ -1,121 +0,0 @@
-""" testing migrations """
-from unittest.mock import patch
-
-from django.test import TestCase
-from django.db.migrations.executor import MigrationExecutor
-from django.db import connection
-
-from bookwyrm import models
-from bookwyrm.management.commands import initdb
-from bookwyrm.settings import DOMAIN
-
-# pylint: disable=missing-class-docstring
-# pylint: disable=missing-function-docstring
-class EraseDeletedUserDataMigration(TestCase):
-
-    migrate_from = "0183_auto_20231105_1607"
-    migrate_to = "0184_auto_20231106_0421"
-
-    # pylint: disable=invalid-name
-    def setUp(self):
-        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
-            "bookwyrm.activitystreams.populate_stream_task.delay"
-        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
-            self.active_user = models.User.objects.create_user(
-                f"activeuser@{DOMAIN}",
-                "activeuser@activeuser.activeuser",
-                "activeuserword",
-                local=True,
-                localname="active",
-                name="a name",
-            )
-            self.inactive_user = models.User.objects.create_user(
-                f"inactiveuser@{DOMAIN}",
-                "inactiveuser@inactiveuser.inactiveuser",
-                "inactiveuserword",
-                local=True,
-                localname="inactive",
-                is_active=False,
-                deactivation_reason="self_deactivation",
-                name="name name",
-            )
-            self.deleted_user = models.User.objects.create_user(
-                f"deleteduser@{DOMAIN}",
-                "deleteduser@deleteduser.deleteduser",
-                "deleteduserword",
-                local=True,
-                localname="deleted",
-                is_active=False,
-                deactivation_reason="self_deletion",
-                name="cool name",
-            )
-        with patch(
-            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
-        ), patch("bookwyrm.activitystreams.add_status_task.delay"):
-            self.active_status = models.Status.objects.create(
-                user=self.active_user, content="don't delete me"
-            )
-            self.inactive_status = models.Status.objects.create(
-                user=self.inactive_user, content="also don't delete me"
-            )
-            self.deleted_status = models.Status.objects.create(
-                user=self.deleted_user, content="yes, delete me"
-            )
-
-        initdb.init_groups()
-        initdb.init_permissions()
-
-        self.migrate_from = [("bookwyrm", self.migrate_from)]
-        self.migrate_to = [("bookwyrm", self.migrate_to)]
-        executor = MigrationExecutor(connection)
-        old_apps = executor.loader.project_state(self.migrate_from).apps
-
-        # Reverse to the original migration
-        executor.migrate(self.migrate_from)
-
-        self.setUpBeforeMigration(old_apps)
-
-        # Run the migration to test
-        executor = MigrationExecutor(connection)
-        executor.loader.build_graph()  # reload.
-        with patch("bookwyrm.activitystreams.remove_status_task.delay"):
-            executor.migrate(self.migrate_to)
-
-        self.apps = executor.loader.project_state(self.migrate_to).apps
-
-    def setUpBeforeMigration(self, apps):
-        pass
-
-    def test_user_data_deleted(self):
-        """Make sure that only the right data was deleted"""
-        self.active_user.refresh_from_db()
-        self.inactive_user.refresh_from_db()
-        self.deleted_user.refresh_from_db()
-        self.active_status.refresh_from_db()
-        self.inactive_status.refresh_from_db()
-        self.deleted_status.refresh_from_db()
-
-        self.assertTrue(self.active_user.is_active)
-        self.assertFalse(self.active_user.is_deleted)
-        self.assertEqual(self.active_user.name, "a name")
-        self.assertNotEqual(self.deleted_user.email, "activeuser@activeuser.activeuser")
-        self.assertFalse(self.active_status.deleted)
-        self.assertEqual(self.active_status.content, "don't delete me")
-
-        self.assertFalse(self.inactive_user.is_active)
-        self.assertFalse(self.inactive_user.is_deleted)
-        self.assertEqual(self.inactive_user.name, "name name")
-        self.assertNotEqual(
-            self.deleted_user.email, "inactiveuser@inactiveuser.inactiveuser"
-        )
-        self.assertFalse(self.inactive_status.deleted)
-        self.assertEqual(self.inactive_status.content, "also don't delete me")
-
-        self.assertFalse(self.deleted_user.is_active)
-        self.assertTrue(self.deleted_user.is_deleted)
-        self.assertIsNone(self.deleted_user.name)
-        self.assertNotEqual(
-            self.deleted_user.email, "deleteduser@deleteduser.deleteduser"
-        )
-        self.assertTrue(self.deleted_status.deleted)
-        self.assertIsNone(self.deleted_status.content)

bookwyrm/tests/models/test_bookwyrm_export_job.py

@@ -0,0 +1,231 @@
+"""test bookwyrm user export functions"""
+import datetime
+import json
+from unittest.mock import patch
+
+from django.core.serializers.json import DjangoJSONEncoder
+from django.test import TestCase
+from django.utils import timezone
+
+from bookwyrm import models
+import bookwyrm.models.bookwyrm_export_job as export_job
+
+
+class BookwyrmExport(TestCase):
+    """testing user export functions"""
+
+    def setUp(self):
+        """lots of stuff to set up for a user export"""
+        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
+            "bookwyrm.activitystreams.populate_stream_task.delay"
+        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"), patch(
+            "bookwyrm.suggested_users.rerank_user_task.delay"
+        ), patch(
+            "bookwyrm.lists_stream.remove_list_task.delay"
+        ), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ), patch(
+            "bookwyrm.activitystreams.add_book_statuses_task"
+        ):
+
+            self.local_user = models.User.objects.create_user(
+                "mouse",
+                "mouse@mouse.mouse",
+                "password",
+                local=True,
+                localname="mouse",
+                name="Mouse",
+                summary="I'm a real bookmouse",
+                manually_approves_followers=False,
+                hide_follows=False,
+                show_goal=False,
+                show_suggested_users=False,
+                discoverable=True,
+                preferred_timezone="America/Los Angeles",
+                default_post_privacy="followers",
+            )
+
+            self.rat_user = models.User.objects.create_user(
+                "rat", "rat@rat.rat", "ratword", local=True, localname="rat"
+            )
+
+            self.badger_user = models.User.objects.create_user(
+                "badger",
+                "badger@badger.badger",
+                "badgerword",
+                local=True,
+                localname="badger",
+            )
+
+            models.AnnualGoal.objects.create(
+                user=self.local_user,
+                year=timezone.now().year,
+                goal=128937123,
+                privacy="followers",
+            )
+
+            self.list = models.List.objects.create(
+                name="My excellent list",
+                user=self.local_user,
+                remote_id="https://local.lists/1111",
+            )
+
+            self.saved_list = models.List.objects.create(
+                name="My cool list",
+                user=self.rat_user,
+                remote_id="https://local.lists/9999",
+            )
+
+            self.local_user.saved_lists.add(self.saved_list)
+            self.local_user.blocks.add(self.badger_user)
+            self.rat_user.followers.add(self.local_user)
+
+            # book, edition, author
+            self.author = models.Author.objects.create(name="Sam Zhu")
+            self.work = models.Work.objects.create(
+                title="Example Work", remote_id="https://example.com/book/1"
+            )
+            self.edition = models.Edition.objects.create(
+                title="Example Edition", parent_work=self.work
+            )
+
+            self.edition.authors.add(self.author)
+
+            # readthrough
+            self.readthrough_start = timezone.now()
+            finish = self.readthrough_start + datetime.timedelta(days=1)
+            models.ReadThrough.objects.create(
+                user=self.local_user,
+                book=self.edition,
+                start_date=self.readthrough_start,
+                finish_date=finish,
+            )
+
+            # shelve
+            read_shelf = models.Shelf.objects.get(
+                user=self.local_user, identifier="read"
+            )
+            models.ShelfBook.objects.create(
+                book=self.edition, shelf=read_shelf, user=self.local_user
+            )
+
+            # add to list
+            models.ListItem.objects.create(
+                book_list=self.list,
+                user=self.local_user,
+                book=self.edition,
+                approved=True,
+                order=1,
+            )
+
+            # review
+            models.Review.objects.create(
+                content="awesome",
+                name="my review",
+                rating=5,
+                user=self.local_user,
+                book=self.edition,
+            )
+            # comment
+            models.Comment.objects.create(
+                content="ok so far",
+                user=self.local_user,
+                book=self.edition,
+                progress=15,
+            )
+            # quote
+            models.Quotation.objects.create(
+                content="check this out",
+                quote="A rose by any other name",
+                user=self.local_user,
+                book=self.edition,
+            )
+
+    def test_json_export_user_settings(self):
+        """Test the json export function for basic user info"""
+        data = export_job.json_export(self.local_user)
+        user_data = json.loads(data)
+        self.assertEqual(user_data["preferredUsername"], "mouse")
+        self.assertEqual(user_data["name"], "Mouse")
+        self.assertEqual(user_data["summary"], "<p>I'm a real bookmouse</p>")
+        self.assertEqual(user_data["manuallyApprovesFollowers"], False)
+        self.assertEqual(user_data["hideFollows"], False)
+        self.assertEqual(user_data["discoverable"], True)
+        self.assertEqual(user_data["settings"]["show_goal"], False)
+        self.assertEqual(user_data["settings"]["show_suggested_users"], False)
+        self.assertEqual(
+            user_data["settings"]["preferred_timezone"], "America/Los Angeles"
+        )
+        self.assertEqual(user_data["settings"]["default_post_privacy"], "followers")
+
+    def test_json_export_extended_user_data(self):
+        """Test the json export function for other non-book user info"""
+        data = export_job.json_export(self.local_user)
+        json_data = json.loads(data)
+
+        # goal
+        self.assertEqual(len(json_data["goals"]), 1)
+        self.assertEqual(json_data["goals"][0]["goal"], 128937123)
+        self.assertEqual(json_data["goals"][0]["year"], timezone.now().year)
+        self.assertEqual(json_data["goals"][0]["privacy"], "followers")
+
+        # saved lists
+        self.assertEqual(len(json_data["saved_lists"]), 1)
+        self.assertEqual(json_data["saved_lists"][0], "https://local.lists/9999")
+
+        # follows
+        self.assertEqual(len(json_data["follows"]), 1)
+        self.assertEqual(json_data["follows"][0], "https://your.domain.here/user/rat")
+        # blocked users
+        self.assertEqual(len(json_data["blocks"]), 1)
+        self.assertEqual(json_data["blocks"][0], "https://your.domain.here/user/badger")
+
+    def test_json_export_books(self):
+        """Test the json export function for extended user info"""
+
+        data = export_job.json_export(self.local_user)
+        json_data = json.loads(data)
+        start_date = json_data["books"][0]["readthroughs"][0]["start_date"]
+
+        self.assertEqual(len(json_data["books"]), 1)
+        self.assertEqual(json_data["books"][0]["edition"]["title"], "Example Edition")
+        self.assertEqual(len(json_data["books"][0]["authors"]), 1)
+        self.assertEqual(json_data["books"][0]["authors"][0]["name"], "Sam Zhu")
+
+        self.assertEqual(
+            f'"{start_date}"', DjangoJSONEncoder().encode(self.readthrough_start)
+        )
+
+        self.assertEqual(json_data["books"][0]["shelves"][0]["name"], "Read")
+
+        self.assertEqual(len(json_data["books"][0]["lists"]), 1)
+        self.assertEqual(json_data["books"][0]["lists"][0]["name"], "My excellent list")
+        self.assertEqual(
+            json_data["books"][0]["lists"][0]["list_item"]["book"],
+            self.edition.remote_id,
+            self.edition.id,
+        )
+
+        self.assertEqual(len(json_data["books"][0]["reviews"]), 1)
+        self.assertEqual(len(json_data["books"][0]["comments"]), 1)
+        self.assertEqual(len(json_data["books"][0]["quotations"]), 1)
+
+        self.assertEqual(json_data["books"][0]["reviews"][0]["name"], "my review")
+        self.assertEqual(
+            json_data["books"][0]["reviews"][0]["content"], "<p>awesome</p>"
+        )
+        self.assertEqual(json_data["books"][0]["reviews"][0]["rating"], 5.0)
+
+        self.assertEqual(
+            json_data["books"][0]["comments"][0]["content"], "<p>ok so far</p>"
+        )
+        self.assertEqual(json_data["books"][0]["comments"][0]["progress"], 15)
+        self.assertEqual(json_data["books"][0]["comments"][0]["progress_mode"], "PG")
+
+        self.assertEqual(
+            json_data["books"][0]["quotations"][0]["content"], "<p>check this out</p>"
+        )
+        self.assertEqual(
+            json_data["books"][0]["quotations"][0]["quote"],
+            "<p>A rose by any other name</p>",
+        )

bookwyrm/tests/models/test_bookwyrm_import_job.py

@@ -0,0 +1,545 @@
+""" testing models """
+
+import json
+import pathlib
+from unittest.mock import patch
+
+from django.db.models import Q
+from django.utils.dateparse import parse_datetime
+from django.test import TestCase
+
+from bookwyrm import models
+from bookwyrm.utils.tar import BookwyrmTarFile
+from bookwyrm.models import bookwyrm_import_job
+
+
+class BookwyrmImport(TestCase):  # pylint: disable=too-many-public-methods
+    """testing user import functions"""
+
+    def setUp(self):
+        """setting stuff up"""
+        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
+            "bookwyrm.activitystreams.populate_stream_task.delay"
+        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"), patch(
+            "bookwyrm.suggested_users.rerank_user_task.delay"
+        ):
+
+            self.local_user = models.User.objects.create_user(
+                "mouse",
+                "mouse@mouse.mouse",
+                "password",
+                local=True,
+                localname="mouse",
+                name="Mouse",
+                summary="I'm a real bookmouse",
+                manually_approves_followers=False,
+                hide_follows=False,
+                show_goal=True,
+                show_suggested_users=True,
+                discoverable=True,
+                preferred_timezone="America/Los Angeles",
+                default_post_privacy="public",
+            )
+
+            self.rat_user = models.User.objects.create_user(
+                "rat", "rat@rat.rat", "password", local=True, localname="rat"
+            )
+
+            self.badger_user = models.User.objects.create_user(
+                "badger",
+                "badger@badger.badger",
+                "password",
+                local=True,
+                localname="badger",
+            )
+
+            self.work = models.Work.objects.create(title="Sand Talk")
+
+            self.book = models.Edition.objects.create(
+                title="Sand Talk",
+                remote_id="https://example.com/book/1234",
+                openlibrary_key="OL28216445M",
+                inventaire_id="isbn:9780062975645",
+                isbn_13="9780062975645",
+                parent_work=self.work,
+            )
+
+        self.json_file = pathlib.Path(__file__).parent.joinpath(
+            "../data/user_import.json"
+        )
+
+        with open(self.json_file, "r", encoding="utf-8") as jsonfile:
+            self.json_data = json.loads(jsonfile.read())
+
+        self.archive_file = pathlib.Path(__file__).parent.joinpath(
+            "../data/bookwyrm_account_export.tar.gz"
+        )
+
+    def test_update_user_profile(self):
+        """Test update the user's profile from import data"""
+
+        with patch("bookwyrm.suggested_users.remove_user_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ), patch("bookwyrm.suggested_users.rerank_user_task.delay"):
+
+            with open(self.archive_file, "rb") as fileobj:
+                with BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile:
+
+                    models.bookwyrm_import_job.update_user_profile(
+                        self.local_user, tarfile, self.json_data
+                    )
+
+            self.local_user.refresh_from_db()
+
+            self.assertEqual(
+                self.local_user.username, "mouse"
+            )  # username should not change
+            self.assertEqual(self.local_user.name, "Rat")
+            self.assertEqual(
+                self.local_user.summary,
+                "I love to make soup in Paris and eat pizza in New York",
+            )
+
+    def test_update_user_settings(self):
+        """Test updating the user's settings from import data"""
+
+        with patch("bookwyrm.suggested_users.remove_user_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ), patch("bookwyrm.suggested_users.rerank_user_task.delay"):
+
+            models.bookwyrm_import_job.update_user_settings(
+                self.local_user, self.json_data
+            )
+            self.local_user.refresh_from_db()
+
+            self.assertEqual(self.local_user.manually_approves_followers, True)
+            self.assertEqual(self.local_user.hide_follows, True)
+            self.assertEqual(self.local_user.show_goal, False)
+            self.assertEqual(self.local_user.show_suggested_users, False)
+            self.assertEqual(self.local_user.discoverable, False)
+            self.assertEqual(self.local_user.preferred_timezone, "Australia/Adelaide")
+            self.assertEqual(self.local_user.default_post_privacy, "followers")
+
+    def test_update_goals(self):
+        """Test update the user's goals from import data"""
+
+        models.AnnualGoal.objects.create(
+            user=self.local_user,
+            year=2023,
+            goal=999,
+            privacy="public",
+        )
+
+        goals = [{"goal": 12, "year": 2023, "privacy": "followers"}]
+
+        with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
+
+            models.bookwyrm_import_job.update_goals(self.local_user, goals)
+
+        self.local_user.refresh_from_db()
+        goal = models.AnnualGoal.objects.get()
+        self.assertEqual(goal.year, 2023)
+        self.assertEqual(goal.goal, 12)
+        self.assertEqual(goal.privacy, "followers")
+
+    def test_upsert_saved_lists_existing(self):
+        """Test upserting an existing saved list"""
+
+        with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ):
+            book_list = models.List.objects.create(
+                name="My cool list",
+                user=self.rat_user,
+                remote_id="https://local.lists/9999",
+            )
+
+        self.assertFalse(self.local_user.saved_lists.filter(id=book_list.id).exists())
+
+        self.local_user.saved_lists.add(book_list)
+
+        self.assertTrue(self.local_user.saved_lists.filter(id=book_list.id).exists())
+
+        with patch("bookwyrm.activitypub.base_activity.resolve_remote_id"):
+            models.bookwyrm_import_job.upsert_saved_lists(
+                self.local_user, ["https://local.lists/9999"]
+            )
+        saved_lists = self.local_user.saved_lists.filter(
+            remote_id="https://local.lists/9999"
+        ).all()
+        self.assertEqual(len(saved_lists), 1)
+
+    def test_upsert_saved_lists_not_existing(self):
+        """Test upserting a new saved list"""
+
+        with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ):
+            book_list = models.List.objects.create(
+                name="My cool list",
+                user=self.rat_user,
+                remote_id="https://local.lists/9999",
+            )
+
+        self.assertFalse(self.local_user.saved_lists.filter(id=book_list.id).exists())
+
+        with patch("bookwyrm.activitypub.base_activity.resolve_remote_id"):
+            models.bookwyrm_import_job.upsert_saved_lists(
+                self.local_user, ["https://local.lists/9999"]
+            )
+
+        self.assertTrue(self.local_user.saved_lists.filter(id=book_list.id).exists())
+
+    def test_upsert_follows(self):
+        """Test take a list of remote ids and add as follows"""
+
+        before_follow = models.UserFollows.objects.filter(
+            user_subject=self.local_user, user_object=self.rat_user
+        ).exists()
+
+        self.assertFalse(before_follow)
+
+        with patch("bookwyrm.activitystreams.add_user_statuses_task.delay"), patch(
+            "bookwyrm.lists_stream.add_user_lists_task.delay"
+        ), patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
+            models.bookwyrm_import_job.upsert_follows(
+                self.local_user, self.json_data.get("follows")
+            )
+
+        after_follow = models.UserFollows.objects.filter(
+            user_subject=self.local_user, user_object=self.rat_user
+        ).exists()
+        self.assertTrue(after_follow)
+
+    def test_upsert_user_blocks(self):
+        """test adding blocked users"""
+
+        blocked_before = models.UserBlocks.objects.filter(
+            Q(
+                user_subject=self.local_user,
+                user_object=self.badger_user,
+            )
+        ).exists()
+        self.assertFalse(blocked_before)
+
+        with patch("bookwyrm.suggested_users.remove_suggestion_task.delay"), patch(
+            "bookwyrm.activitystreams.remove_user_statuses_task.delay"
+        ), patch("bookwyrm.lists_stream.remove_user_lists_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ):
+            models.bookwyrm_import_job.upsert_user_blocks(
+                self.local_user, self.json_data.get("blocks")
+            )
+
+        blocked_after = models.UserBlocks.objects.filter(
+            Q(
+                user_subject=self.local_user,
+                user_object=self.badger_user,
+            )
+        ).exists()
+        self.assertTrue(blocked_after)
+
+    def test_get_or_create_edition_existing(self):
+        """Test take a JSON string of books and editions,
+        find or create the editions in the database and
+        return a list of edition instances"""
+
+        self.assertEqual(models.Edition.objects.count(), 1)
+
+        with open(self.archive_file, "rb") as fileobj:
+            with BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile:
+
+                bookwyrm_import_job.get_or_create_edition(
+                    self.json_data["books"][1], tarfile
+                )  # Sand Talk
+
+                self.assertEqual(models.Edition.objects.count(), 1)
+
+    def test_get_or_create_edition_not_existing(self):
+        """Test take a JSON string of books and editions,
+        find or create the editions in the database and
+        return a list of edition instances"""
+
+        self.assertEqual(models.Edition.objects.count(), 1)
+
+        with open(self.archive_file, "rb") as fileobj:
+            with BookwyrmTarFile.open(mode="r:gz", fileobj=fileobj) as tarfile:
+
+                bookwyrm_import_job.get_or_create_edition(
+                    self.json_data["books"][0], tarfile
+                )  # Seeing like a state
+
+        self.assertTrue(models.Edition.objects.filter(isbn_13="9780300070163").exists())
+        self.assertEqual(models.Edition.objects.count(), 2)
+
+    def test_upsert_readthroughs(self):
+        """Test take a JSON string of readthroughs, find or create the
+        instances in the database and return a list of saved instances"""
+
+        readthroughs = [
+            {
+                "id": 1,
+                "created_date": "2023-08-24T10:18:45.923Z",
+                "updated_date": "2023-08-24T10:18:45.928Z",
+                "remote_id": "https://example.com/mouse/readthrough/1",
+                "user_id": 1,
+                "book_id": 1234,
+                "progress": 23,
+                "progress_mode": "PG",
+                "start_date": "2022-12-31T13:30:00Z",
+                "finish_date": "2023-08-23T14:30:00Z",
+                "stopped_date": None,
+                "is_active": False,
+            }
+        ]
+
+        self.assertEqual(models.ReadThrough.objects.count(), 0)
+        bookwyrm_import_job.upsert_readthroughs(
+            readthroughs, self.local_user, self.book.id
+        )
+
+        self.assertEqual(models.ReadThrough.objects.count(), 1)
+        self.assertEqual(models.ReadThrough.objects.first().progress_mode, "PG")
+        self.assertEqual(
+            models.ReadThrough.objects.first().start_date,
+            parse_datetime("2022-12-31T13:30:00Z"),
+        )
+        self.assertEqual(models.ReadThrough.objects.first().book_id, self.book.id)
+        self.assertEqual(models.ReadThrough.objects.first().user, self.local_user)
+
+    def test_get_or_create_review(self):
+        """Test get_or_create_review_status with a review"""
+
+        self.assertEqual(models.Review.objects.filter(user=self.local_user).count(), 0)
+        reviews = self.json_data["books"][0]["reviews"]
+        with patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True):
+
+            bookwyrm_import_job.upsert_statuses(
+                self.local_user, models.Review, reviews, self.book.remote_id
+            )
+
+        self.assertEqual(models.Review.objects.filter(user=self.local_user).count(), 1)
+        self.assertEqual(
+            models.Review.objects.filter(book=self.book).first().content,
+            "<p>I like it</p>",
+        )
+        self.assertEqual(
+            models.Review.objects.filter(book=self.book).first().content_warning,
+            "Here's a spoiler alert",
+        )
+        self.assertEqual(
+            models.Review.objects.filter(book=self.book).first().sensitive, True
+        )
+        self.assertEqual(
+            models.Review.objects.filter(book=self.book).first().name, "great book"
+        )
+        self.assertAlmostEqual(
+            models.Review.objects.filter(book=self.book).first().rating, 5.00
+        )
+
+        self.assertEqual(
+            models.Review.objects.filter(book=self.book).first().privacy, "followers"
+        )
+
+    def test_get_or_create_comment(self):
+        """Test get_or_create_review_status with a comment"""
+
+        self.assertEqual(models.Comment.objects.filter(user=self.local_user).count(), 0)
+        comments = self.json_data["books"][1]["comments"]
+
+        with patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True):
+
+            bookwyrm_import_job.upsert_statuses(
+                self.local_user, models.Comment, comments, self.book.remote_id
+            )
+        self.assertEqual(models.Comment.objects.filter(user=self.local_user).count(), 1)
+        self.assertEqual(
+            models.Comment.objects.filter(book=self.book).first().content,
+            "<p>this is a comment about an amazing book</p>",
+        )
+        self.assertEqual(
+            models.Comment.objects.filter(book=self.book).first().content_warning, None
+        )
+        self.assertEqual(
+            models.Comment.objects.filter(book=self.book).first().sensitive, False
+        )
+        self.assertEqual(
+            models.Comment.objects.filter(book=self.book).first().progress_mode, "PG"
+        )
+
+    def test_get_or_create_quote(self):
+        """Test get_or_create_review_status with a quote"""
+
+        self.assertEqual(
+            models.Quotation.objects.filter(user=self.local_user).count(), 0
+        )
+        quotes = self.json_data["books"][1]["quotations"]
+        with patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=True):
+
+            bookwyrm_import_job.upsert_statuses(
+                self.local_user, models.Quotation, quotes, self.book.remote_id
+            )
+        self.assertEqual(
+            models.Quotation.objects.filter(user=self.local_user).count(), 1
+        )
+        self.assertEqual(
+            models.Quotation.objects.filter(book=self.book).first().content,
+            "<p>not actually from this book lol</p>",
+        )
+        self.assertEqual(
+            models.Quotation.objects.filter(book=self.book).first().content_warning,
+            "spoiler ahead!",
+        )
+        self.assertEqual(
+            models.Quotation.objects.filter(book=self.book).first().quote,
+            "<p>To be or not to be</p>",
+        )
+        self.assertEqual(
+            models.Quotation.objects.filter(book=self.book).first().position_mode, "PG"
+        )
+
+    def test_get_or_create_quote_unauthorized(self):
+        """Test get_or_create_review_status with a quote but not authorized"""
+
+        self.assertEqual(
+            models.Quotation.objects.filter(user=self.local_user).count(), 0
+        )
+        quotes = self.json_data["books"][1]["quotations"]
+        with patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ), patch("bookwyrm.models.bookwyrm_import_job.is_alias", return_value=False):
+
+            bookwyrm_import_job.upsert_statuses(
+                self.local_user, models.Quotation, quotes, self.book.remote_id
+            )
+        self.assertEqual(
+            models.Quotation.objects.filter(user=self.local_user).count(), 0
+        )
+
+    def test_upsert_list_existing(self):
+        """Take a list and ListItems as JSON and create DB entries
+        if they don't already exist"""
+
+        book_data = self.json_data["books"][0]
+
+        other_book = models.Edition.objects.create(
+            title="Another Book", remote_id="https://example.com/book/9876"
+        )
+
+        with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ):
+            book_list = models.List.objects.create(
+                name="my list of books", user=self.local_user
+            )
+
+            models.ListItem.objects.create(
+                book=self.book, book_list=book_list, user=self.local_user, order=1
+            )
+
+        self.assertTrue(models.List.objects.filter(id=book_list.id).exists())
+        self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 1)
+        self.assertEqual(
+            models.ListItem.objects.filter(
+                user=self.local_user, book_list=book_list
+            ).count(),
+            1,
+        )
+
+        with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ):
+            bookwyrm_import_job.upsert_lists(
+                self.local_user,
+                book_data["lists"],
+                other_book.id,
+            )
+
+        self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 1)
+        self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 1)
+        self.assertEqual(
+            models.ListItem.objects.filter(
+                user=self.local_user, book_list=book_list
+            ).count(),
+            2,
+        )
+
+    def test_upsert_list_not_existing(self):
+        """Take a list and ListItems as JSON and create DB entries
+        if they don't already exist"""
+
+        book_data = self.json_data["books"][0]
+
+        self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 0)
+        self.assertFalse(models.ListItem.objects.filter(book=self.book.id).exists())
+
+        with patch("bookwyrm.lists_stream.remove_list_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ):
+            bookwyrm_import_job.upsert_lists(
+                self.local_user,
+                book_data["lists"],
+                self.book.id,
+            )
+
+        self.assertEqual(models.List.objects.filter(user=self.local_user).count(), 1)
+        self.assertEqual(
+            models.ListItem.objects.filter(user=self.local_user).count(), 1
+        )
+
+    def test_upsert_shelves_existing(self):
+        """Take shelf and ShelfBooks JSON objects and create
+        DB entries if they don't already exist"""
+
+        self.assertEqual(
+            models.ShelfBook.objects.filter(user=self.local_user.id).count(), 0
+        )
+
+        shelf = models.Shelf.objects.get(name="Read", user=self.local_user)
+
+        with patch("bookwyrm.activitystreams.add_book_statuses_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ):
+            models.ShelfBook.objects.create(
+                book=self.book, shelf=shelf, user=self.local_user
+            )
+
+        book_data = self.json_data["books"][0]
+        with patch("bookwyrm.activitystreams.add_book_statuses_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ):
+            bookwyrm_import_job.upsert_shelves(self.book, self.local_user, book_data)
+
+        self.assertEqual(
+            models.ShelfBook.objects.filter(user=self.local_user.id).count(), 2
+        )
+
+    def test_upsert_shelves_not_existing(self):
+        """Take shelf and ShelfBooks JSON objects and create
+        DB entries if they don't already exist"""
+
+        self.assertEqual(
+            models.ShelfBook.objects.filter(user=self.local_user.id).count(), 0
+        )
+
+        book_data = self.json_data["books"][0]
+
+        with patch("bookwyrm.activitystreams.add_book_statuses_task.delay"), patch(
+            "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
+        ):
+            bookwyrm_import_job.upsert_shelves(self.book, self.local_user, book_data)
+
+        self.assertEqual(
+            models.ShelfBook.objects.filter(user=self.local_user.id).count(), 2
+        )
+
+        # check we didn't create an extra shelf
+        self.assertEqual(
+            models.Shelf.objects.filter(user=self.local_user.id).count(), 4
+        )

bookwyrm/tests/utils/test_tar.py

@@ -0,0 +1,25 @@
+import os
+import pytest
+from bookwyrm.utils.tar import BookwyrmTarFile
+
+
+@pytest.fixture
+def read_tar():
+    archive_path = "../data/bookwyrm_account_export.tar.gz"
+    with open(archive_path, "rb") as archive_file:
+        with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar:
+            yield tar
+
+
+@pytest.fixture
+def write_tar():
+    archive_path = "/tmp/test.tar.gz"
+    with open(archive_path, "wb") as archive_file:
+        with BookwyrmTarFile.open(mode="w:gz", fileobj=archive_file) as tar:
+            yield tar
+
+    os.remove(archive_path)
+
+
+def test_write_bytes(write_tar):
+    write_tar.write_bytes(b"ABCDEF")

bookwyrm/tests/views/imports/test_user_import.py

@@ -0,0 +1,68 @@
+""" test for app action functionality """
+import pathlib
+from unittest.mock import patch
+
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.template.response import TemplateResponse
+from django.test import TestCase
+from django.test.client import RequestFactory
+
+from bookwyrm import forms, models, views
+from bookwyrm.tests.validate_html import validate_html
+
+
+class ImportUserViews(TestCase):
+    """user import views"""
+
+    # pylint: disable=invalid-name
+    def setUp(self):
+        """we need basic test data and mocks"""
+        self.factory = RequestFactory()
+        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
+            "bookwyrm.activitystreams.populate_stream_task.delay"
+        ), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
+            self.local_user = models.User.objects.create_user(
+                "mouse@local.com",
+                "mouse@mouse.mouse",
+                "password",
+                local=True,
+                localname="mouse",
+            )
+        models.SiteSettings.objects.create()
+
+    def test_get_user_import_page(self):
+        """there are so many views, this just makes sure it LOADS"""
+        view = views.UserImport.as_view()
+        request = self.factory.get("")
+        request.user = self.local_user
+        result = view(request)
+        self.assertIsInstance(result, TemplateResponse)
+        validate_html(result.render())
+        self.assertEqual(result.status_code, 200)
+
+    def test_user_import_post(self):
+        """does the import job start?"""
+
+        view = views.UserImport.as_view()
+        form = forms.ImportUserForm()
+        archive_file = pathlib.Path(__file__).parent.joinpath(
+            "../../data/bookwyrm_account_export.tar.gz"
+        )
+
+        form.data["archive_file"] = SimpleUploadedFile(
+            # pylint: disable=consider-using-with
+            archive_file,
+            open(archive_file, "rb").read(),
+            content_type="application/gzip",
+        )
+
+        form.data["include_user_settings"] = ""
+        form.data["include_goals"] = "on"
+
+        request = self.factory.post("", form.data)
+        request.user = self.local_user
+
+        with patch("bookwyrm.models.bookwyrm_import_job.BookwyrmImportJob.start_job"):
+            view(request)
+        job = models.BookwyrmImportJob.objects.get()
+        self.assertEqual(job.required, ["include_goals"])

bookwyrm/tests/views/preferences/test_export_user.py

@@ -0,0 +1,50 @@
+""" test for user export app functionality """
+from unittest.mock import patch
+
+from django.http import HttpResponse
+from django.test import TestCase
+from django.test.client import RequestFactory
+
+from bookwyrm import models, views
+from bookwyrm.tests.validate_html import validate_html
+
+
+class ExportUserViews(TestCase):
+    """exporting user data"""
+
+    def setUp(self):
+        self.factory = RequestFactory()
+        models.SiteSettings.objects.create()
+        with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
+            "bookwyrm.activitystreams.populate_stream_task.delay"
+        ):
+            self.local_user = models.User.objects.create_user(
+                "hugh@example.com",
+                "hugh@example.com",
+                "password",
+                local=True,
+                localname="Hugh",
+                summary="just a test account",
+                remote_id="https://example.com/users/hugh",
+                preferred_timezone="Australia/Broken_Hill",
+            )
+
+    def test_export_user_get(self, *_):
+        """request export"""
+        request = self.factory.get("")
+        request.user = self.local_user
+        result = views.ExportUser.as_view()(request)
+        validate_html(result.render())
+
+    def test_trigger_export_user_file(self, *_):
+        """simple user export"""
+
+        request = self.factory.post("")
+        request.user = self.local_user
+        with patch("bookwyrm.models.bookwyrm_export_job.start_export_task.delay"):
+            export = views.ExportUser.as_view()(request)
+        self.assertIsInstance(export, HttpResponse)
+        self.assertEqual(export.status_code, 302)
+
+        jobs = models.bookwyrm_export_job.BookwyrmExportJob.objects.count()
+        self.assertEqual(jobs, 1)

bookwyrm/urls.py

@@ -328,6 +328,11 @@ urlpatterns = [
         views.ImportList.as_view(),
         name="settings-imports-complete",
     ),
+    re_path(
+        r"^settings/user-imports/(?P<import_id>\d+)/complete/?$",
+        views.set_user_import_completed,
+        name="settings-user-import-complete",
+    ),
     re_path(
         r"^settings/imports/disable/?$",
         views.disable_imports,
@@ -343,6 +348,11 @@ urlpatterns = [
         views.set_import_size_limit,
         name="settings-imports-set-limit",
     ),
+    re_path(
+        r"^settings/user-imports/set-limit/?$",
+        views.set_user_import_limit,
+        name="settings-user-imports-set-limit",
+    ),
     re_path(
         r"^settings/celery/?$", views.CeleryStatus.as_view(), name="settings-celery"
     ),
@@ -408,6 +418,7 @@ urlpatterns = [
     re_path(r"^search/?$", views.Search.as_view(), name="search"),
     # imports
     re_path(r"^import/?$", views.Import.as_view(), name="import"),
+    re_path(r"^user-import/?$", views.UserImport.as_view(), name="user-import"),
     re_path(
         r"^import/(?P<job_id>\d+)/?$",
         views.ImportStatus.as_view(),
@@ -605,6 +616,16 @@ urlpatterns = [
         name="prompt-2fa",
     ),
     re_path(r"^preferences/export/?$", views.Export.as_view(), name="prefs-export"),
+    re_path(
+        r"^preferences/user-export/?$",
+        views.ExportUser.as_view(),
+        name="prefs-user-export",
+    ),
+    path(
+        "preferences/user-export/<archive_id>",
+        views.ExportArchive.as_view(),
+        name="prefs-export-file",
+    ),
     re_path(r"^preferences/move/?$", views.MoveUser.as_view(), name="prefs-move"),
     re_path(r"^preferences/alias/?$", views.AliasUser.as_view(), name="prefs-alias"),
     re_path(

bookwyrm/utils/tar.py

@@ -0,0 +1,49 @@
+"""manage tar files for user exports"""
+import io
+import tarfile
+from typing import Any, Optional
+from uuid import uuid4
+from django.core.files import File
+
+
+class BookwyrmTarFile(tarfile.TarFile):
+    """Create tar files for user exports"""
+
+    def write_bytes(self, data: bytes) -> None:
+        """Add a file containing bytes to the archive"""
+        buffer = io.BytesIO(data)
+        info = tarfile.TarInfo("archive.json")
+        info.size = len(data)
+        self.addfile(info, fileobj=buffer)
+
+    def add_image(
+        self, image: Any, filename: Optional[str] = None, directory: Any = ""
+    ) -> None:
+        """
+        Add an image to the tar archive
+        :param str filename: overrides the file name set by image
+        :param str directory: the directory in the archive to put the image
+        """
+        if filename is not None:
+            file_type = image.name.rsplit(".", maxsplit=1)[-1]
+            filename = f"{directory}{filename}.{file_type}"
+        else:
+            filename = f"{directory}{image.name}"
+
+        info = tarfile.TarInfo(name=filename)
+        info.size = image.size
+
+        self.addfile(info, fileobj=image)
+
+    def read(self, filename: str) -> Any:
+        """read data from the tar"""
+        if reader := self.extractfile(filename):
+            return reader.read()
+        return None
+
+    def write_image_to_file(self, filename: str, file_field: Any) -> None:
+        """add an image to the tar"""
+        extension = filename.rsplit(".")[-1]
+        if buf := self.extractfile(filename):
+            filename = f"{str(uuid4())}.{extension}"
+            file_field.save(filename, File(buf))

bookwyrm/views/__init__.py

@@ -16,6 +16,8 @@ from .admin.imports import (
     disable_imports,
     enable_imports,
     set_import_size_limit,
+    set_user_import_completed,
+    set_user_import_limit,
 )
 from .admin.ip_blocklist import IPBlocklist
 from .admin.invite import ManageInvites, Invite, InviteRequest
@@ -36,7 +38,7 @@ from .admin.user_admin import UserAdmin, UserAdminList, ActivateUserAdmin
 # user preferences
 from .preferences.change_password import ChangePassword
 from .preferences.edit_user import EditUser
-from .preferences.export import Export
+from .preferences.export import Export, ExportUser, ExportArchive
 from .preferences.move_user import MoveUser, AliasUser, remove_alias, unmove
 from .preferences.delete_user import DeleteUser, DeactivateUser, ReactivateUser
 from .preferences.block import Block, unblock
@@ -81,7 +83,7 @@ from .shelf.shelf_actions import create_shelf, delete_shelf
 from .shelf.shelf_actions import shelve, unshelve
 
 # csv import
-from .imports.import_data import Import
+from .imports.import_data import Import, UserImport
 from .imports.import_status import ImportStatus, retry_item, stop_import
 from .imports.troubleshoot import ImportTroubleshoot
 from .imports.manually_review import (

bookwyrm/views/admin/imports.py

@@ -40,9 +40,17 @@ class ImportList(View):
         paginated = Paginator(imports, PAGE_LENGTH)
         page = paginated.get_page(request.GET.get("page"))
 
+        user_imports = models.BookwyrmImportJob.objects.filter(
+            complete=complete
+        ).order_by("created_date")
+
+        user_paginated = Paginator(user_imports, PAGE_LENGTH)
+        user_page = user_paginated.get_page(request.GET.get("page"))
+
         site_settings = models.SiteSettings.objects.get()
         data = {
             "imports": page,
+            "user_imports": user_page,
             "page_range": paginated.get_elided_page_range(
                 page.number, on_each_side=2, on_ends=1
             ),
@@ -50,6 +58,7 @@ class ImportList(View):
             "sort": sort,
             "import_size_limit": site_settings.import_size_limit,
             "import_limit_reset": site_settings.import_limit_reset,
+            "user_import_time_limit": site_settings.user_import_time_limit,
         }
         return TemplateResponse(request, "settings/imports/imports.html", data)
 
@@ -95,3 +104,25 @@ def set_import_size_limit(request):
     site.import_limit_reset = import_limit_reset
     site.save(update_fields=["import_size_limit", "import_limit_reset"])
     return redirect("settings-imports")
+
+
+@require_POST
+@login_required
+@permission_required("bookwyrm.moderate_user", raise_exception=True)
+# pylint: disable=unused-argument
+def set_user_import_completed(request, import_id):
+    """Mark a user import as complete"""
+    import_job = get_object_or_404(models.BookwyrmImportJob, id=import_id)
+    import_job.stop_job()
+    return redirect("settings-imports")
+
+
+@require_POST
+@permission_required("bookwyrm.edit_instance_settings", raise_exception=True)
+# pylint: disable=unused-argument
+def set_user_import_limit(request):
+    """Limit how ofter users can import and export their account"""
+    site = models.SiteSettings.objects.get()
+    site.user_import_time_limit = int(request.POST.get("limit"))
+    site.save(update_fields=["user_import_time_limit"])
+    return redirect("settings-imports")

bookwyrm/views/imports/import_data.py

@@ -15,12 +15,14 @@ from django.views import View
 
 from bookwyrm import forms, models
 from bookwyrm.importers import (
+    BookwyrmImporter,
     CalibreImporter,
     LibrarythingImporter,
     GoodreadsImporter,
     StorygraphImporter,
     OpenLibraryImporter,
 )
+from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
 from bookwyrm.settings import PAGE_LENGTH
 from bookwyrm.utils.cache import get_or_set
 
@@ -127,3 +129,61 @@ def get_average_import_time() -> float:
     if recent_avg:
         return recent_avg.total_seconds()
     return None
+
+
+# pylint: disable= no-self-use
+@method_decorator(login_required, name="dispatch")
+class UserImport(View):
+    """import user view"""
+
+    def get(self, request, invalid=False):
+        """load user import page"""
+
+        jobs = BookwyrmImportJob.objects.filter(user=request.user).order_by(
+            "-created_date"
+        )
+        site = models.SiteSettings.objects.get()
+        hours = site.user_import_time_limit
+        allowed = (
+            jobs.first().created_date < timezone.now() - datetime.timedelta(hours=hours)
+            if jobs.first()
+            else True
+        )
+        next_available = (
+            jobs.first().created_date + datetime.timedelta(hours=hours)
+            if not allowed
+            else False
+        )
+        paginated = Paginator(jobs, PAGE_LENGTH)
+        page = paginated.get_page(request.GET.get("page"))
+        data = {
+            "import_form": forms.ImportUserForm(),
+            "jobs": page,
+            "user_import_hours": hours,
+            "next_available": next_available,
+            "page_range": paginated.get_elided_page_range(
+                page.number, on_each_side=2, on_ends=1
+            ),
+            "invalid": invalid,
+        }
+
+        return TemplateResponse(request, "import/import_user.html", data)
+
+    def post(self, request):
+        """ingest a Bookwyrm json file"""
+
+        importer = BookwyrmImporter()
+
+        form = forms.ImportUserForm(request.POST, request.FILES)
+        if not form.is_valid():
+            return HttpResponseBadRequest()
+
+        job = importer.process_import(
+            user=request.user,
+            archive_file=request.FILES["archive_file"],
+            settings=request.POST,
+        )
+
+        job.start_job()
+
+        return redirect("user-import")

bookwyrm/views/preferences/export.py

@@ -1,15 +1,21 @@
 """ Let users export their book data """
+from datetime import timedelta
 import csv
 import io
 
 from django.contrib.auth.decorators import login_required
+from django.core.paginator import Paginator
 from django.db.models import Q
 from django.http import HttpResponse
 from django.template.response import TemplateResponse
+from django.utils import timezone
 from django.views import View
 from django.utils.decorators import method_decorator
+from django.shortcuts import redirect
 
 from bookwyrm import models
+from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob
+from bookwyrm.settings import PAGE_LENGTH
 
 # pylint: disable=no-self-use
 @method_decorator(login_required, name="dispatch")
@@ -84,3 +90,61 @@ class Export(View):
                 "Content-Disposition": 'attachment; filename="bookwyrm-export.csv"'
             },
         )
+
+
+# pylint: disable=no-self-use
+@method_decorator(login_required, name="dispatch")
+class ExportUser(View):
+    """Let users export user data to import into another Bookwyrm instance"""
+
+    def get(self, request):
+        """Request tar file"""
+
+        jobs = BookwyrmExportJob.objects.filter(user=request.user).order_by(
+            "-created_date"
+        )
+        site = models.SiteSettings.objects.get()
+        hours = site.user_import_time_limit
+        allowed = (
+            jobs.first().created_date < timezone.now() - timedelta(hours=hours)
+            if jobs.first()
+            else True
+        )
+        next_available = (
+            jobs.first().created_date + timedelta(hours=hours) if not allowed else False
+        )
+        paginated = Paginator(jobs, PAGE_LENGTH)
+        page = paginated.get_page(request.GET.get("page"))
+        data = {
+            "jobs": page,
+            "next_available": next_available,
+            "page_range": paginated.get_elided_page_range(
+                page.number, on_each_side=2, on_ends=1
+            ),
+        }
+
+        return TemplateResponse(request, "preferences/export-user.html", data)
+
+    def post(self, request):
+        """Download the json file of a user's data"""
+
+        job = BookwyrmExportJob.objects.create(user=request.user)
+        job.start_job()
+
+        return redirect("prefs-user-export")
+
+
+@method_decorator(login_required, name="dispatch")
+class ExportArchive(View):
+    """Serve the archive file"""
+
+    def get(self, request, archive_id):
+        """download user export file"""
+        export = BookwyrmExportJob.objects.get(task_id=archive_id, user=request.user)
+        return HttpResponse(
+            export.export_data,
+            content_type="application/gzip",
+            headers={
+                "Content-Disposition": 'attachment; filename="bookwyrm-account-export.tar.gz"'  # pylint: disable=line-too-long
+            },
+        )