merge latest changes and add tests

bookwyrm/importers/__init__.py

@@ -1,7 +1,7 @@
 """ import classes """
 
 from .importer import Importer
-from .bookwyrm_import import BookwyrmImporter
+from .bookwyrm_import import BookwyrmImporter, BookwyrmBooksImporter
 from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter

bookwyrm/importers/bookwyrm_import.py

@@ -1,8 +1,10 @@
 """Import data from Bookwyrm export files"""
+from typing import Any
 from django.http import QueryDict
 
 from bookwyrm.models import User
 from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
+from . import Importer
 
 
 class BookwyrmImporter:
@@ -22,3 +24,17 @@ class BookwyrmImporter:
             user=user, archive_file=archive_file, required=required
         )
         return job
+
+
+class BookwyrmBooksImporter(Importer):
+    """
+    Handle reading a csv from BookWyrm.
+    Goodreads is the default importer, we basically just use the same structure
+    But BookWyrm has a shelf.id (shelf) and a shelf.name (shelf_name)
+    """
+
+    service = "BookWyrm"
+
+    def __init__(self, *args: Any, **kwargs: Any):
+        self.row_mappings_guesses.append(("shelf_name", ["shelf_name"]))
+        super().__init__(*args, **kwargs)

bookwyrm/importers/importer.py

@@ -19,16 +19,25 @@ class Importer:
         ("id", ["id", "book id"]),
         ("title", ["title"]),
         ("authors", ["author_text", "author", "authors", "primary author"]),
-        ("isbn_10", ["isbn10", "isbn", "isbn/uid"]),
+        ("isbn_10", ["isbn_10", "isbn10", "isbn", "isbn/uid"]),
-        ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]),
+        ("isbn_13", ["isbn_13", "isbn13", "isbn", "isbns", "isbn/uid"]),
         ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]),
         ("review_name", ["review_name", "review name"]),
         ("review_body", ["review_content", "my review", "review"]),
         ("rating", ["my rating", "rating", "star rating"]),
-        ("date_added", ["date_added", "date added", "entry date", "added"]),
+        (
-        ("date_started", ["date started", "started"]),
+            "date_added",
-        ("date_finished", ["date finished", "last date read", "date read", "finished"]),
+            ["shelf_date", "date_added", "date added", "entry date", "added"],
+        ),
+        ("date_started", ["start_date", "date started", "started"]),
+        (
+            "date_finished",
+            ["finish_date", "date finished", "last date read", "date read", "finished"],
+        ),
     ]
+
+    # TODO: stopped
+
     date_fields = ["date_added", "date_started", "date_finished"]
     shelf_mapping_guesses = {
         "to-read": ["to-read", "want to read"],
@@ -36,14 +45,14 @@ class Importer:
         "reading": ["currently-reading", "reading", "currently reading"],
     }
 
-    # pylint: disable=too-many-locals
+    # pylint: disable=too-many-locals.too-many-arguments
     def create_job(
         self,
         user: User,
         csv_file: Iterable[str],
         include_reviews: bool,
-        create_shelves: bool,
         privacy: str,
+        create_shelves: bool = True,
     ) -> ImportJob:
         """check over a csv and creates a database entry for the job"""
         csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)

bookwyrm/migrations/0207_merge_20240629_0626.py

@@ -0,0 +1,13 @@
+# Generated by Django 4.2.11 on 2024-06-29 06:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0189_importjob_create_shelves"),
+        ("bookwyrm", "0206_merge_20240415_1537"),
+    ]
+
+    operations = []

bookwyrm/models/import_job.py

@@ -375,7 +375,7 @@ def import_item_task(item_id):
     item.update_job()
 
 
-def handle_imported_book(item):
+def handle_imported_book(item):  # pylint: disable=too-many-branches
     """process a csv and then post about it"""
     job = item.job
     if job.complete:
@@ -392,27 +392,20 @@ def handle_imported_book(item):
         item.book = item.book.edition
 
     existing_shelf = ShelfBook.objects.filter(book=item.book, user=user).exists()
-
+    if job.create_shelves and item.shelf and not existing_shelf:
         # shelve the book if it hasn't been shelved already
-    if item.shelf and not existing_shelf:
 
         shelved_date = item.date_added or timezone.now()
+        shelfname = getattr(item, "shelf_name", item.shelf)
 
         try:
-
+            shelf = Shelf.objects.get(name=shelfname, user=user)
-            desired_shelf = Shelf.objects.get(identifier=item.shelf, user=user)
-            shelved_date = item.date_added or timezone.now()
-            ShelfBook(
-                book=item.book,
-                shelf=desired_shelf,
-                user=user,
-                shelved_date=shelved_date,
-            ).save(priority=IMPORT_TRIGGERED)
-
         except ObjectDoesNotExist:
-            if job.create_shelves:
+            try:
-                shelfname = getattr(item, "shelf_name", item.shelf)
+                shelf = Shelf.objects.get(identifier=item.shelf, user=user)
-                new_shelf = Shelf.objects.create(
+            except ObjectDoesNotExist:
+
+                shelf = Shelf.objects.create(
                     user=user,
                     identifier=item.shelf,
                     name=shelfname,
@@ -421,7 +414,7 @@ def handle_imported_book(item):
 
         ShelfBook(
             book=item.book,
-                    shelf=new_shelf,
+            shelf=shelf,
             user=user,
             shelved_date=shelved_date,
         ).save(priority=IMPORT_TRIGGERED)

bookwyrm/tests/data/bookwyrm.csv

@@ -0,0 +1,4 @@
+title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date
+Gideon the Ninth (The Locked Tomb #1),Tamsyn Muir,https://example.com/book2,,,,,,,,,,,1250313198,9781250313195,,2020-10-21,2020-10-25,,3,,,,,read,Read,2020-10-21
+Subcutanean,Aaron A. Reed,https://example.com/book3,,,,,,,,,,,,,,2020-03-05,2020-03-06,,0,,,,,read,Read,2020-03-05
+Patisserie at Home,Mélanie Dupuis,https://example.com/book4,,,,,,,,,,,0062445316,9780062445315,,2019-07-08,,,2,,,mixed feelings,2019-07-08,cooking,Cooking,2019-07-08

bookwyrm/tests/importers/test_bookwyrm_import.py

@@ -0,0 +1,173 @@
+""" testing bookwyrm csv import """
+import pathlib
+from unittest.mock import patch
+import datetime
+
+from django.test import TestCase
+
+from bookwyrm import models
+from bookwyrm.importers import BookwyrmBooksImporter
+from bookwyrm.models.import_job import handle_imported_book
+
+
+def make_date(*args):
+    """helper function to easily generate a date obj"""
+    return datetime.datetime(*args, tzinfo=datetime.timezone.utc)
+
+
+@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
+@patch("bookwyrm.activitystreams.populate_stream_task.delay")
+@patch("bookwyrm.activitystreams.add_book_statuses_task.delay")
+class BookwyrmBooksImport(TestCase):
+    """importing from BookWyrm csv"""
+
+    def setUp(self):
+        """use a test csv"""
+        self.importer = BookwyrmBooksImporter()
+        datafile = pathlib.Path(__file__).parent.joinpath("../data/bookwyrm.csv")
+        # pylint: disable-next=consider-using-with
+        self.csv = open(datafile, "r", encoding=self.importer.encoding)
+
+    def tearDown(self):
+        """close test csv"""
+        self.csv.close()
+
+    @classmethod
+    def setUpTestData(cls):
+        """populate database"""
+        with (
+            patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"),
+            patch("bookwyrm.activitystreams.populate_stream_task.delay"),
+            patch("bookwyrm.lists_stream.populate_lists_task.delay"),
+        ):
+            cls.local_user = models.User.objects.create_user(
+                "mouse", "mouse@mouse.mouse", "password", local=True
+            )
+        models.SiteSettings.objects.create()
+        work = models.Work.objects.create(title="Test Work")
+        cls.book = models.Edition.objects.create(
+            title="Example Edition",
+            remote_id="https://example.com/book/1",
+            parent_work=work,
+        )
+
+    def test_create_job(self, *_):
+        """creates the import job entry and checks csv"""
+        import_job = self.importer.create_job(
+            self.local_user, self.csv, False, "public"
+        )
+
+        import_items = models.ImportItem.objects.filter(job=import_job).all()
+        self.assertEqual(len(import_items), 3)
+        self.assertEqual(import_items[0].index, 0)
+        self.assertEqual(import_items[0].normalized_data["isbn_13"], "9781250313195")
+        self.assertEqual(import_items[0].normalized_data["isbn_10"], "1250313198")
+        self.assertEqual(import_items[1].index, 1)
+        self.assertEqual(import_items[2].index, 2)
+        self.assertEqual(import_items[2].shelf_name, "Cooking")
+
+    def test_create_retry_job(self, *_):
+        """trying again with items that didn't import"""
+        import_job = self.importer.create_job(
+            self.local_user, self.csv, False, "unlisted"
+        )
+        import_items = models.ImportItem.objects.filter(job=import_job).all()[:2]
+
+        retry = self.importer.create_retry_job(
+            self.local_user, import_job, import_items
+        )
+        self.assertNotEqual(import_job, retry)
+        self.assertEqual(retry.user, self.local_user)
+        self.assertEqual(retry.include_reviews, False)
+        self.assertEqual(retry.privacy, "unlisted")
+
+        retry_items = models.ImportItem.objects.filter(job=retry).all()
+        self.assertEqual(len(retry_items), 2)
+        self.assertEqual(retry_items[0].index, 0)
+        self.assertEqual(retry_items[1].index, 1)
+
+    def test_handle_imported_book(self, *_):
+        """import added a book, this adds related connections"""
+        shelf = self.local_user.shelf_set.filter(
+            identifier=models.Shelf.READ_FINISHED
+        ).first()
+        self.assertIsNone(shelf.books.first())
+
+        import_job = self.importer.create_job(
+            self.local_user, self.csv, False, "public"
+        )
+        import_item = import_job.items.first()
+        import_item.book = self.book
+        import_item.save()
+
+        with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
+            handle_imported_book(import_item)
+
+        shelf.refresh_from_db()
+        self.assertEqual(shelf.books.first(), self.book)
+        self.assertEqual(
+            shelf.shelfbook_set.first().shelved_date, make_date(2020, 10, 21)
+        )
+
+        readthrough = models.ReadThrough.objects.get(user=self.local_user)
+        self.assertEqual(readthrough.book, self.book)
+        self.assertEqual(readthrough.start_date, make_date(2020, 10, 21))
+        self.assertEqual(readthrough.finish_date, make_date(2020, 10, 25))
+
+    def test_create_new_shelf(self, *_):
+        """import added a book, was a new shelf created?"""
+        shelf = self.local_user.shelf_set.filter(identifier="cooking").first()
+        self.assertIsNone(shelf)
+
+        import_job = self.importer.create_job(
+            self.local_user, self.csv, False, "public"
+        )
+        import_item = models.ImportItem.objects.filter(job=import_job).all()[2]
+        import_item.book = self.book
+        import_item.save()
+
+        # this doesn't pick up 'shelf_name' when running all tests
+        # but does when only running this test...????
+        with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
+            handle_imported_book(import_item)
+
+        shelf_after = self.local_user.shelf_set.filter(identifier="cooking").first()
+        self.assertEqual(shelf_after.books.first(), self.book)
+
+    @patch("bookwyrm.activitystreams.add_status_task.delay")
+    def test_handle_imported_book_review(self, *_):
+        """review import"""
+        import_job = self.importer.create_job(
+            self.local_user, self.csv, True, "unlisted"
+        )
+        import_item = import_job.items.get(index=2)
+        import_item.book = self.book
+        import_item.save()
+
+        with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
+            handle_imported_book(import_item)
+
+        review = models.Review.objects.get(book=self.book, user=self.local_user)
+        self.assertEqual(review.content, "mixed feelings")
+        self.assertEqual(review.rating, 2)
+        self.assertEqual(review.published_date, make_date(2019, 7, 8))
+        self.assertEqual(review.privacy, "unlisted")
+
+    @patch("bookwyrm.activitystreams.add_status_task.delay")
+    def test_handle_imported_book_rating(self, *_):
+        """rating import"""
+        import_job = self.importer.create_job(
+            self.local_user, self.csv, True, "unlisted"
+        )
+        import_item = import_job.items.filter(index=0).first()
+        import_item.book = self.book
+        import_item.save()
+
+        with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
+            handle_imported_book(import_item)
+
+        review = models.ReviewRating.objects.get(book=self.book, user=self.local_user)
+        self.assertIsInstance(review, models.ReviewRating)
+        self.assertEqual(review.rating, 3)
+        self.assertEqual(review.published_date, make_date(2020, 10, 25))
+        self.assertEqual(review.privacy, "unlisted")

bookwyrm/views/imports/import_data.py

@@ -16,6 +16,7 @@ from django.views import View
 from bookwyrm import forms, models
 from bookwyrm.importers import (
     BookwyrmImporter,
+    BookwyrmBooksImporter,
     CalibreImporter,
     LibrarythingImporter,
     GoodreadsImporter,
@@ -105,8 +106,8 @@ class Import(View):
                 request.user,
                 TextIOWrapper(request.FILES["csv_file"], encoding=importer.encoding),
                 include_reviews,
-                create_shelves,
                 privacy,
+                create_shelves,
             )
         except (UnicodeDecodeError, ValueError, KeyError):
             return self.get(request, invalid=True)