diff --git a/bookwyrm/importers/__init__.py b/bookwyrm/importers/__init__.py index 8e92872f2..3c895741b 100644 --- a/bookwyrm/importers/__init__.py +++ b/bookwyrm/importers/__init__.py @@ -1,7 +1,7 @@ """ import classes """ from .importer import Importer -from .bookwyrm_import import BookwyrmImporter +from .bookwyrm_import import BookwyrmImporter, BookwyrmBooksImporter from .calibre_import import CalibreImporter from .goodreads_import import GoodreadsImporter from .librarything_import import LibrarythingImporter diff --git a/bookwyrm/importers/bookwyrm_import.py b/bookwyrm/importers/bookwyrm_import.py index 206cd6219..7b343d5c9 100644 --- a/bookwyrm/importers/bookwyrm_import.py +++ b/bookwyrm/importers/bookwyrm_import.py @@ -1,8 +1,10 @@ """Import data from Bookwyrm export files""" +from typing import Any from django.http import QueryDict from bookwyrm.models import User from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob +from . import Importer class BookwyrmImporter: @@ -22,3 +24,17 @@ class BookwyrmImporter: user=user, archive_file=archive_file, required=required ) return job + + +class BookwyrmBooksImporter(Importer): + """ + Handle reading a csv from BookWyrm. + Goodreads is the default importer, we basically just use the same structure + But BookWyrm has a shelf.id (shelf) and a shelf.name (shelf_name) + """ + + service = "BookWyrm" + + def __init__(self, *args: Any, **kwargs: Any): + self.row_mappings_guesses.append(("shelf_name", ["shelf_name"])) + super().__init__(*args, **kwargs) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 8e60f7c5f..fc8d4ac13 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -19,16 +19,25 @@ class Importer: ("id", ["id", "book id"]), ("title", ["title"]), ("authors", ["author_text", "author", "authors", "primary author"]), - ("isbn_10", ["isbn10", "isbn", "isbn/uid"]), - ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]), + ("isbn_10", ["isbn_10", "isbn10", "isbn", "isbn/uid"]), + ("isbn_13", ["isbn_13", "isbn13", "isbn", "isbns", "isbn/uid"]), ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]), ("review_name", ["review_name", "review name"]), ("review_body", ["review_content", "my review", "review"]), ("rating", ["my rating", "rating", "star rating"]), - ("date_added", ["date_added", "date added", "entry date", "added"]), - ("date_started", ["date started", "started"]), - ("date_finished", ["date finished", "last date read", "date read", "finished"]), + ( + "date_added", + ["shelf_date", "date_added", "date added", "entry date", "added"], + ), + ("date_started", ["start_date", "date started", "started"]), + ( + "date_finished", + ["finish_date", "date finished", "last date read", "date read", "finished"], + ), ] + + # TODO: stopped + date_fields = ["date_added", "date_started", "date_finished"] shelf_mapping_guesses = { "to-read": ["to-read", "want to read"], @@ -36,14 +45,14 @@ class Importer: "reading": ["currently-reading", "reading", "currently reading"], } - # pylint: disable=too-many-locals + # pylint: disable=too-many-locals.too-many-arguments def create_job( self, user: User, csv_file: Iterable[str], include_reviews: bool, - create_shelves: bool, privacy: str, + create_shelves: bool = True, ) -> ImportJob: """check over a csv and creates a database entry for the job""" csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter) diff --git a/bookwyrm/migrations/0207_merge_20240629_0626.py b/bookwyrm/migrations/0207_merge_20240629_0626.py new file mode 100644 index 000000000..b5a1a4556 --- /dev/null +++ b/bookwyrm/migrations/0207_merge_20240629_0626.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.11 on 2024-06-29 06:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0189_importjob_create_shelves"), + ("bookwyrm", "0206_merge_20240415_1537"), + ] + + operations = [] diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 6e1cae1ee..8ca4c346d 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -375,7 +375,7 @@ def import_item_task(item_id): item.update_job() -def handle_imported_book(item): +def handle_imported_book(item): # pylint: disable=too-many-branches """process a csv and then post about it""" job = item.job if job.complete: @@ -392,39 +392,32 @@ def handle_imported_book(item): item.book = item.book.edition existing_shelf = ShelfBook.objects.filter(book=item.book, user=user).exists() - - # shelve the book if it hasn't been shelved already - if item.shelf and not existing_shelf: + if job.create_shelves and item.shelf and not existing_shelf: + # shelve the book if it hasn't been shelved already shelved_date = item.date_added or timezone.now() + shelfname = getattr(item, "shelf_name", item.shelf) try: - - desired_shelf = Shelf.objects.get(identifier=item.shelf, user=user) - shelved_date = item.date_added or timezone.now() - ShelfBook( - book=item.book, - shelf=desired_shelf, - user=user, - shelved_date=shelved_date, - ).save(priority=IMPORT_TRIGGERED) - + shelf = Shelf.objects.get(name=shelfname, user=user) except ObjectDoesNotExist: - if job.create_shelves: - shelfname = getattr(item, "shelf_name", item.shelf) - new_shelf = Shelf.objects.create( + try: + shelf = Shelf.objects.get(identifier=item.shelf, user=user) + except ObjectDoesNotExist: + + shelf = Shelf.objects.create( user=user, identifier=item.shelf, name=shelfname, privacy=job.privacy, ) - ShelfBook( - book=item.book, - shelf=new_shelf, - user=user, - shelved_date=shelved_date, - ).save(priority=IMPORT_TRIGGERED) + ShelfBook( + book=item.book, + shelf=shelf, + user=user, + shelved_date=shelved_date, + ).save(priority=IMPORT_TRIGGERED) for read in item.reads: # check for an existing readthrough with the same dates diff --git a/bookwyrm/tests/data/bookwyrm.csv b/bookwyrm/tests/data/bookwyrm.csv new file mode 100644 index 000000000..d770f1dcf --- /dev/null +++ b/bookwyrm/tests/data/bookwyrm.csv @@ -0,0 +1,4 @@ +title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date +Gideon the Ninth (The Locked Tomb #1),Tamsyn Muir,https://example.com/book2,,,,,,,,,,,1250313198,9781250313195,,2020-10-21,2020-10-25,,3,,,,,read,Read,2020-10-21 +Subcutanean,Aaron A. Reed,https://example.com/book3,,,,,,,,,,,,,,2020-03-05,2020-03-06,,0,,,,,read,Read,2020-03-05 +Patisserie at Home,Mélanie Dupuis,https://example.com/book4,,,,,,,,,,,0062445316,9780062445315,,2019-07-08,,,2,,,mixed feelings,2019-07-08,cooking,Cooking,2019-07-08 diff --git a/bookwyrm/tests/importers/test_bookwyrm_import.py b/bookwyrm/tests/importers/test_bookwyrm_import.py new file mode 100644 index 000000000..3a9169c99 --- /dev/null +++ b/bookwyrm/tests/importers/test_bookwyrm_import.py @@ -0,0 +1,173 @@ +""" testing bookwyrm csv import """ +import pathlib +from unittest.mock import patch +import datetime + +from django.test import TestCase + +from bookwyrm import models +from bookwyrm.importers import BookwyrmBooksImporter +from bookwyrm.models.import_job import handle_imported_book + + +def make_date(*args): + """helper function to easily generate a date obj""" + return datetime.datetime(*args, tzinfo=datetime.timezone.utc) + + +@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") +@patch("bookwyrm.activitystreams.populate_stream_task.delay") +@patch("bookwyrm.activitystreams.add_book_statuses_task.delay") +class BookwyrmBooksImport(TestCase): + """importing from BookWyrm csv""" + + def setUp(self): + """use a test csv""" + self.importer = BookwyrmBooksImporter() + datafile = pathlib.Path(__file__).parent.joinpath("../data/bookwyrm.csv") + # pylint: disable-next=consider-using-with + self.csv = open(datafile, "r", encoding=self.importer.encoding) + + def tearDown(self): + """close test csv""" + self.csv.close() + + @classmethod + def setUpTestData(cls): + """populate database""" + with ( + patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), + patch("bookwyrm.activitystreams.populate_stream_task.delay"), + patch("bookwyrm.lists_stream.populate_lists_task.delay"), + ): + cls.local_user = models.User.objects.create_user( + "mouse", "mouse@mouse.mouse", "password", local=True + ) + models.SiteSettings.objects.create() + work = models.Work.objects.create(title="Test Work") + cls.book = models.Edition.objects.create( + title="Example Edition", + remote_id="https://example.com/book/1", + parent_work=work, + ) + + def test_create_job(self, *_): + """creates the import job entry and checks csv""" + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + + import_items = models.ImportItem.objects.filter(job=import_job).all() + self.assertEqual(len(import_items), 3) + self.assertEqual(import_items[0].index, 0) + self.assertEqual(import_items[0].normalized_data["isbn_13"], "9781250313195") + self.assertEqual(import_items[0].normalized_data["isbn_10"], "1250313198") + self.assertEqual(import_items[1].index, 1) + self.assertEqual(import_items[2].index, 2) + self.assertEqual(import_items[2].shelf_name, "Cooking") + + def test_create_retry_job(self, *_): + """trying again with items that didn't import""" + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) + import_items = models.ImportItem.objects.filter(job=import_job).all()[:2] + + retry = self.importer.create_retry_job( + self.local_user, import_job, import_items + ) + self.assertNotEqual(import_job, retry) + self.assertEqual(retry.user, self.local_user) + self.assertEqual(retry.include_reviews, False) + self.assertEqual(retry.privacy, "unlisted") + + retry_items = models.ImportItem.objects.filter(job=retry).all() + self.assertEqual(len(retry_items), 2) + self.assertEqual(retry_items[0].index, 0) + self.assertEqual(retry_items[1].index, 1) + + def test_handle_imported_book(self, *_): + """import added a book, this adds related connections""" + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READ_FINISHED + ).first() + self.assertIsNone(shelf.books.first()) + + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() + + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + handle_imported_book(import_item) + + shelf.refresh_from_db() + self.assertEqual(shelf.books.first(), self.book) + self.assertEqual( + shelf.shelfbook_set.first().shelved_date, make_date(2020, 10, 21) + ) + + readthrough = models.ReadThrough.objects.get(user=self.local_user) + self.assertEqual(readthrough.book, self.book) + self.assertEqual(readthrough.start_date, make_date(2020, 10, 21)) + self.assertEqual(readthrough.finish_date, make_date(2020, 10, 25)) + + def test_create_new_shelf(self, *_): + """import added a book, was a new shelf created?""" + shelf = self.local_user.shelf_set.filter(identifier="cooking").first() + self.assertIsNone(shelf) + + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = models.ImportItem.objects.filter(job=import_job).all()[2] + import_item.book = self.book + import_item.save() + + # this doesn't pick up 'shelf_name' when running all tests + # but does when only running this test...???? + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + handle_imported_book(import_item) + + shelf_after = self.local_user.shelf_set.filter(identifier="cooking").first() + self.assertEqual(shelf_after.books.first(), self.book) + + @patch("bookwyrm.activitystreams.add_status_task.delay") + def test_handle_imported_book_review(self, *_): + """review import""" + import_job = self.importer.create_job( + self.local_user, self.csv, True, "unlisted" + ) + import_item = import_job.items.get(index=2) + import_item.book = self.book + import_item.save() + + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + handle_imported_book(import_item) + + review = models.Review.objects.get(book=self.book, user=self.local_user) + self.assertEqual(review.content, "mixed feelings") + self.assertEqual(review.rating, 2) + self.assertEqual(review.published_date, make_date(2019, 7, 8)) + self.assertEqual(review.privacy, "unlisted") + + @patch("bookwyrm.activitystreams.add_status_task.delay") + def test_handle_imported_book_rating(self, *_): + """rating import""" + import_job = self.importer.create_job( + self.local_user, self.csv, True, "unlisted" + ) + import_item = import_job.items.filter(index=0).first() + import_item.book = self.book + import_item.save() + + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + handle_imported_book(import_item) + + review = models.ReviewRating.objects.get(book=self.book, user=self.local_user) + self.assertIsInstance(review, models.ReviewRating) + self.assertEqual(review.rating, 3) + self.assertEqual(review.published_date, make_date(2020, 10, 25)) + self.assertEqual(review.privacy, "unlisted") diff --git a/bookwyrm/views/imports/import_data.py b/bookwyrm/views/imports/import_data.py index 839c5cc8b..59686c1f0 100644 --- a/bookwyrm/views/imports/import_data.py +++ b/bookwyrm/views/imports/import_data.py @@ -16,6 +16,7 @@ from django.views import View from bookwyrm import forms, models from bookwyrm.importers import ( BookwyrmImporter, + BookwyrmBooksImporter, CalibreImporter, LibrarythingImporter, GoodreadsImporter, @@ -105,8 +106,8 @@ class Import(View): request.user, TextIOWrapper(request.FILES["csv_file"], encoding=importer.encoding), include_reviews, - create_shelves, privacy, + create_shelves, ) except (UnicodeDecodeError, ValueError, KeyError): return self.get(request, invalid=True)