diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index b0458bab..f6abef0e 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -28,8 +28,8 @@ class Importer: "isbn_13": ["isbn13", "isbn"], "isbn_10": ["isbn10", "isbn"], "shelf": ["shelf", "exclusive shelf", "read status"], - "review_name": [], - "review_body": ["my review"], + "review_name": ["review name"], + "review_body": ["my review", "review"], "rating": ["my rating", "rating", "star rating"], "date_added": ["date added", "entry date", "added"], "date_started": ["date started", "started"], @@ -48,7 +48,6 @@ class Importer: ) for index, entry in rows: - print(index, entry) self.create_item(job, index, entry) return job @@ -65,7 +64,6 @@ class Importer: def create_item(self, job, index, data): """creates and saves an import item""" - print(data) normalized = self.normalize_row(data, job.mappings) ImportItem(job=job, index=index, data=data, normalized_data=normalized).save() diff --git a/bookwyrm/tests/data/goodreads-rating.csv b/bookwyrm/tests/data/goodreads-rating.csv deleted file mode 100644 index fec0c77d..00000000 --- a/bookwyrm/tests/data/goodreads-rating.csv +++ /dev/null @@ -1,5 +0,0 @@ -Book Id,Title,Author,Author l-f,Additional Authors,ISBN,ISBN13,My Rating,Average Rating,Publisher,Binding,Number of Pages,Year Published,Original Publication Year,Date Read,Date Added,Bookshelves,Bookshelves with positions,Exclusive Shelf,My Review,Spoiler,Private Notes,Read Count,Recommended For,Recommended By,Owned Copies,Original Purchase Date,Original Purchase Location,Condition,Condition Description,BCID -42036538,Gideon the Ninth (The Locked Tomb #1),Tamsyn Muir,"Muir, Tamsyn",,"=""1250313198""","=""9781250313195""",0,4.20,Tor,Hardcover,448,2019,2019,2020/10/25,2020/10/21,,,read,,,,1,,,0,,,,, -52691223,Subcutanean,Aaron A. Reed,"Reed, Aaron A.",,"=""""","=""""",0,4.45,,Paperback,232,2020,,2020/03/06,2020/03/05,,,read,,,,1,,,0,,,,, -28694510,Patisserie at Home,Mélanie Dupuis,"Dupuis, Mélanie",Anne Cazor,"=""0062445316""","=""9780062445315""",2,4.60,Harper Design,Hardcover,288,2016,,,2019/07/08,,,read,,,,2,,,0,,,,, - diff --git a/bookwyrm/tests/data/goodreads.csv b/bookwyrm/tests/data/goodreads.csv index 5f124edc..a0a8232e 100644 --- a/bookwyrm/tests/data/goodreads.csv +++ b/bookwyrm/tests/data/goodreads.csv @@ -1,4 +1,4 @@ Book Id,Title,Author,Author l-f,Additional Authors,ISBN,ISBN13,My Rating,Average Rating,Publisher,Binding,Number of Pages,Year Published,Original Publication Year,Date Read,Date Added,Bookshelves,Bookshelves with positions,Exclusive Shelf,My Review,Spoiler,Private Notes,Read Count,Recommended For,Recommended By,Owned Copies,Original Purchase Date,Original Purchase Location,Condition,Condition Description,BCID -42036538,Gideon the Ninth (The Locked Tomb #1),Tamsyn Muir,"Muir, Tamsyn",,"=""1250313198""","=""9781250313195""",0,4.20,Tor,Hardcover,448,2019,2019,2020/10/25,2020/10/21,,,read,,,,1,,,0,,,,, +42036538,Gideon the Ninth (The Locked Tomb #1),Tamsyn Muir,"Muir, Tamsyn",,"=""1250313198""","=""9781250313195""",3,4.20,Tor,Hardcover,448,2019,2019,2020/10/25,2020/10/21,,,read,,,,1,,,0,,,,, 52691223,Subcutanean,Aaron A. Reed,"Reed, Aaron A.",,"=""""","=""""",0,4.45,,Paperback,232,2020,,2020/03/06,2020/03/05,,,read,,,,1,,,0,,,,, 28694510,Patisserie at Home,Mélanie Dupuis,"Dupuis, Mélanie",Anne Cazor,"=""0062445316""","=""9780062445315""",2,4.60,Harper Design,Hardcover,288,2016,,,2019/07/08,,,read,"mixed feelings",,,2,,,0,,,,, diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index a2451292..87624a0e 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -1,5 +1,4 @@ """ testing import """ -import csv import pathlib from unittest.mock import patch import datetime @@ -32,7 +31,7 @@ class GoodreadsImport(TestCase): with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( "bookwyrm.activitystreams.populate_stream_task.delay" ): - self.user = models.User.objects.create_user( + self.local_user = models.User.objects.create_user( "mouse", "mouse@mouse.mouse", "password", local=True ) @@ -45,7 +44,9 @@ class GoodreadsImport(TestCase): def test_create_job(self, *_): """creates the import job entry and checks csv""" - import_job = self.importer.create_job(self.user, self.csv, False, "public") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) import_items = models.ImportItem.objects.filter(job=import_job).all() self.assertEqual(len(import_items), 3) @@ -58,12 +59,16 @@ class GoodreadsImport(TestCase): def test_create_retry_job(self, *_): """trying again with items that didn't import""" - import_job = self.importer.create_job(self.user, self.csv, False, "unlisted") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) import_items = models.ImportItem.objects.filter(job=import_job).all()[:2] - retry = self.importer.create_retry_job(self.user, import_job, import_items) + retry = self.importer.create_retry_job( + self.local_user, import_job, import_items + ) self.assertNotEqual(import_job, retry) - self.assertEqual(retry.user, self.user) + self.assertEqual(retry.user, self.local_user) self.assertEqual(retry.include_reviews, False) self.assertEqual(retry.privacy, "unlisted") @@ -76,22 +81,19 @@ class GoodreadsImport(TestCase): def test_handle_imported_book(self, *_): """goodreads import added a book, this adds related connections""" - shelf = self.user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter(identifier="read").first() self.assertIsNone(shelf.books.first()) - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - for index, entry in enumerate(list(csv.DictReader(csv_file))): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book - ) - break + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, False, "public" + self.importer.service, self.local_user, import_item, False, "public" ) shelf.refresh_from_db() @@ -100,7 +102,7 @@ class GoodreadsImport(TestCase): shelf.shelfbook_set.first().shelved_date, make_date(2020, 10, 21) ) - readthrough = models.ReadThrough.objects.get(user=self.user) + readthrough = models.ReadThrough.objects.get(user=self.local_user) self.assertEqual(readthrough.book, self.book) self.assertEqual(readthrough.start_date, make_date(2020, 10, 21)) self.assertEqual(readthrough.finish_date, make_date(2020, 10, 25)) @@ -108,20 +110,16 @@ class GoodreadsImport(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_review(self, *_): """goodreads review import""" - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - entry = list(csv.DictReader(csv_file))[2] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book - ) + import_job = self.importer.create_job(self.local_user, self.csv, True, "public") + import_item = import_job.items.get(index=2) + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, True, "unlisted" + self.importer.service, self.local_user, import_item, True, "unlisted" ) - review = models.Review.objects.get(book=self.book, user=self.user) + review = models.Review.objects.get(book=self.book, user=self.local_user) self.assertEqual(review.content, "mixed feelings") self.assertEqual(review.rating, 2) self.assertEqual(review.published_date, make_date(2019, 7, 8)) @@ -130,23 +128,19 @@ class GoodreadsImport(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_rating(self, *_): """goodreads rating import""" - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath( - "../data/goodreads-rating.csv" - ) - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - entry = list(csv.DictReader(csv_file))[2] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" ) + import_item = import_job.items.filter(index=0).first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, True, "unlisted" + self.importer.service, self.local_user, import_item, True, "unlisted" ) - review = models.ReviewRating.objects.get(book=self.book, user=self.user) + review = models.ReviewRating.objects.get(book=self.book, user=self.local_user) self.assertIsInstance(review, models.ReviewRating) - self.assertEqual(review.rating, 2) - self.assertEqual(review.published_date, make_date(2019, 7, 8)) + self.assertEqual(review.rating, 3) + self.assertEqual(review.published_date, make_date(2020, 10, 25)) self.assertEqual(review.privacy, "unlisted") diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index f76666a7..00bb0ffa 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -35,7 +35,7 @@ class LibrarythingImport(TestCase): with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( "bookwyrm.activitystreams.populate_stream_task.delay" ): - self.user = models.User.objects.create_user( + self.local_user = models.User.objects.create_user( "mmai", "mmai@mmai.mmai", "password", local=True ) work = models.Work.objects.create(title="Test Work") @@ -47,8 +47,10 @@ class LibrarythingImport(TestCase): def test_create_job(self, *_): """creates the import job entry and checks csv""" - import_job = self.importer.create_job(self.user, self.csv, False, "public") - self.assertEqual(import_job.user, self.user) + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + self.assertEqual(import_job.user, self.local_user) self.assertEqual(import_job.include_reviews, False) self.assertEqual(import_job.privacy, "public") @@ -63,12 +65,16 @@ class LibrarythingImport(TestCase): def test_create_retry_job(self, *_): """trying again with items that didn't import""" - import_job = self.importer.create_job(self.user, self.csv, False, "unlisted") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) import_items = models.ImportItem.objects.filter(job=import_job).all()[:2] - retry = self.importer.create_retry_job(self.user, import_job, import_items) + retry = self.importer.create_retry_job( + self.local_user, import_job, import_items + ) self.assertNotEqual(import_job, retry) - self.assertEqual(retry.user, self.user) + self.assertEqual(retry.user, self.local_user) self.assertEqual(retry.include_reviews, False) self.assertEqual(retry.privacy, "unlisted") @@ -82,7 +88,9 @@ class LibrarythingImport(TestCase): @responses.activate def test_import_data(self, *_): """resolve entry""" - import_job = self.importer.create_job(self.user, self.csv, False, "unlisted") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) book = models.Edition.objects.create(title="Test Book") with patch( @@ -97,30 +105,25 @@ class LibrarythingImport(TestCase): def test_handle_imported_book(self, *_): """librarything import added a book, this adds related connections""" - shelf = self.user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter(identifier="read").first() self.assertIsNone(shelf.books.first()) - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/librarything.tsv") - csv_file = open(datafile, "r", encoding=self.importer.encoding) - for index, entry in enumerate( - list(csv.DictReader(csv_file, delimiter=self.importer.delimiter)) - ): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book - ) - break + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, False, "public" + self.importer.service, self.local_user, import_item, False, "public" ) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) - readthrough = models.ReadThrough.objects.get(user=self.user) + readthrough = models.ReadThrough.objects.get(user=self.local_user) self.assertEqual(readthrough.book, self.book) self.assertEqual(readthrough.start_date, make_date(2007, 4, 16)) self.assertEqual(readthrough.finish_date, make_date(2007, 5, 8)) @@ -128,31 +131,30 @@ class LibrarythingImport(TestCase): def test_handle_imported_book_already_shelved(self, *_): """librarything import added a book, this adds related connections""" with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): - shelf = self.user.shelf_set.filter(identifier="to-read").first() - models.ShelfBook.objects.create(shelf=shelf, user=self.user, book=self.book) - - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/librarything.tsv") - csv_file = open(datafile, "r", encoding=self.importer.encoding) - for index, entry in enumerate( - list(csv.DictReader(csv_file, delimiter=self.importer.delimiter)) - ): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book + shelf = self.local_user.shelf_set.filter(identifier="to-read").first() + models.ShelfBook.objects.create( + shelf=shelf, user=self.local_user, book=self.book ) - break + + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, False, "public" + self.importer.service, self.local_user, import_item, False, "public" ) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) - self.assertIsNone(self.user.shelf_set.get(identifier="read").books.first()) + self.assertIsNone( + self.local_user.shelf_set.get(identifier="read").books.first() + ) - readthrough = models.ReadThrough.objects.get(user=self.user) + readthrough = models.ReadThrough.objects.get(user=self.local_user) self.assertEqual(readthrough.book, self.book) self.assertEqual(readthrough.start_date, make_date(2007, 4, 16)) self.assertEqual(readthrough.finish_date, make_date(2007, 5, 8)) @@ -160,20 +162,16 @@ class LibrarythingImport(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_review(self, *_): """librarything review import""" - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/librarything.tsv") - csv_file = open(datafile, "r", encoding=self.importer.encoding) - entry = list(csv.DictReader(csv_file, delimiter=self.importer.delimiter))[0] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book - ) + import_job = self.importer.create_job(self.local_user, self.csv, True, "public") + import_item = import_job.items.filter(index=0).first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, True, "unlisted" + self.importer.service, self.local_user, import_item, True, "unlisted" ) - review = models.Review.objects.get(book=self.book, user=self.user) + review = models.Review.objects.get(book=self.book, user=self.local_user) self.assertEqual(review.content, "chef d'oeuvre") self.assertEqual(review.rating, 5) self.assertEqual(review.published_date, make_date(2007, 5, 8)) diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index addf362c..91a5dfa6 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -32,7 +32,7 @@ class StorygraphImport(TestCase): with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( "bookwyrm.activitystreams.populate_stream_task.delay" ): - self.user = models.User.objects.create_user( + self.local_user = models.User.objects.create_user( "mouse", "mouse@mouse.mouse", "password", local=True ) @@ -45,7 +45,9 @@ class StorygraphImport(TestCase): def test_create_job(self, *_): """creates the import job entry and checks csv""" - import_job = self.importer.create_job(self.user, self.csv, False, "public") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) import_items = models.ImportItem.objects.filter(job=import_job).all() self.assertEqual(len(import_items), 2) @@ -57,12 +59,16 @@ class StorygraphImport(TestCase): def test_create_retry_job(self, *_): """trying again with items that didn't import""" - import_job = self.importer.create_job(self.user, self.csv, False, "unlisted") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) import_items = models.ImportItem.objects.filter(job=import_job).all()[:2] - retry = self.importer.create_retry_job(self.user, import_job, import_items) + retry = self.importer.create_retry_job( + self.local_user, import_job, import_items + ) self.assertNotEqual(import_job, retry) - self.assertEqual(retry.user, self.user) + self.assertEqual(retry.user, self.local_user) self.assertEqual(retry.include_reviews, False) self.assertEqual(retry.privacy, "unlisted") @@ -75,22 +81,19 @@ class StorygraphImport(TestCase): def test_handle_imported_book(self, *_): """storygraph import added a book, this adds related connections""" - shelf = self.user.shelf_set.filter(identifier="to-read").first() + shelf = self.local_user.shelf_set.filter(identifier="to-read").first() self.assertIsNone(shelf.books.first()) - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/storygraph.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - for index, entry in enumerate(list(csv.DictReader(csv_file))): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book - ) - break + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, False, "public" + self.importer.service, self.local_user, import_item, False, "public" ) shelf.refresh_from_db() @@ -102,20 +105,18 @@ class StorygraphImport(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_rating(self, *_): """storygraph rating import""" - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/storygraph.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - entry = list(csv.DictReader(csv_file))[1] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" ) + import_item = import_job.items.filter(index=1).first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, True, "unlisted" + self.importer.service, self.local_user, import_item, True, "unlisted" ) - review = models.ReviewRating.objects.get(book=self.book, user=self.user) + review = models.ReviewRating.objects.get(book=self.book, user=self.local_user) self.assertIsInstance(review, models.ReviewRating) self.assertEqual(review.rating, 5.0) self.assertEqual(review.published_date, make_date(2021, 5, 10))