From 6e218a85defbfd7e965cfbf9857ed94e7f8f134f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 29 Apr 2020 10:09:14 -0700 Subject: [PATCH 1/5] Adds more fields to book data --- fedireads/activitypub/book.py | 3 +- fedireads/connectors/openlibrary.py | 6 ++-- .../migrations/0035_auto_20200429_1708.py | 33 +++++++++++++++++++ fedireads/models/book.py | 9 +++-- fedireads/models/import_job.py | 2 +- fedireads/templates/edit_book.html | 2 +- fedireads/views.py | 2 +- 7 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 fedireads/migrations/0035_auto_20200429_1708.py diff --git a/fedireads/activitypub/book.py b/fedireads/activitypub/book.py index a1a0a637f..cbaa7a047 100644 --- a/fedireads/activitypub/book.py +++ b/fedireads/activitypub/book.py @@ -7,13 +7,12 @@ def get_book(book): fields = [ 'sort_title', 'subtitle', - 'isbn', + 'isbn_13', 'oclc_number', 'openlibrary_key', 'librarything_key', 'fedireads_key', 'lccn', - 'isbn', 'oclc_number', 'pages', 'physical_format', diff --git a/fedireads/connectors/openlibrary.py b/fedireads/connectors/openlibrary.py index 7b34e5f76..54a191483 100644 --- a/fedireads/connectors/openlibrary.py +++ b/fedireads/connectors/openlibrary.py @@ -19,7 +19,7 @@ class Connector(AbstractConnector): 'publish_date': ('published_date', get_date), 'first_publish_date': ('first_published_date', get_date), 'description': ('description', get_description), - 'isbn_13': ('isbn', get_first), + 'isbn_13': ('isbn_13', get_first), 'oclc_numbers': ('oclc_number', get_first), 'lccn': ('lccn', get_first), 'languages': ('languages', get_languages), @@ -118,8 +118,10 @@ class Connector(AbstractConnector): update_from_mappings(book, data, self.book_mappings) book.save() - for author in self.get_authors_from_data(data): + authors = self.get_authors_from_data(data) + for author in authors: book.authors.add(author) + book.author_text = ', '.join(a.name for a in authors) if data.get('covers'): book.cover.save(*self.get_cover(data['covers'][0]), save=True) diff --git a/fedireads/migrations/0035_auto_20200429_1708.py b/fedireads/migrations/0035_auto_20200429_1708.py new file mode 100644 index 000000000..e8d6ed301 --- /dev/null +++ b/fedireads/migrations/0035_auto_20200429_1708.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.3 on 2020-04-29 17:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fedireads', '0034_importjob_import_status'), + ] + + operations = [ + migrations.RenameField( + model_name='edition', + old_name='isbn', + new_name='isbn_13', + ), + migrations.AddField( + model_name='book', + name='author_text', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='edition', + name='asin', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='edition', + name='isbn_10', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/fedireads/models/book.py b/fedireads/models/book.py index fd21e2def..e501fc6b4 100644 --- a/fedireads/models/book.py +++ b/fedireads/models/book.py @@ -82,6 +82,8 @@ class Book(FedireadsModel): ) # TODO: include an annotation about the type of authorship (ie, translator) authors = models.ManyToManyField('Author') + # preformatted authorship string for search and easier display + author_text = models.CharField(max_length=255, blank=True, null=True) cover = models.ImageField(upload_to='covers/', blank=True, null=True) first_published_date = models.DateTimeField(blank=True, null=True) published_date = models.DateTimeField(blank=True, null=True) @@ -123,10 +125,13 @@ class Work(Book): class Edition(Book): ''' an edition of a book ''' + # default -> this is what gets displayed for a work default = models.BooleanField(default=False) - # these identifiers only apply to work - isbn = models.CharField(max_length=255, blank=True, null=True) + # these identifiers only apply to editions, not works + isbn_10 = models.CharField(max_length=255, blank=True, null=True) + isbn_13 = models.CharField(max_length=255, blank=True, null=True) oclc_number = models.CharField(max_length=255, blank=True, null=True) + asin = models.CharField(max_length=255, blank=True, null=True) pages = models.IntegerField(blank=True, null=True) physical_format = models.CharField(max_length=255, blank=True, null=True) publishers = ArrayField( diff --git a/fedireads/models/import_job.py b/fedireads/models/import_job.py index 2465fe12a..8e61ed00a 100644 --- a/fedireads/models/import_job.py +++ b/fedireads/models/import_job.py @@ -61,7 +61,7 @@ class ImportItem(models.Model): def get_book_from_db_isbn(self): ''' see if we already know about the book ''' try: - return Edition.objects.get(isbn=self.isbn) + return Edition.objects.get(isbn_13=self.isbn) except Edition.DoesNotExist: return None diff --git a/fedireads/templates/edit_book.html b/fedireads/templates/edit_book.html index ed6414823..053cecd6c 100644 --- a/fedireads/templates/edit_book.html +++ b/fedireads/templates/edit_book.html @@ -40,7 +40,7 @@

Book Identifiers

-

{{ form.isbn }}

+

{{ form.isbn_13 }}

{{ form.fedireads_key }}

{{ form.openlibrary_key }}

{{ form.librarything_key }}

diff --git a/fedireads/views.py b/fedireads/views.py index ff76166a5..95b707f27 100644 --- a/fedireads/views.py +++ b/fedireads/views.py @@ -431,7 +431,7 @@ def book_page(request, book_identifier, tab='friends'): 'path': '/book/%s' % book_identifier, 'cover_form': forms.CoverForm(instance=book), 'info_fields': [ - {'name': 'ISBN', 'value': book.isbn}, + {'name': 'ISBN', 'value': book.isbn_13}, {'name': 'OCLC number', 'value': book.oclc_number}, {'name': 'OpenLibrary ID', 'value': book.openlibrary_key}, {'name': 'Goodreads ID', 'value': book.goodreads_key}, From e3525b13f5d3ed1cca6c34373df78e7f52fa6831 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 29 Apr 2020 10:57:20 -0700 Subject: [PATCH 2/5] Adds fulltext search of postgres --- fedireads/books_manager.py | 31 ++++++++++++++++++++--- fedireads/connectors/self_connector.py | 35 ++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/fedireads/books_manager.py b/fedireads/books_manager.py index 0821269ec..9810dc8d3 100644 --- a/fedireads/books_manager.py +++ b/fedireads/books_manager.py @@ -31,8 +31,21 @@ def load_more_data(book_id): def search(query): ''' try an external datasource for books ''' + self = self_connector() + results = self.search(query) + if len(results) >= 10: + return results + connector = get_connector() - return connector.search(query) + external_results = connector.search(query) + dedupe_slug = lambda r: '%s %s %s' % (r.title, r.author, r.year) + result_index = [dedupe_slug(r) for r in results] + for result in external_results: + if dedupe_slug(result) in result_index: + continue + results.append(result) + + return results def update_book(book): ''' re-sync with the original data source ''' @@ -40,12 +53,24 @@ def update_book(book): connector.update_book(book) -def get_connector(book=None): +def self_connector(): + ''' load the connector for the local database ''' + return get_connector(self=True) + + +def get_connector(book=None, self=False): ''' pick a book data connector ''' if book and book.connector: connector_info = book.connector + elif self: + connector_info = models.Connector.objects.filter( + connector_file='self_connector' + ).first() else: - connector_info = models.Connector.objects.first() + # only select from external connectors + connector_info = models.Connector.objects.exclude( + connector_file='self_connector' + ).first() connector = importlib.import_module( 'fedireads.connectors.%s' % connector_info.connector_file diff --git a/fedireads/connectors/self_connector.py b/fedireads/connectors/self_connector.py index d44dac607..86cd1d385 100644 --- a/fedireads/connectors/self_connector.py +++ b/fedireads/connectors/self_connector.py @@ -1,8 +1,9 @@ ''' using a fedireads instance as a source of book data ''' +from django.contrib.postgres.search import SearchVector from django.core.exceptions import ObjectDoesNotExist from fedireads import models -from .abstract_connector import AbstractConnector +from .abstract_connector import AbstractConnector, SearchResult class Connector(AbstractConnector): @@ -14,7 +15,33 @@ class Connector(AbstractConnector): def search(self, query): ''' right now you can't search fedireads sorry, but when that gets implemented it will totally rule ''' - return [] + results = models.Edition.objects.annotate( + search=SearchVector('title', weight='A') +\ + SearchVector('subtitle', weight='B') +\ + SearchVector('author_text', weight='A') +\ + SearchVector('isbn_13', weight='A') +\ + SearchVector('isbn_10', weight='A') +\ + SearchVector('openlibrary_key', weight='B') +\ + SearchVector('goodreads_key', weight='B') +\ + SearchVector('source_url', weight='B') +\ + SearchVector('asin', weight='B') +\ + SearchVector('oclc_number', weight='B') +\ + SearchVector('description', weight='C') +\ + SearchVector('series', weight='C') + ).filter(search=query) + + search_results = [] + for book in results[:10]: + search_results.append( + SearchResult( + book.title, + book.fedireads_key, + book.author_text, + book.published_date.year if book.published_date else None, + None + ) + ) + return search_results def get_or_create_book(self, fedireads_key): @@ -38,3 +65,7 @@ class Connector(AbstractConnector): def update_book(self, book_obj): pass + + + def expand_book_data(self, book): + pass From f7b8453594a3b19c5185608b62903377b121126c Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 29 Apr 2020 11:08:51 -0700 Subject: [PATCH 3/5] Set author text when data is inherited from works --- fedireads/connectors/openlibrary.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fedireads/connectors/openlibrary.py b/fedireads/connectors/openlibrary.py index 54a191483..c42cf2630 100644 --- a/fedireads/connectors/openlibrary.py +++ b/fedireads/connectors/openlibrary.py @@ -98,6 +98,7 @@ class Connector(AbstractConnector): edition.save() if not edition.authors and work.authors: edition.authors.set(work.authors.all()) + edition.author_text = ', '.join(a.name for a in edition.authors) return edition @@ -121,7 +122,8 @@ class Connector(AbstractConnector): authors = self.get_authors_from_data(data) for author in authors: book.authors.add(author) - book.author_text = ', '.join(a.name for a in authors) + if authors: + book.author_text = ', '.join(a.name for a in authors) if data.get('covers'): book.cover.save(*self.get_cover(data['covers'][0]), save=True) From 99adb2ef385e68b04b80c4ffaf4f91918b8b8e15 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 29 Apr 2020 11:21:36 -0700 Subject: [PATCH 4/5] Mark default editions --- fedireads/connectors/openlibrary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fedireads/connectors/openlibrary.py b/fedireads/connectors/openlibrary.py index c42cf2630..62d558a6a 100644 --- a/fedireads/connectors/openlibrary.py +++ b/fedireads/connectors/openlibrary.py @@ -80,6 +80,7 @@ class Connector(AbstractConnector): edition_data = pick_default_edition(edition_options) key = edition_data.get('key').split('/')[-1] edition = self.create_book(key, edition_data, models.Edition) + edition.default = True edition.parent_work = work edition.save() else: From 3f437069bd773cadfcedaadb2434f605097465c2 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 29 Apr 2020 11:22:29 -0700 Subject: [PATCH 5/5] Prefer to only show default editions of works --- fedireads/connectors/self_connector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fedireads/connectors/self_connector.py b/fedireads/connectors/self_connector.py index 86cd1d385..e82b4f08c 100644 --- a/fedireads/connectors/self_connector.py +++ b/fedireads/connectors/self_connector.py @@ -29,6 +29,7 @@ class Connector(AbstractConnector): SearchVector('description', weight='C') +\ SearchVector('series', weight='C') ).filter(search=query) + results = results.filter(default=True) or results search_results = [] for book in results[:10]: