diff --git a/fedireads/books_manager.py b/fedireads/books_manager.py index 9810dc8d..28effe7e 100644 --- a/fedireads/books_manager.py +++ b/fedireads/books_manager.py @@ -30,48 +30,60 @@ def load_more_data(book_id): def search(query): - ''' try an external datasource for books ''' - self = self_connector() - results = self.search(query) - if len(results) >= 10: - return results + ''' find books based on arbitary keywords ''' + results = [] + dedup_slug = lambda r: '%s/%s/%s' % (r.title, r.author, r.year) + result_index = set() + for connector in get_connectors(): + result_set = connector.search(query) - connector = get_connector() - external_results = connector.search(query) - dedupe_slug = lambda r: '%s %s %s' % (r.title, r.author, r.year) - result_index = [dedupe_slug(r) for r in results] - for result in external_results: - if dedupe_slug(result) in result_index: - continue - results.append(result) + result_set = [r for r in result_set \ + if dedup_slug(r) not in result_index] + # `|=` concats two sets. WE ARE GETTING FANCY HERE + result_index |= set(dedup_slug(r) for r in result_set) + results.append({ + 'connector': connector, + 'results': result_set, + }) return results + +def first_search_result(query): + ''' search until you find a result that fits ''' + for connector in get_connectors(): + result = connector.search(query) + if result: + return result[0] + return None + + def update_book(book): ''' re-sync with the original data source ''' connector = get_connector(book) connector.update_book(book) -def self_connector(): - ''' load the connector for the local database ''' - return get_connector(self=True) +def get_connectors(): + ''' load all connectors ''' + connectors_info = models.Connector.objects.order_by('priority').all() + return [load_connector(c) for c in connectors_info] -def get_connector(book=None, self=False): +def get_connector(book=None): ''' pick a book data connector ''' if book and book.connector: connector_info = book.connector - elif self: - connector_info = models.Connector.objects.filter( - connector_file='self_connector' - ).first() else: # only select from external connectors - connector_info = models.Connector.objects.exclude( - connector_file='self_connector' - ).first() + connector_info = models.Connector.objects.filter( + local=False + ).order_by('priority').first() + return load_connector(connector_info) + +def load_connector(connector_info): + ''' instantiate the connector class ''' connector = importlib.import_module( 'fedireads.connectors.%s' % connector_info.connector_file ) diff --git a/fedireads/connectors/abstract_connector.py b/fedireads/connectors/abstract_connector.py index 6eb864a0..bba5cda8 100644 --- a/fedireads/connectors/abstract_connector.py +++ b/fedireads/connectors/abstract_connector.py @@ -14,11 +14,14 @@ class AbstractConnector(ABC): info = models.Connector.objects.get(identifier=identifier) self.connector = info - self.url = info.base_url + self.base_url = info.base_url + self.books_url = info.books_url self.covers_url = info.covers_url self.search_url = info.search_url self.key_name = info.key_name self.max_query_count = info.max_query_count + self.name = info.name + self.local = info.local def is_available(self): diff --git a/fedireads/connectors/openlibrary.py b/fedireads/connectors/openlibrary.py index 473780dd..72be68fd 100644 --- a/fedireads/connectors/openlibrary.py +++ b/fedireads/connectors/openlibrary.py @@ -166,7 +166,7 @@ class Connector(AbstractConnector): def load_book_data(self, olkey): ''' query openlibrary for data on a book ''' - response = requests.get('%s/works/%s.json' % (self.url, olkey)) + response = requests.get('%s/works/%s.json' % (self.books_url, olkey)) if not response.ok: response.raise_for_status() data = response.json() @@ -176,7 +176,7 @@ class Connector(AbstractConnector): def load_edition_data(self, olkey): ''' query openlibrary for editions of a work ''' response = requests.get( - '%s/works/%s/editions.json' % (self.url, olkey)) + '%s/works/%s/editions.json' % (self.books_url, olkey)) if not response.ok: response.raise_for_status() data = response.json() @@ -209,7 +209,7 @@ class Connector(AbstractConnector): except models.Author.DoesNotExist: pass - response = requests.get('%s/authors/%s.json' % (self.url, olkey)) + response = requests.get('%s/authors/%s.json' % (self.base_url, olkey)) if not response.ok: response.raise_for_status() diff --git a/fedireads/migrations/0036_auto_20200503_2007.py b/fedireads/migrations/0036_auto_20200503_2007.py new file mode 100644 index 00000000..98327371 --- /dev/null +++ b/fedireads/migrations/0036_auto_20200503_2007.py @@ -0,0 +1,39 @@ +# Generated by Django 3.0.3 on 2020-05-03 20:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fedireads', '0035_auto_20200429_1708'), + ] + + operations = [ + migrations.AddField( + model_name='connector', + name='books_url', + field=models.CharField(default='https://openlibrary.org', max_length=255), + preserve_default=False, + ), + migrations.AddField( + model_name='connector', + name='local', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='connector', + name='name', + field=models.CharField(max_length=255, null=True), + ), + migrations.AddField( + model_name='connector', + name='priority', + field=models.IntegerField(default=2), + ), + migrations.AlterField( + model_name='connector', + name='connector_file', + field=models.CharField(choices=[('openlibrary', 'Openlibrary'), ('self_connector', 'Self Connector'), ('fedireads_connector', 'Fedireads Connector')], max_length=255), + ), + ] diff --git a/fedireads/models/book.py b/fedireads/models/book.py index e501fc6b..2585d92f 100644 --- a/fedireads/models/book.py +++ b/fedireads/models/book.py @@ -16,14 +16,17 @@ ConnectorFiles = models.TextChoices('ConnectorFiles', CONNECTORS) class Connector(FedireadsModel): ''' book data source connectors ''' identifier = models.CharField(max_length=255, unique=True) + priority = models.IntegerField(default=2) + name = models.CharField(max_length=255, null=True) + local = models.BooleanField(default=False) connector_file = models.CharField( max_length=255, - default='openlibrary', choices=ConnectorFiles.choices ) api_key = models.CharField(max_length=255, null=True) base_url = models.CharField(max_length=255) + books_url = models.CharField(max_length=255) covers_url = models.CharField(max_length=255) search_url = models.CharField(max_length=255, null=True) diff --git a/fedireads/models/import_job.py b/fedireads/models/import_job.py index 72ff2e73..9836bea9 100644 --- a/fedireads/models/import_job.py +++ b/fedireads/models/import_job.py @@ -53,23 +53,15 @@ class ImportItem(models.Model): def resolve(self): ''' try various ways to lookup a book ''' self.book = ( - self.get_book_from_db_isbn() or self.get_book_from_isbn() or self.get_book_from_title_author() ) - def get_book_from_db_isbn(self): - ''' see if we already know about the book ''' - try: - return Edition.objects.filter(isbn_13=self.isbn).first() - except Edition.DoesNotExist: - return None - def get_book_from_isbn(self): ''' search by isbn ''' - search_results = books_manager.search(self.isbn) - if search_results: - return books_manager.get_or_create_book(search_results[0].key) + search_result = books_manager.first_search_result(self.isbn) + if search_result: + return books_manager.get_or_create_book(search_result.key) def get_book_from_title_author(self): ''' search by title and author ''' @@ -77,9 +69,9 @@ class ImportItem(models.Model): self.data['Title'], self.data['Author'] ) - search_results = books_manager.search(search_term) - if search_results: - return books_manager.get_or_create_book(search_results[0].key) + search_result = books_manager.first_search_result(search_term) + if search_result: + return books_manager.get_or_create_book(search_result.key) @property def isbn(self): diff --git a/fedireads/static/format.css b/fedireads/static/format.css index 89459c2f..59a27c97 100644 --- a/fedireads/static/format.css +++ b/fedireads/static/format.css @@ -45,6 +45,10 @@ h3 small { font-weight: normal; } +section { + margin-bottom: 1em; +} + /* fixed display top bar */ body { diff --git a/fedireads/templates/book_results.html b/fedireads/templates/book_results.html index bb7fa735..fdba919b 100644 --- a/fedireads/templates/book_results.html +++ b/fedireads/templates/book_results.html @@ -1,13 +1,21 @@ {% extends 'layout.html' %} {% block content %} -