diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 68ff2a48..e6372438 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -26,6 +26,7 @@ class AbstractMinimalConnector(ABC): 'books_url', 'covers_url', 'search_url', + 'isbn_search_url', 'max_query_count', 'name', 'identifier', @@ -61,6 +62,30 @@ class AbstractMinimalConnector(ABC): results.append(self.format_search_result(doc)) return results + def isbn_search(self, query): + ''' isbn search ''' + params = {} + resp = requests.get( + '%s%s' % (self.isbn_search_url, query), + params=params, + headers={ + 'Accept': 'application/json; charset=utf-8', + 'User-Agent': settings.USER_AGENT, + }, + ) + if not resp.ok: + resp.raise_for_status() + try: + data = resp.json() + except ValueError as e: + logger.exception(e) + raise ConnectorException('Unable to parse json response', e) + results = [] + + for doc in self.parse_isbn_search_data(data): + results.append(self.format_isbn_search_result(doc)) + return results + @abstractmethod def get_or_create_book(self, remote_id): ''' pull up a book record by whatever means possible ''' @@ -73,6 +98,14 @@ class AbstractMinimalConnector(ABC): def format_search_result(self, search_result): ''' create a SearchResult obj from json ''' + @abstractmethod + def parse_isbn_search_data(self, data): + ''' turn the result json from a search into a list ''' + + @abstractmethod + def format_isbn_search_result(self, search_result): + ''' create a SearchResult obj from json ''' + class AbstractConnector(AbstractMinimalConnector): ''' generic book data connector ''' diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index 00e6c62f..96b72f26 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -19,3 +19,11 @@ class Connector(AbstractMinimalConnector): def format_search_result(self, search_result): search_result['connector'] = self return SearchResult(**search_result) + + def parse_isbn_search_data(self, data): + return data + + def format_isbn_search_result(self, search_result): + search_result['connector'] = self + return SearchResult(**search_result) + diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index a63a788e..053e1f9e 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -1,5 +1,6 @@ ''' interface with whatever connectors the app has ''' import importlib +import re from urllib.parse import urlparse from requests import HTTPError @@ -15,13 +16,31 @@ class ConnectorException(HTTPError): def search(query, min_confidence=0.1): ''' find books based on arbitary keywords ''' results = [] + + # Have we got a ISBN ? + isbn = re.sub('[\W_]', '', query) + maybe_isbn = len(isbn) in [10, 13] # ISBN10 or ISBN13 + dedup_slug = lambda r: '%s/%s/%s' % (r.title, r.author, r.year) result_index = set() for connector in get_connectors(): - try: - result_set = connector.search(query, min_confidence=min_confidence) - except (HTTPError, ConnectorException): - continue + result_set = None + if maybe_isbn: + # Search on ISBN + if not connector.isbn_search_url or connector.isbn_search_url == '': + result_set = [] + else: + try: + result_set = connector.isbn_search(isbn) + except (HTTPError, ConnectorException): + pass + + # if no isbn search or results, we fallback to generic search + if result_set == None or result_set == []: + try: + result_set = connector.search(query, min_confidence=min_confidence) + except (HTTPError, ConnectorException): + continue result_set = [r for r in result_set \ if dedup_slug(r) not in result_index] @@ -41,6 +60,12 @@ def local_search(query, min_confidence=0.1, raw=False): return connector.search(query, min_confidence=min_confidence, raw=raw) +def isbn_local_search(query, raw=False): + ''' only look at local search results ''' + connector = load_connector(models.Connector.objects.get(local=True)) + return connector.isbn_search(query, raw=raw) + + def first_search_result(query, min_confidence=0.1): ''' search until you find a result that fits ''' for connector in get_connectors(): diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index a767a45a..8d227eef 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -129,6 +129,22 @@ class Connector(AbstractConnector): ) + def parse_isbn_search_data(self, data): + return list(data.values()) + + def format_isbn_search_result(self, search_result): + # build the remote id from the openlibrary key + key = self.books_url + search_result['key'] + authors = search_result.get('authors') or [{'name': 'Unknown'}] + author_names = [ author.get('name') for author in authors] + return SearchResult( + title=search_result.get('title'), + key=key, + author=', '.join(author_names), + connector=self, + year=search_result.get('publish_date'), + ) + def load_edition_data(self, olkey): ''' query openlibrary for editions of a work ''' url = '%s/works/%s/editions' % (self.books_url, olkey) diff --git a/bookwyrm/connectors/self_connector.py b/bookwyrm/connectors/self_connector.py index f57fbc1c..b3a4d6f9 100644 --- a/bookwyrm/connectors/self_connector.py +++ b/bookwyrm/connectors/self_connector.py @@ -33,6 +33,31 @@ class Connector(AbstractConnector): search_results.sort(key=lambda r: r.confidence, reverse=True) return search_results + def isbn_search(self, query, raw=False): + ''' search your local database ''' + if not query: + return [] + + filters = [{f: query} for f in ['isbn_10', 'isbn_13']] + results = models.Edition.objects.filter( + reduce(operator.or_, (Q(**f) for f in filters)) + ).distinct() + + # when there are multiple editions of the same work, pick the default. + # it would be odd for this to happen. + results = results.filter(parent_work__default_edition__id=F('id')) \ + or results + + search_results = [] + for result in results: + if raw: + search_results.append(result) + else: + search_results.append(self.format_search_result(result)) + if len(search_results) >= 10: + break + return search_results + def format_search_result(self, search_result): return SearchResult( @@ -47,6 +72,19 @@ class Connector(AbstractConnector): ) + def format_isbn_search_result(self, search_result): + return SearchResult( + title=search_result.title, + key=search_result.remote_id, + author=search_result.author_text, + year=search_result.published_date.year if \ + search_result.published_date else None, + connector=self, + confidence=search_result.rank if \ + hasattr(search_result, 'rank') else 1, + ) + + def is_work_data(self, data): pass @@ -59,6 +97,10 @@ class Connector(AbstractConnector): def get_authors_from_data(self, data): return None + def parse_isbn_search_data(self, data): + ''' it's already in the right format, don't even worry about it ''' + return data + def parse_search_data(self, data): ''' it's already in the right format, don't even worry about it ''' return data diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py index 9fd11787..5759abfc 100644 --- a/bookwyrm/management/commands/initdb.py +++ b/bookwyrm/management/commands/initdb.py @@ -66,6 +66,7 @@ def init_connectors(): books_url='https://%s/book' % DOMAIN, covers_url='https://%s/images/covers' % DOMAIN, search_url='https://%s/search?q=' % DOMAIN, + isbn_search_url='https://%s/isbn/' % DOMAIN, priority=1, ) @@ -77,6 +78,7 @@ def init_connectors(): books_url='https://bookwyrm.social/book', covers_url='https://bookwyrm.social/images/covers', search_url='https://bookwyrm.social/search?q=', + isbn_search_url='https://bookwyrm.social/isbn/', priority=2, ) @@ -88,6 +90,7 @@ def init_connectors(): books_url='https://openlibrary.org', covers_url='https://covers.openlibrary.org', search_url='https://openlibrary.org/search?q=', + isbn_search_url='https://openlibrary.org/api/books?jscmd=data&format=json&bibkeys=ISBN:', priority=3, ) diff --git a/bookwyrm/migrations/0047_connector_isbn_search_url.py b/bookwyrm/migrations/0047_connector_isbn_search_url.py new file mode 100644 index 00000000..617a89d9 --- /dev/null +++ b/bookwyrm/migrations/0047_connector_isbn_search_url.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.7 on 2021-02-28 16:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0046_sitesettings_privacy_policy'), + ] + + operations = [ + migrations.AddField( + model_name='connector', + name='isbn_search_url', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/bookwyrm/models/connector.py b/bookwyrm/models/connector.py index 6f64cdf3..c1fbf58b 100644 --- a/bookwyrm/models/connector.py +++ b/bookwyrm/models/connector.py @@ -22,6 +22,7 @@ class Connector(BookWyrmModel): books_url = models.CharField(max_length=255) covers_url = models.CharField(max_length=255) search_url = models.CharField(max_length=255, null=True, blank=True) + isbn_search_url = models.CharField(max_length=255, null=True, blank=True) politeness_delay = models.IntegerField(null=True, blank=True) #seconds max_query_count = models.IntegerField(null=True, blank=True) diff --git a/bookwyrm/templates/isbn_search_results.html b/bookwyrm/templates/isbn_search_results.html new file mode 100644 index 00000000..a3861a68 --- /dev/null +++ b/bookwyrm/templates/isbn_search_results.html @@ -0,0 +1,33 @@ +{% extends 'layout.html' %} +{% load i18n %} + +{% block title %}{% trans "Search Results" %}{% endblock %} + +{% block content %} +{% with book_results|first as local_results %} +
{% blocktrans %}No books found for "{{ query }}"{% endblocktrans %}
+ {% else %} +