From 1f06d1a1d8386fcf6aa09441d09b6f2d0dd4941b Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 14 Sep 2021 15:26:18 -0700 Subject: [PATCH 01/19] Removes local connector --- bookwyrm/connectors/__init__.py | 2 +- bookwyrm/connectors/abstract_connector.py | 1 - bookwyrm/connectors/connector_manager.py | 16 +- bookwyrm/connectors/self_connector.py | 164 ------------------ bookwyrm/management/commands/initdb.py | 13 -- .../migrations/0097_remove_connector_local.py | 17 ++ bookwyrm/models/connector.py | 1 - bookwyrm/templates/search/book.html | 19 +- .../snippets/search_result_text.html | 22 +-- bookwyrm/views/search.py | 19 +- 10 files changed, 55 insertions(+), 219 deletions(-) delete mode 100644 bookwyrm/connectors/self_connector.py create mode 100644 bookwyrm/migrations/0097_remove_connector_local.py diff --git a/bookwyrm/connectors/__init__.py b/bookwyrm/connectors/__init__.py index 689f27018..efbdb1666 100644 --- a/bookwyrm/connectors/__init__.py +++ b/bookwyrm/connectors/__init__.py @@ -3,4 +3,4 @@ from .settings import CONNECTORS from .abstract_connector import ConnectorException from .abstract_connector import get_data, get_image -from .connector_manager import search, local_search, first_search_result +from .connector_manager import search, first_search_result diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index ffacffdf0..9f20a6c35 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -31,7 +31,6 @@ class AbstractMinimalConnector(ABC): "isbn_search_url", "name", "identifier", - "local", ] for field in self_fields: setattr(self, field, getattr(info, field)) diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index 1d9588d6b..6f62b59a3 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -55,7 +55,7 @@ def search(query, min_confidence=0.1, return_first=False): # if we found anything, return it return result_set[0] - if result_set or connector.local: + if result_set: results.append( { "connector": connector, @@ -71,20 +71,6 @@ def search(query, min_confidence=0.1, return_first=False): return results -def local_search(query, min_confidence=0.1, raw=False, filters=None): - """only look at local search results""" - connector = load_connector(models.Connector.objects.get(local=True)) - return connector.search( - query, min_confidence=min_confidence, raw=raw, filters=filters - ) - - -def isbn_local_search(query, raw=False): - """only look at local search results""" - connector = load_connector(models.Connector.objects.get(local=True)) - return connector.isbn_search(query, raw=raw) - - def first_search_result(query, min_confidence=0.1): """search until you find a result that fits""" return search(query, min_confidence=min_confidence, return_first=True) or None diff --git a/bookwyrm/connectors/self_connector.py b/bookwyrm/connectors/self_connector.py deleted file mode 100644 index 8d5a7614e..000000000 --- a/bookwyrm/connectors/self_connector.py +++ /dev/null @@ -1,164 +0,0 @@ -""" using a bookwyrm instance as a source of book data """ -from functools import reduce -import operator - -from django.contrib.postgres.search import SearchRank, SearchQuery -from django.db.models import OuterRef, Subquery, F, Q - -from bookwyrm import models -from .abstract_connector import AbstractConnector, SearchResult - - -class Connector(AbstractConnector): - """instantiate a connector""" - - # pylint: disable=arguments-differ - def search(self, query, min_confidence=0, raw=False, filters=None): - """search your local database""" - filters = filters or [] - if not query: - return [] - # first, try searching unqiue identifiers - results = search_identifiers(query, *filters) - if not results: - # then try searching title/author - results = search_title_author(query, min_confidence, *filters) - search_results = [] - for result in results: - if raw: - search_results.append(result) - else: - search_results.append(self.format_search_result(result)) - if len(search_results) >= 10: - break - if not raw: - search_results.sort(key=lambda r: r.confidence, reverse=True) - return search_results - - def isbn_search(self, query, raw=False): - """search your local database""" - if not query: - return [] - - filters = [{f: query} for f in ["isbn_10", "isbn_13"]] - results = models.Edition.objects.filter( - reduce(operator.or_, (Q(**f) for f in filters)) - ).distinct() - - # when there are multiple editions of the same work, pick the default. - # it would be odd for this to happen. - - default_editions = models.Edition.objects.filter( - parent_work=OuterRef("parent_work") - ).order_by("-edition_rank") - results = ( - results.annotate( - default_id=Subquery(default_editions.values("id")[:1]) - ).filter(default_id=F("id")) - or results - ) - - search_results = [] - for result in results: - if raw: - search_results.append(result) - else: - search_results.append(self.format_search_result(result)) - if len(search_results) >= 10: - break - return search_results - - def format_search_result(self, search_result): - cover = None - if search_result.cover: - cover = "%s%s" % (self.covers_url, search_result.cover) - - return SearchResult( - title=search_result.title, - key=search_result.remote_id, - author=search_result.author_text, - year=search_result.published_date.year - if search_result.published_date - else None, - connector=self, - cover=cover, - confidence=search_result.rank if hasattr(search_result, "rank") else 1, - ) - - def format_isbn_search_result(self, search_result): - return self.format_search_result(search_result) - - def is_work_data(self, data): - pass - - def get_edition_from_work_data(self, data): - pass - - def get_work_from_edition_data(self, data): - pass - - def get_authors_from_data(self, data): - return None - - def parse_isbn_search_data(self, data): - """it's already in the right format, don't even worry about it""" - return data - - def parse_search_data(self, data): - """it's already in the right format, don't even worry about it""" - return data - - def expand_book_data(self, book): - pass - - -def search_identifiers(query, *filters): - """tries remote_id, isbn; defined as dedupe fields on the model""" - # pylint: disable=W0212 - or_filters = [ - {f.name: query} - for f in models.Edition._meta.get_fields() - if hasattr(f, "deduplication_field") and f.deduplication_field - ] - results = models.Edition.objects.filter( - *filters, reduce(operator.or_, (Q(**f) for f in or_filters)) - ).distinct() - if results.count() <= 1: - return results - - # when there are multiple editions of the same work, pick the default. - # it would be odd for this to happen. - default_editions = models.Edition.objects.filter( - parent_work=OuterRef("parent_work") - ).order_by("-edition_rank") - return ( - results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter( - default_id=F("id") - ) - or results - ) - - -def search_title_author(query, min_confidence, *filters): - """searches for title and author""" - query = SearchQuery(query, config="simple") | SearchQuery(query, config="english") - results = ( - models.Edition.objects.filter(*filters, search_vector=query) - .annotate(rank=SearchRank(F("search_vector"), query)) - .filter(rank__gt=min_confidence) - .order_by("-rank") - ) - - # when there are multiple editions of the same work, pick the closest - editions_of_work = results.values("parent_work__id").values_list("parent_work__id") - - # filter out multiple editions of the same work - for work_id in set(editions_of_work): - editions = results.filter(parent_work=work_id) - default = editions.order_by("-edition_rank").first() - default_rank = default.rank if default else 0 - # if mutliple books have the top rank, pick the default edition - if default_rank == editions.first().rank: - yield default - else: - yield editions.first() diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py index 71ac511a0..9f2f29cda 100644 --- a/bookwyrm/management/commands/initdb.py +++ b/bookwyrm/management/commands/initdb.py @@ -73,19 +73,6 @@ def init_permissions(): def init_connectors(): """access book data sources""" - Connector.objects.create( - identifier=DOMAIN, - name="Local", - local=True, - connector_file="self_connector", - base_url="https://%s" % DOMAIN, - books_url="https://%s/book" % DOMAIN, - covers_url="https://%s/images/" % DOMAIN, - search_url="https://%s/search?q=" % DOMAIN, - isbn_search_url="https://%s/isbn/" % DOMAIN, - priority=1, - ) - Connector.objects.create( identifier="bookwyrm.social", name="BookWyrm dot Social", diff --git a/bookwyrm/migrations/0097_remove_connector_local.py b/bookwyrm/migrations/0097_remove_connector_local.py new file mode 100644 index 000000000..fc04bccec --- /dev/null +++ b/bookwyrm/migrations/0097_remove_connector_local.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-09-14 22:10 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0096_merge_20210912_0044"), + ] + + operations = [ + migrations.RemoveField( + model_name="connector", + name="local", + ), + ] diff --git a/bookwyrm/models/connector.py b/bookwyrm/models/connector.py index 17ba31489..b982c0ff6 100644 --- a/bookwyrm/models/connector.py +++ b/bookwyrm/models/connector.py @@ -14,7 +14,6 @@ class Connector(BookWyrmModel): identifier = models.CharField(max_length=255, unique=True) priority = models.IntegerField(default=2) name = models.CharField(max_length=255, null=True, blank=True) - local = models.BooleanField(default=False) connector_file = models.CharField(max_length=255, choices=ConnectorFiles.choices) api_key = models.CharField(max_length=255, null=True, blank=True) active = models.BooleanField(default=True) diff --git a/bookwyrm/templates/search/book.html b/bookwyrm/templates/search/book.html index 39f837328..12164bb3f 100644 --- a/bookwyrm/templates/search/book.html +++ b/bookwyrm/templates/search/book.html @@ -8,7 +8,24 @@ diff --git a/bookwyrm/templates/snippets/search_result_text.html b/bookwyrm/templates/snippets/search_result_text.html index 40fa5a3d5..973f1f5bc 100644 --- a/bookwyrm/templates/snippets/search_result_text.html +++ b/bookwyrm/templates/snippets/search_result_text.html @@ -9,10 +9,8 @@ {{ result.title }}

@@ -26,16 +24,14 @@ {% endif %}

- {% if remote_result %} -
- {% csrf_token %} + + {% csrf_token %} - + - -
- {% endif %} + + diff --git a/bookwyrm/views/search.py b/bookwyrm/views/search.py index cdea86631..6c9593a1d 100644 --- a/bookwyrm/views/search.py +++ b/bookwyrm/views/search.py @@ -10,6 +10,7 @@ from django.views import View from bookwyrm import models from bookwyrm.connectors import connector_manager +from bookwyrm.book_search import search from bookwyrm.settings import PAGE_LENGTH from bookwyrm.utils import regex from .helpers import is_api_request, privacy_filter @@ -31,9 +32,7 @@ class Search(View): if is_api_request(request): # only return local book results via json so we don't cascade - book_results = connector_manager.local_search( - query, min_confidence=min_confidence - ) + book_results = search(query, min_confidence=min_confidence) return JsonResponse([r.json() for r in book_results], safe=False) if query and not search_type: @@ -69,13 +68,13 @@ class Search(View): def book_search(query, _, min_confidence, search_remote=False): """the real business is elsewhere""" # try a local-only search - if not search_remote: - results = connector_manager.local_search(query, min_confidence=min_confidence) - if results: - # gret, we found something - return [{"results": results}], False - # if there weere no local results, or the request was for remote, search all sources - return connector_manager.search(query, min_confidence=min_confidence), True + results = [{"results": search(query, min_confidence=min_confidence)}] + if results and not search_remote: + return results, False + + # if there were no local results, or the request was for remote, search all sources + results += connector_manager.search(query, min_confidence=min_confidence) + return results, True def user_search(query, viewer, *_): From 98325818b2295ca4786a8f51c055cea13b28cbfe Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 10:44:33 -0700 Subject: [PATCH 02/19] Display search results in api mode and regular --- bookwyrm/book_search.py | 120 ++++++++++++++++++ bookwyrm/templates/search/book.html | 28 +++- .../snippets/search_result_text.html | 37 ------ bookwyrm/urls.py | 1 + bookwyrm/views/helpers.py | 4 +- bookwyrm/views/search.py | 6 +- 6 files changed, 155 insertions(+), 41 deletions(-) create mode 100644 bookwyrm/book_search.py delete mode 100644 bookwyrm/templates/snippets/search_result_text.html diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py new file mode 100644 index 000000000..8682b83e1 --- /dev/null +++ b/bookwyrm/book_search.py @@ -0,0 +1,120 @@ +""" using a bookwyrm instance as a source of book data """ +from functools import reduce +import operator + +from django.contrib.postgres.search import SearchRank, SearchQuery +from django.db.models import OuterRef, Subquery, F, Q + +from bookwyrm import models +from bookwyrm.connectors.abstract_connector import SearchResult +from bookwyrm.settings import MEDIA_FULL_URL + + +# pylint: disable=arguments-differ +def search(query, min_confidence=0, filters=None): + """search your local database""" + filters = filters or [] + if not query: + return [] + # first, try searching unqiue identifiers + results = search_identifiers(query, *filters) + if not results: + # then try searching title/author + results = search_title_author(query, min_confidence, *filters) + return results + + +def isbn_search(query): + """search your local database""" + if not query: + return [] + + filters = [{f: query} for f in ["isbn_10", "isbn_13"]] + results = models.Edition.objects.filter( + reduce(operator.or_, (Q(**f) for f in filters)) + ).distinct() + + # when there are multiple editions of the same work, pick the default. + # it would be odd for this to happen. + + default_editions = models.Edition.objects.filter( + parent_work=OuterRef("parent_work") + ).order_by("-edition_rank") + results = ( + results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter( + default_id=F("id") + ) + or results + ) + return results + + +def format_search_result(search_result): + """convert a book object into a search result object""" + cover = None + if search_result.cover: + cover = f"{MEDIA_FULL_URL}{search_result.cover}" + + return SearchResult( + title=search_result.title, + key=search_result.remote_id, + author=search_result.author_text, + year=search_result.published_date.year + if search_result.published_date + else None, + cover=cover, + confidence=search_result.rank if hasattr(search_result, "rank") else 1, + connector="", + ).json() + + +def search_identifiers(query, *filters): + """tries remote_id, isbn; defined as dedupe fields on the model""" + # pylint: disable=W0212 + or_filters = [ + {f.name: query} + for f in models.Edition._meta.get_fields() + if hasattr(f, "deduplication_field") and f.deduplication_field + ] + results = models.Edition.objects.filter( + *filters, reduce(operator.or_, (Q(**f) for f in or_filters)) + ).distinct() + if results.count() <= 1: + return results + + # when there are multiple editions of the same work, pick the default. + # it would be odd for this to happen. + default_editions = models.Edition.objects.filter( + parent_work=OuterRef("parent_work") + ).order_by("-edition_rank") + return ( + results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter( + default_id=F("id") + ) + or results + ) + + +def search_title_author(query, min_confidence, *filters): + """searches for title and author""" + query = SearchQuery(query, config="simple") | SearchQuery(query, config="english") + results = ( + models.Edition.objects.filter(*filters, search_vector=query) + .annotate(rank=SearchRank(F("search_vector"), query)) + .filter(rank__gt=min_confidence) + .order_by("-rank") + ) + + # when there are multiple editions of the same work, pick the closest + editions_of_work = results.values("parent_work__id").values_list("parent_work__id") + + # filter out multiple editions of the same work + for work_id in set(editions_of_work): + editions = results.filter(parent_work=work_id) + default = editions.order_by("-edition_rank").first() + default_rank = default.rank if default else 0 + # if mutliple books have the top rank, pick the default edition + if default_rank == editions.first().rank: + yield default + else: + yield editions.first() diff --git a/bookwyrm/templates/search/book.html b/bookwyrm/templates/search/book.html index 12164bb3f..98590f202 100644 --- a/bookwyrm/templates/search/book.html +++ b/bookwyrm/templates/search/book.html @@ -60,7 +60,33 @@ diff --git a/bookwyrm/templates/snippets/search_result_text.html b/bookwyrm/templates/snippets/search_result_text.html deleted file mode 100644 index 973f1f5bc..000000000 --- a/bookwyrm/templates/snippets/search_result_text.html +++ /dev/null @@ -1,37 +0,0 @@ -{% load i18n %} -
-
- {% include 'snippets/book_cover.html' with book=result cover_class='is-w-xs is-h-xs' external_path=True %} -
- -
-

- - {{ result.title }} - -

-

- {% if result.author %} - {{ result.author }} - {% endif %} - - {% if result.year %} - ({{ result.year }}) - {% endif %} -

- -
- {% csrf_token %} - - - - -
-
-
diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 6acf75cb3..15086a666 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -221,6 +221,7 @@ urlpatterns = [ ), # search re_path(r"^search/?$", views.Search.as_view(), name="search"), + re_path(r"^search.json/?$", views.Search.as_view(), name="search"), # imports re_path(r"^import/?$", views.Import.as_view(), name="import"), re_path(r"^import/(\d+)/?$", views.ImportStatus.as_view(), name="import-status"), diff --git a/bookwyrm/views/helpers.py b/bookwyrm/views/helpers.py index b1e5f68c7..aa26ae182 100644 --- a/bookwyrm/views/helpers.py +++ b/bookwyrm/views/helpers.py @@ -32,7 +32,9 @@ def get_user_from_username(viewer, username): def is_api_request(request): """check whether a request is asking for html or data""" - return "json" in request.headers.get("Accept", "") or request.path[-5:] == ".json" + return "json" in request.headers.get("Accept", "") or re.match( + r".*\.json/?$", request.path + ) def is_bookwyrm_request(request): diff --git a/bookwyrm/views/search.py b/bookwyrm/views/search.py index 6c9593a1d..60a1879aa 100644 --- a/bookwyrm/views/search.py +++ b/bookwyrm/views/search.py @@ -10,7 +10,7 @@ from django.views import View from bookwyrm import models from bookwyrm.connectors import connector_manager -from bookwyrm.book_search import search +from bookwyrm.book_search import search, format_search_result from bookwyrm.settings import PAGE_LENGTH from bookwyrm.utils import regex from .helpers import is_api_request, privacy_filter @@ -33,7 +33,9 @@ class Search(View): if is_api_request(request): # only return local book results via json so we don't cascade book_results = search(query, min_confidence=min_confidence) - return JsonResponse([r.json() for r in book_results], safe=False) + return JsonResponse( + [format_search_result(r) for r in book_results], safe=False + ) if query and not search_type: search_type = "user" if "@" in query else "book" From 0d5e05a3c2ca57922b45db4dfa5f47a842d66716 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 10:55:23 -0700 Subject: [PATCH 03/19] Updates other calls to the search endpoint --- bookwyrm/views/get_started.py | 5 ++--- bookwyrm/views/isbn.py | 4 ++-- bookwyrm/views/list.py | 6 ++---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/bookwyrm/views/get_started.py b/bookwyrm/views/get_started.py index 3de88e104..981c62a21 100644 --- a/bookwyrm/views/get_started.py +++ b/bookwyrm/views/get_started.py @@ -11,8 +11,7 @@ from django.template.response import TemplateResponse from django.utils.decorators import method_decorator from django.views import View -from bookwyrm import forms, models -from bookwyrm.connectors import connector_manager +from bookwyrm import book_search, forms, models from bookwyrm.suggested_users import suggested_users from .edit_user import save_user_form @@ -55,7 +54,7 @@ class GetStartedBooks(View): query = request.GET.get("query") book_results = popular_books = [] if query: - book_results = connector_manager.local_search(query, raw=True)[:5] + book_results = book_search.search(query)[:5] if len(book_results) < 5: popular_books = ( models.Edition.objects.exclude( diff --git a/bookwyrm/views/isbn.py b/bookwyrm/views/isbn.py index 3055a3542..173a3adb9 100644 --- a/bookwyrm/views/isbn.py +++ b/bookwyrm/views/isbn.py @@ -4,7 +4,7 @@ from django.http import JsonResponse from django.template.response import TemplateResponse from django.views import View -from bookwyrm.connectors import connector_manager +from bookwyrm import book_search from bookwyrm.settings import PAGE_LENGTH from .helpers import is_api_request @@ -14,7 +14,7 @@ class Isbn(View): def get(self, request, isbn): """info about a book""" - book_results = connector_manager.isbn_local_search(isbn) + book_results = book_search.isbn_search(isbn) if is_api_request(request): return JsonResponse([r.json() for r in book_results], safe=False) diff --git a/bookwyrm/views/list.py b/bookwyrm/views/list.py index af99e9f55..e2eab7ef4 100644 --- a/bookwyrm/views/list.py +++ b/bookwyrm/views/list.py @@ -16,9 +16,8 @@ from django.utils.decorators import method_decorator from django.views import View from django.views.decorators.http import require_POST -from bookwyrm import forms, models +from bookwyrm import book_search, forms, models from bookwyrm.activitypub import ActivitypubResponse -from bookwyrm.connectors import connector_manager from bookwyrm.settings import PAGE_LENGTH from .helpers import is_api_request, privacy_filter from .helpers import get_user_from_username @@ -150,9 +149,8 @@ class List(View): if query and request.user.is_authenticated: # search for books - suggestions = connector_manager.local_search( + suggestions = book_search.search( query, - raw=True, filters=[~Q(parent_work__editions__in=book_list.books.all())], ) elif request.user.is_authenticated: From 76ab5a763c081b08219d183229fc72c75d994527 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 10:56:28 -0700 Subject: [PATCH 04/19] Remove outdated test --- bookwyrm/tests/connectors/test_connector_manager.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/bookwyrm/tests/connectors/test_connector_manager.py b/bookwyrm/tests/connectors/test_connector_manager.py index 67b108dd1..32acba7a2 100644 --- a/bookwyrm/tests/connectors/test_connector_manager.py +++ b/bookwyrm/tests/connectors/test_connector_manager.py @@ -96,12 +96,6 @@ class ConnectorManager(TestCase): self.assertEqual(len(results[0]["results"]), 1) self.assertEqual(results[0]["results"][0].title, "Example Edition") - def test_local_search(self): - """search only the local database""" - results = connector_manager.local_search("Example") - self.assertEqual(len(results), 1) - self.assertEqual(results[0].title, "Example Edition") - def test_first_search_result(self): """only get one search result""" result = connector_manager.first_search_result("Example") From fbe05623ff3b994c6c91d9e36144fef33fa93138 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 11:07:36 -0700 Subject: [PATCH 05/19] Updates first_search_result functionality --- bookwyrm/book_search.py | 26 ++++++++++++++++-------- bookwyrm/connectors/connector_manager.py | 7 ++++++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index 8682b83e1..8693209cf 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -11,16 +11,18 @@ from bookwyrm.settings import MEDIA_FULL_URL # pylint: disable=arguments-differ -def search(query, min_confidence=0, filters=None): +def search(query, min_confidence=0, filters=None, return_first=False): """search your local database""" filters = filters or [] if not query: return [] # first, try searching unqiue identifiers - results = search_identifiers(query, *filters) + results = search_identifiers(query, *filters, return_first=return_first) if not results: # then try searching title/author - results = search_title_author(query, min_confidence, *filters) + results = search_title_author( + query, min_confidence, *filters, return_first=return_first + ) return results @@ -68,7 +70,7 @@ def format_search_result(search_result): ).json() -def search_identifiers(query, *filters): +def search_identifiers(query, *filters, return_first=False): """tries remote_id, isbn; defined as dedupe fields on the model""" # pylint: disable=W0212 or_filters = [ @@ -87,15 +89,18 @@ def search_identifiers(query, *filters): default_editions = models.Edition.objects.filter( parent_work=OuterRef("parent_work") ).order_by("-edition_rank") - return ( + results = ( results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter( default_id=F("id") ) or results ) + if return_first: + return results.first() + return results -def search_title_author(query, min_confidence, *filters): +def search_title_author(query, min_confidence, *filters, return_first=False): """searches for title and author""" query = SearchQuery(query, config="simple") | SearchQuery(query, config="english") results = ( @@ -109,12 +114,17 @@ def search_title_author(query, min_confidence, *filters): editions_of_work = results.values("parent_work__id").values_list("parent_work__id") # filter out multiple editions of the same work + results = [] for work_id in set(editions_of_work): editions = results.filter(parent_work=work_id) default = editions.order_by("-edition_rank").first() default_rank = default.rank if default else 0 # if mutliple books have the top rank, pick the default edition if default_rank == editions.first().rank: - yield default + result = default else: - yield editions.first() + result = editions.first() + if return_first: + return result + results.append(result) + return results diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index 6f62b59a3..d9c997e6e 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -10,7 +10,7 @@ from django.db.models import signals from requests import HTTPError -from bookwyrm import models +from bookwyrm import book_search, models from bookwyrm.tasks import app logger = logging.getLogger(__name__) @@ -73,6 +73,11 @@ def search(query, min_confidence=0.1, return_first=False): def first_search_result(query, min_confidence=0.1): """search until you find a result that fits""" + # try local search first + result = book_search.search(query, min_confidence=min_confidence, return_first=True) + if result: + return result + # otherwise, try remote endpoints return search(query, min_confidence=min_confidence, return_first=True) or None From 18591c7b561b1b2372f681f0fd4261d48b5bca01 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 11:30:04 -0700 Subject: [PATCH 06/19] Fixes circular import --- bookwyrm/book_search.py | 27 ++++++++++++++++++++++- bookwyrm/connectors/abstract_connector.py | 26 ---------------------- bookwyrm/connectors/bookwyrm_connector.py | 3 ++- bookwyrm/connectors/inventaire.py | 3 ++- bookwyrm/connectors/openlibrary.py | 3 ++- 5 files changed, 32 insertions(+), 30 deletions(-) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index 8693209cf..c4d03caf6 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -1,4 +1,5 @@ """ using a bookwyrm instance as a source of book data """ +from dataclasses import asdict, dataclass from functools import reduce import operator @@ -6,7 +7,6 @@ from django.contrib.postgres.search import SearchRank, SearchQuery from django.db.models import OuterRef, Subquery, F, Q from bookwyrm import models -from bookwyrm.connectors.abstract_connector import SearchResult from bookwyrm.settings import MEDIA_FULL_URL @@ -128,3 +128,28 @@ def search_title_author(query, min_confidence, *filters, return_first=False): return result results.append(result) return results + + +@dataclass +class SearchResult: + """standardized search result object""" + + title: str + key: str + connector: object + view_link: str = None + author: str = None + year: str = None + cover: str = None + confidence: int = 1 + + def __repr__(self): + return "".format( + self.key, self.title, self.author + ) + + def json(self): + """serialize a connector for json response""" + serialized = asdict(self) + del serialized["connector"] + return serialized diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 9f20a6c35..5c58f8f83 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -1,6 +1,5 @@ """ functionality outline for a book data connector """ from abc import ABC, abstractmethod -from dataclasses import asdict, dataclass import logging from django.db import transaction @@ -268,31 +267,6 @@ def get_image(url, timeout=10): return resp -@dataclass -class SearchResult: - """standardized search result object""" - - title: str - key: str - connector: object - view_link: str = None - author: str = None - year: str = None - cover: str = None - confidence: int = 1 - - def __repr__(self): - return "".format( - self.key, self.title, self.author - ) - - def json(self): - """serialize a connector for json response""" - serialized = asdict(self) - del serialized["connector"] - return serialized - - class Mapping: """associate a local database field with a field in an external dataset""" diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index 10a633b2d..6dcba7c31 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -1,6 +1,7 @@ """ using another bookwyrm instance as a source of book data """ from bookwyrm import activitypub, models -from .abstract_connector import AbstractMinimalConnector, SearchResult +from bookwyrm.book_search import SearchResult +from .abstract_connector import AbstractMinimalConnector class Connector(AbstractMinimalConnector): diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py index d2a7b9faa..8b85dbd96 100644 --- a/bookwyrm/connectors/inventaire.py +++ b/bookwyrm/connectors/inventaire.py @@ -2,7 +2,8 @@ import re from bookwyrm import models -from .abstract_connector import AbstractConnector, SearchResult, Mapping +from bookwyrm.book_search import SearchResult +from .abstract_connector import AbstractConnector, Mapping from .abstract_connector import get_data from .connector_manager import ConnectorException diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index e58749c13..0bfc6ef1e 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -2,7 +2,8 @@ import re from bookwyrm import models -from .abstract_connector import AbstractConnector, SearchResult, Mapping +from bookwyrm.book_search import SearchResult +from .abstract_connector import AbstractConnector, Mapping from .abstract_connector import get_data from .connector_manager import ConnectorException from .openlibrary_languages import languages From beb482f1db43a6b9349373aab1ca6e494e9ce157 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 11:47:44 -0700 Subject: [PATCH 07/19] Linter fixes Temporarily disables C0209 because it's out of scope to fix here --- .github/workflows/pylint.yml | 2 +- bookwyrm/templates/search/book.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 1b14149f2..2a81eaa3e 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -24,5 +24,5 @@ jobs: pip install pylint - name: Analysing the code with pylint run: | - pylint bookwyrm/ --ignore=migrations,tests --disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801 + pylint bookwyrm/ --ignore=migrations,tests --disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801,C0209 diff --git a/bookwyrm/templates/search/book.html b/bookwyrm/templates/search/book.html index 98590f202..704f055bf 100644 --- a/bookwyrm/templates/search/book.html +++ b/bookwyrm/templates/search/book.html @@ -8,7 +8,7 @@
    {% for result in local_results.results %}
  • -
    +
    {% include 'snippets/book_cover.html' with book=result cover_class='is-w-xs is-h-xs' %}
    From 4cdf895d777d37066efb454f906bc90bc639ab3e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 12:29:02 -0700 Subject: [PATCH 08/19] Removes references to local field in connector tests --- bookwyrm/tests/connectors/test_abstract_minimal_connector.py | 1 - bookwyrm/tests/connectors/test_connector_manager.py | 2 -- bookwyrm/tests/connectors/test_self_connector.py | 1 - bookwyrm/tests/importers/test_goodreads_import.py | 1 - bookwyrm/tests/importers/test_librarything_import.py | 1 - bookwyrm/tests/views/test_get_started.py | 2 +- bookwyrm/tests/views/test_search.py | 2 +- 7 files changed, 2 insertions(+), 8 deletions(-) diff --git a/bookwyrm/tests/connectors/test_abstract_minimal_connector.py b/bookwyrm/tests/connectors/test_abstract_minimal_connector.py index 846291399..30db40cd6 100644 --- a/bookwyrm/tests/connectors/test_abstract_minimal_connector.py +++ b/bookwyrm/tests/connectors/test_abstract_minimal_connector.py @@ -53,7 +53,6 @@ class AbstractConnector(TestCase): self.assertEqual(connector.isbn_search_url, "https://example.com/isbn?q=") self.assertIsNone(connector.name) self.assertEqual(connector.identifier, "example.com") - self.assertFalse(connector.local) @responses.activate def test_search(self): diff --git a/bookwyrm/tests/connectors/test_connector_manager.py b/bookwyrm/tests/connectors/test_connector_manager.py index 32acba7a2..f87b9c438 100644 --- a/bookwyrm/tests/connectors/test_connector_manager.py +++ b/bookwyrm/tests/connectors/test_connector_manager.py @@ -25,7 +25,6 @@ class ConnectorManager(TestCase): self.connector = models.Connector.objects.create( identifier="test_connector", priority=1, - local=True, connector_file="self_connector", base_url="http://test.com/", books_url="http://test.com/", @@ -36,7 +35,6 @@ class ConnectorManager(TestCase): self.remote_connector = models.Connector.objects.create( identifier="test_connector_remote", priority=1, - local=False, connector_file="bookwyrm_connector", base_url="http://fake.ciom/", books_url="http://fake.ciom/", diff --git a/bookwyrm/tests/connectors/test_self_connector.py b/bookwyrm/tests/connectors/test_self_connector.py index 86aa7add4..a5eab8797 100644 --- a/bookwyrm/tests/connectors/test_self_connector.py +++ b/bookwyrm/tests/connectors/test_self_connector.py @@ -16,7 +16,6 @@ class SelfConnector(TestCase): models.Connector.objects.create( identifier=DOMAIN, name="Local", - local=True, connector_file="self_connector", base_url="https://%s" % DOMAIN, books_url="https://%s/book" % DOMAIN, diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index 387d9f4f2..c332bf693 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -42,7 +42,6 @@ class GoodreadsImport(TestCase): models.Connector.objects.create( identifier=DOMAIN, name="Local", - local=True, connector_file="self_connector", base_url="https://%s" % DOMAIN, books_url="https://%s/book" % DOMAIN, diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index dfdd515e1..7a1c5d510 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -43,7 +43,6 @@ class LibrarythingImport(TestCase): models.Connector.objects.create( identifier=DOMAIN, name="Local", - local=True, connector_file="self_connector", base_url="https://%s" % DOMAIN, books_url="https://%s/book" % DOMAIN, diff --git a/bookwyrm/tests/views/test_get_started.py b/bookwyrm/tests/views/test_get_started.py index 135896dc2..cc53986fa 100644 --- a/bookwyrm/tests/views/test_get_started.py +++ b/bookwyrm/tests/views/test_get_started.py @@ -30,7 +30,7 @@ class GetStartedViews(TestCase): remote_id="https://example.com/book/1", ) models.Connector.objects.create( - identifier="self", connector_file="self_connector", local=True + identifier="self", connector_file="self_connector" ) models.SiteSettings.objects.create() diff --git a/bookwyrm/tests/views/test_search.py b/bookwyrm/tests/views/test_search.py index dacbcbded..7eb75dab8 100644 --- a/bookwyrm/tests/views/test_search.py +++ b/bookwyrm/tests/views/test_search.py @@ -37,7 +37,7 @@ class Views(TestCase): parent_work=self.work, ) models.Connector.objects.create( - identifier="self", connector_file="self_connector", local=True + identifier="self", connector_file="self_connector" ) models.SiteSettings.objects.create() From 22af7ece7161ef450bd7c9b0071830fd57330822 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 12:41:30 -0700 Subject: [PATCH 09/19] Fixes SearchResult imports in tests --- .../connectors/test_abstract_connector.py | 2 +- .../test_abstract_minimal_connector.py | 3 +- .../connectors/test_bookwyrm_connector.py | 2 +- .../connectors/test_openlibrary_connector.py | 2 +- .../tests/connectors/test_self_connector.py | 106 ------------------ 5 files changed, 5 insertions(+), 110 deletions(-) delete mode 100644 bookwyrm/tests/connectors/test_self_connector.py diff --git a/bookwyrm/tests/connectors/test_abstract_connector.py b/bookwyrm/tests/connectors/test_abstract_connector.py index 8ce4c96bf..129b3f0c6 100644 --- a/bookwyrm/tests/connectors/test_abstract_connector.py +++ b/bookwyrm/tests/connectors/test_abstract_connector.py @@ -119,7 +119,7 @@ class AbstractConnector(TestCase): @responses.activate def test_get_or_create_author(self): """load an author""" - self.connector.author_mappings = ( + self.connector.author_mappings = ( # pylint: disable=attribute-defined-outside-init [ # pylint: disable=attribute-defined-outside-init Mapping("id"), Mapping("name"), diff --git a/bookwyrm/tests/connectors/test_abstract_minimal_connector.py b/bookwyrm/tests/connectors/test_abstract_minimal_connector.py index 30db40cd6..589072123 100644 --- a/bookwyrm/tests/connectors/test_abstract_minimal_connector.py +++ b/bookwyrm/tests/connectors/test_abstract_minimal_connector.py @@ -3,8 +3,9 @@ from django.test import TestCase import responses from bookwyrm import models +from bookwyrm.book_search import SearchResult from bookwyrm.connectors import abstract_connector -from bookwyrm.connectors.abstract_connector import Mapping, SearchResult +from bookwyrm.connectors.abstract_connector import Mapping class AbstractConnector(TestCase): diff --git a/bookwyrm/tests/connectors/test_bookwyrm_connector.py b/bookwyrm/tests/connectors/test_bookwyrm_connector.py index 46ea54a91..585080e66 100644 --- a/bookwyrm/tests/connectors/test_bookwyrm_connector.py +++ b/bookwyrm/tests/connectors/test_bookwyrm_connector.py @@ -4,8 +4,8 @@ import pathlib from django.test import TestCase from bookwyrm import models +from bookwyrm.book_search import SearchResult from bookwyrm.connectors.bookwyrm_connector import Connector -from bookwyrm.connectors.abstract_connector import SearchResult class BookWyrmConnector(TestCase): diff --git a/bookwyrm/tests/connectors/test_openlibrary_connector.py b/bookwyrm/tests/connectors/test_openlibrary_connector.py index 699b26ed4..75a273d63 100644 --- a/bookwyrm/tests/connectors/test_openlibrary_connector.py +++ b/bookwyrm/tests/connectors/test_openlibrary_connector.py @@ -7,11 +7,11 @@ from django.test import TestCase import responses from bookwyrm import models +from bookwyrm.book_search import SearchResult from bookwyrm.connectors.openlibrary import Connector from bookwyrm.connectors.openlibrary import ignore_edition from bookwyrm.connectors.openlibrary import get_languages, get_description from bookwyrm.connectors.openlibrary import pick_default_edition, get_openlibrary_key -from bookwyrm.connectors.abstract_connector import SearchResult from bookwyrm.connectors.connector_manager import ConnectorException diff --git a/bookwyrm/tests/connectors/test_self_connector.py b/bookwyrm/tests/connectors/test_self_connector.py deleted file mode 100644 index a5eab8797..000000000 --- a/bookwyrm/tests/connectors/test_self_connector.py +++ /dev/null @@ -1,106 +0,0 @@ -""" testing book data connectors """ -import datetime -from django.test import TestCase -from django.utils import timezone - -from bookwyrm import models -from bookwyrm.connectors.self_connector import Connector -from bookwyrm.settings import DOMAIN - - -class SelfConnector(TestCase): - """just uses local data""" - - def setUp(self): - """creating the connector""" - models.Connector.objects.create( - identifier=DOMAIN, - name="Local", - connector_file="self_connector", - base_url="https://%s" % DOMAIN, - books_url="https://%s/book" % DOMAIN, - covers_url="https://%s/images/covers" % DOMAIN, - search_url="https://%s/search?q=" % DOMAIN, - priority=1, - ) - self.connector = Connector(DOMAIN) - - def test_format_search_result(self): - """create a SearchResult""" - author = models.Author.objects.create(name="Anonymous") - edition = models.Edition.objects.create( - title="Edition of Example Work", - published_date=datetime.datetime(1980, 5, 10, tzinfo=timezone.utc), - ) - edition.authors.add(author) - result = self.connector.search("Edition of Example")[0] - self.assertEqual(result.title, "Edition of Example Work") - self.assertEqual(result.key, edition.remote_id) - self.assertEqual(result.author, "Anonymous") - self.assertEqual(result.year, 1980) - self.assertEqual(result.connector, self.connector) - - def test_search_rank(self): - """prioritize certain results""" - author = models.Author.objects.create(name="Anonymous") - edition = models.Edition.objects.create( - title="Edition of Example Work", - published_date=datetime.datetime(1980, 5, 10, tzinfo=timezone.utc), - parent_work=models.Work.objects.create(title=""), - ) - # author text is rank B - edition.authors.add(author) - - # series is rank D - models.Edition.objects.create( - title="Another Edition", - series="Anonymous", - parent_work=models.Work.objects.create(title=""), - ) - # subtitle is rank B - models.Edition.objects.create( - title="More Editions", - subtitle="The Anonymous Edition", - parent_work=models.Work.objects.create(title=""), - ) - # title is rank A - models.Edition.objects.create(title="Anonymous") - # doesn't rank in this search - models.Edition.objects.create( - title="An Edition", parent_work=models.Work.objects.create(title="") - ) - - results = self.connector.search("Anonymous") - self.assertEqual(len(results), 4) - self.assertEqual(results[0].title, "Anonymous") - self.assertEqual(results[1].title, "More Editions") - self.assertEqual(results[2].title, "Edition of Example Work") - self.assertEqual(results[3].title, "Another Edition") - - def test_search_multiple_editions(self): - """it should get rid of duplicate editions for the same work""" - work = models.Work.objects.create(title="Work Title") - edition_1 = models.Edition.objects.create( - title="Edition 1 Title", parent_work=work - ) - edition_2 = models.Edition.objects.create( - title="Edition 2 Title", - parent_work=work, - isbn_13="123456789", # this is now the defualt edition - ) - edition_3 = models.Edition.objects.create(title="Fish", parent_work=work) - - # pick the best edition - results = self.connector.search("Edition 1 Title") - self.assertEqual(len(results), 1) - self.assertEqual(results[0].key, edition_1.remote_id) - - # pick the default edition when no match is best - results = self.connector.search("Edition Title") - self.assertEqual(len(results), 1) - self.assertEqual(results[0].key, edition_2.remote_id) - - # only matches one edition, so no deduplication takes place - results = self.connector.search("Fish") - self.assertEqual(len(results), 1) - self.assertEqual(results[0].key, edition_3.remote_id) From 8c4cafed79a65354a0bfe4952074f17e87d9cead Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 12:52:10 -0700 Subject: [PATCH 10/19] Fixes formatting isbn endpoint results --- bookwyrm/book_search.py | 6 +++--- bookwyrm/tests/connectors/test_abstract_connector.py | 10 ++++------ bookwyrm/tests/views/test_isbn.py | 2 +- bookwyrm/views/isbn.py | 4 +++- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index c4d03caf6..91a4e67a5 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -114,7 +114,7 @@ def search_title_author(query, min_confidence, *filters, return_first=False): editions_of_work = results.values("parent_work__id").values_list("parent_work__id") # filter out multiple editions of the same work - results = [] + list_results = [] for work_id in set(editions_of_work): editions = results.filter(parent_work=work_id) default = editions.order_by("-edition_rank").first() @@ -126,8 +126,8 @@ def search_title_author(query, min_confidence, *filters, return_first=False): result = editions.first() if return_first: return result - results.append(result) - return results + list_results.append(result) + return list_results @dataclass diff --git a/bookwyrm/tests/connectors/test_abstract_connector.py b/bookwyrm/tests/connectors/test_abstract_connector.py index 129b3f0c6..a453f6133 100644 --- a/bookwyrm/tests/connectors/test_abstract_connector.py +++ b/bookwyrm/tests/connectors/test_abstract_connector.py @@ -119,12 +119,10 @@ class AbstractConnector(TestCase): @responses.activate def test_get_or_create_author(self): """load an author""" - self.connector.author_mappings = ( # pylint: disable=attribute-defined-outside-init - [ # pylint: disable=attribute-defined-outside-init - Mapping("id"), - Mapping("name"), - ] - ) + self.connector.author_mappings = [ # pylint: disable=attribute-defined-outside-init # pylint: disable=attribute-defined-outside-init + Mapping("id"), + Mapping("name"), + ] responses.add( responses.GET, diff --git a/bookwyrm/tests/views/test_isbn.py b/bookwyrm/tests/views/test_isbn.py index a6a451748..05a109568 100644 --- a/bookwyrm/tests/views/test_isbn.py +++ b/bookwyrm/tests/views/test_isbn.py @@ -35,7 +35,7 @@ class IsbnViews(TestCase): parent_work=self.work, ) models.Connector.objects.create( - identifier="self", connector_file="self_connector", local=True + identifier="self", connector_file="self_connector" ) models.SiteSettings.objects.create() diff --git a/bookwyrm/views/isbn.py b/bookwyrm/views/isbn.py index 173a3adb9..e5343488d 100644 --- a/bookwyrm/views/isbn.py +++ b/bookwyrm/views/isbn.py @@ -17,7 +17,9 @@ class Isbn(View): book_results = book_search.isbn_search(isbn) if is_api_request(request): - return JsonResponse([r.json() for r in book_results], safe=False) + return JsonResponse( + [book_search.format_search_result(r) for r in book_results], safe=False + ) paginated = Paginator(book_results, PAGE_LENGTH).get_page( request.GET.get("page") From d9284ede9b3f367a9c343bc9e5f915a1f79b28e8 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 15:27:06 -0700 Subject: [PATCH 11/19] updates search tests --- bookwyrm/tests/views/test_search.py | 53 ++++++++++------------------- 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/bookwyrm/tests/views/test_search.py b/bookwyrm/tests/views/test_search.py index 7eb75dab8..cd0273bf8 100644 --- a/bookwyrm/tests/views/test_search.py +++ b/bookwyrm/tests/views/test_search.py @@ -1,5 +1,6 @@ """ test for app action functionality """ import json +import pathlib from unittest.mock import patch from django.contrib.auth.models import AnonymousUser @@ -7,9 +8,9 @@ from django.http import JsonResponse from django.template.response import TemplateResponse from django.test import TestCase from django.test.client import RequestFactory +import responses from bookwyrm import models, views -from bookwyrm.connectors import abstract_connector from bookwyrm.settings import DOMAIN @@ -36,15 +37,11 @@ class Views(TestCase): remote_id="https://example.com/book/1", parent_work=self.work, ) - models.Connector.objects.create( - identifier="self", connector_file="self_connector" - ) models.SiteSettings.objects.create() def test_search_json_response(self): """searches local data only and returns book data in json format""" view = views.Search.as_view() - # we need a connector for this, sorry request = self.factory.get("", {"q": "Test Book"}) with patch("bookwyrm.views.search.is_api_request") as is_api: is_api.return_value = True @@ -67,28 +64,11 @@ class Views(TestCase): self.assertIsInstance(response, TemplateResponse) response.render() + @responses.activate def test_search_books(self): """searches remote connectors""" view = views.Search.as_view() - class TestConnector(abstract_connector.AbstractMinimalConnector): - """nothing added here""" - - def format_search_result(self, search_result): - pass - - def get_or_create_book(self, remote_id): - pass - - def parse_search_data(self, data): - pass - - def format_isbn_search_result(self, search_result): - return search_result - - def parse_isbn_search_data(self, data): - return data - models.Connector.objects.create( identifier="example.com", connector_file="openlibrary", @@ -97,26 +77,29 @@ class Views(TestCase): covers_url="https://example.com/covers", search_url="https://example.com/search?q=", ) - connector = TestConnector("example.com") - - search_result = abstract_connector.SearchResult( - key="http://www.example.com/book/1", - title="Gideon the Ninth", - author="Tamsyn Muir", - year="2019", - connector=connector, + datafile = pathlib.Path(__file__).parent.joinpath("../data/ol_search.json") + search_data = json.loads(datafile.read_bytes()) + responses.add( + responses.GET, + "https://example.com/search?q=Test%20Book", + json=search_data ) request = self.factory.get("", {"q": "Test Book", "remote": True}) request.user = self.local_user with patch("bookwyrm.views.search.is_api_request") as is_api: is_api.return_value = False - with patch("bookwyrm.connectors.connector_manager.search") as manager: - manager.return_value = [search_result] - response = view(request) + response = view(request) self.assertIsInstance(response, TemplateResponse) response.render() - self.assertEqual(response.context_data["results"][0].title, "Gideon the Ninth") + connector_results = response.context_data["results"] + self.assertEqual( + connector_results[0]["results"][0].title, "Test Book" + ) + self.assertEqual( + connector_results[1]["results"][0].title, + "This Is How You Lose the Time War" + ) def test_search_users(self): """searches remote connectors""" From 146538545209250a2117448746cf2b2dc95ec5ef Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 16 Sep 2021 15:29:06 -0700 Subject: [PATCH 12/19] Python formatting --- bookwyrm/tests/views/test_search.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/bookwyrm/tests/views/test_search.py b/bookwyrm/tests/views/test_search.py index cd0273bf8..da35f5571 100644 --- a/bookwyrm/tests/views/test_search.py +++ b/bookwyrm/tests/views/test_search.py @@ -80,9 +80,7 @@ class Views(TestCase): datafile = pathlib.Path(__file__).parent.joinpath("../data/ol_search.json") search_data = json.loads(datafile.read_bytes()) responses.add( - responses.GET, - "https://example.com/search?q=Test%20Book", - json=search_data + responses.GET, "https://example.com/search?q=Test%20Book", json=search_data ) request = self.factory.get("", {"q": "Test Book", "remote": True}) @@ -93,12 +91,10 @@ class Views(TestCase): self.assertIsInstance(response, TemplateResponse) response.render() connector_results = response.context_data["results"] - self.assertEqual( - connector_results[0]["results"][0].title, "Test Book" - ) + self.assertEqual(connector_results[0]["results"][0].title, "Test Book") self.assertEqual( connector_results[1]["results"][0].title, - "This Is How You Lose the Time War" + "This Is How You Lose the Time War", ) def test_search_users(self): From 967e26ce489a0570aaf881f12f6b192ce8afd907 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 17 Sep 2021 11:29:10 -0700 Subject: [PATCH 13/19] Updates connector manager tests --- .../connectors/test_connector_manager.py | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/bookwyrm/tests/connectors/test_connector_manager.py b/bookwyrm/tests/connectors/test_connector_manager.py index f87b9c438..186f918bf 100644 --- a/bookwyrm/tests/connectors/test_connector_manager.py +++ b/bookwyrm/tests/connectors/test_connector_manager.py @@ -5,7 +5,6 @@ import responses from bookwyrm import models from bookwyrm.connectors import connector_manager from bookwyrm.connectors.bookwyrm_connector import Connector as BookWyrmConnector -from bookwyrm.connectors.self_connector import Connector as SelfConnector class ConnectorManager(TestCase): @@ -22,16 +21,6 @@ class ConnectorManager(TestCase): title="Another Edition", parent_work=self.work, isbn_10="1111111111" ) - self.connector = models.Connector.objects.create( - identifier="test_connector", - priority=1, - connector_file="self_connector", - base_url="http://test.com/", - books_url="http://test.com/", - covers_url="http://test.com/", - isbn_search_url="http://test.com/isbn/", - ) - self.remote_connector = models.Connector.objects.create( identifier="test_connector_remote", priority=1, @@ -57,9 +46,8 @@ class ConnectorManager(TestCase): def test_get_connectors(self): """load all connectors""" connectors = list(connector_manager.get_connectors()) - self.assertEqual(len(connectors), 2) - self.assertIsInstance(connectors[0], SelfConnector) - self.assertIsInstance(connectors[1], BookWyrmConnector) + self.assertEqual(len(connectors), 1) + self.assertIsInstance(connectors[0], BookWyrmConnector) @responses.activate def test_search(self): @@ -71,7 +59,6 @@ class ConnectorManager(TestCase): ) results = connector_manager.search("Example") self.assertEqual(len(results), 1) - self.assertIsInstance(results[0]["connector"], SelfConnector) self.assertEqual(len(results[0]["results"]), 1) self.assertEqual(results[0]["results"][0].title, "Example Edition") @@ -90,7 +77,6 @@ class ConnectorManager(TestCase): ) results = connector_manager.search("0000000000") self.assertEqual(len(results), 1) - self.assertIsInstance(results[0]["connector"], SelfConnector) self.assertEqual(len(results[0]["results"]), 1) self.assertEqual(results[0]["results"][0].title, "Example Edition") @@ -117,6 +103,5 @@ class ConnectorManager(TestCase): def test_load_connector(self): """load a connector object from the database entry""" - connector = connector_manager.load_connector(self.connector) - self.assertIsInstance(connector, SelfConnector) - self.assertEqual(connector.identifier, "test_connector") + connector = connector_manager.load_connector(self.remote_connector) + self.assertEqual(connector.identifier, "test_connector_remote") From e6e44decf9a8a723c0342d545e22042a2c1065d5 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 30 Sep 2021 10:47:53 -0700 Subject: [PATCH 14/19] Updates migration --- bookwyrm/book_search.py | 1 + ...move_connector_local.py => 0102_remove_connector_local.py} | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) rename bookwyrm/migrations/{0097_remove_connector_local.py => 0102_remove_connector_local.py} (70%) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index 91a4e67a5..6c89b61fb 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -144,6 +144,7 @@ class SearchResult: confidence: int = 1 def __repr__(self): + # pylint: disable=consider-using-f-string return "".format( self.key, self.title, self.author ) diff --git a/bookwyrm/migrations/0097_remove_connector_local.py b/bookwyrm/migrations/0102_remove_connector_local.py similarity index 70% rename from bookwyrm/migrations/0097_remove_connector_local.py rename to bookwyrm/migrations/0102_remove_connector_local.py index fc04bccec..bd0dd9200 100644 --- a/bookwyrm/migrations/0097_remove_connector_local.py +++ b/bookwyrm/migrations/0102_remove_connector_local.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.4 on 2021-09-14 22:10 +# Generated by Django 3.2.5 on 2021-09-30 17:46 from django.db import migrations @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ("bookwyrm", "0096_merge_20210912_0044"), + ("bookwyrm", "0101_auto_20210929_1847"), ] operations = [ From 38f82fe6606dc987877688f3a972510fdaa862ec Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 30 Sep 2021 10:54:31 -0700 Subject: [PATCH 15/19] Remove/re-add connector in migration --- .../migrations/0102_remove_connector_local.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/bookwyrm/migrations/0102_remove_connector_local.py b/bookwyrm/migrations/0102_remove_connector_local.py index bd0dd9200..5deaf32af 100644 --- a/bookwyrm/migrations/0102_remove_connector_local.py +++ b/bookwyrm/migrations/0102_remove_connector_local.py @@ -1,6 +1,34 @@ # Generated by Django 3.2.5 on 2021-09-30 17:46 from django.db import migrations +from bookwyrm.settings import DOMAIN + + +def remove_self_connector(app_registry, schema_editor): + """set the new phsyical format field based on existing format data""" + db_alias = schema_editor.connection.alias + + app_registry.get_model("bookwyrm", "Connector").objects.using(db_alias).filter( + local=True + ).delete() + + +def reverse(app_registry, schema_editor): + """doesn't need to do anything""" + db_alias = schema_editor.connection.alias + model = app_registry.get_model("bookwyrm", "Connector") + model.objects.using(db_alias).create( + identifier=DOMAIN, + name="Local", + local=True, + connector_file="self_connector", + base_url=f"https://{DOMAIN}", + books_url=f"https://{DOMAIN}/book", + covers_url=f"https://{DOMAIN}/images/", + search_url=f"https://{DOMAIN}/search?q=", + isbn_search_url=f"https://{DOMAIN}/isbn/", + priority=1, + ) class Migration(migrations.Migration): @@ -10,6 +38,7 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RunPython(remove_self_connector, reverse), migrations.RemoveField( model_name="connector", name="local", From 232e4bed79f2c8a12732e78299538426910bbf8a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 30 Sep 2021 11:06:12 -0700 Subject: [PATCH 16/19] Updates migrations --- .../migrations/0102_remove_connector_local.py | 7 +------ .../migrations/0103_remove_connector_local.py | 17 +++++++++++++++++ bookwyrm/urls.py | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 bookwyrm/migrations/0103_remove_connector_local.py diff --git a/bookwyrm/migrations/0102_remove_connector_local.py b/bookwyrm/migrations/0102_remove_connector_local.py index 5deaf32af..857f0f589 100644 --- a/bookwyrm/migrations/0102_remove_connector_local.py +++ b/bookwyrm/migrations/0102_remove_connector_local.py @@ -7,9 +7,8 @@ from bookwyrm.settings import DOMAIN def remove_self_connector(app_registry, schema_editor): """set the new phsyical format field based on existing format data""" db_alias = schema_editor.connection.alias - app_registry.get_model("bookwyrm", "Connector").objects.using(db_alias).filter( - local=True + connector_file="self_connector" ).delete() @@ -39,8 +38,4 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython(remove_self_connector, reverse), - migrations.RemoveField( - model_name="connector", - name="local", - ), ] diff --git a/bookwyrm/migrations/0103_remove_connector_local.py b/bookwyrm/migrations/0103_remove_connector_local.py new file mode 100644 index 000000000..788ce5f81 --- /dev/null +++ b/bookwyrm/migrations/0103_remove_connector_local.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.5 on 2021-09-30 18:03 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0102_remove_connector_local"), + ] + + operations = [ + migrations.RemoveField( + model_name="connector", + name="local", + ), + ] diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 8c260b9ef..c81d97903 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -233,8 +233,8 @@ urlpatterns = [ name="direct-messages-user", ), # search - re_path(r"^search/?$", views.Search.as_view(), name="search"), re_path(r"^search.json/?$", views.Search.as_view(), name="search"), + re_path(r"^search/?$", views.Search.as_view(), name="search"), # imports re_path(r"^import/?$", views.Import.as_view(), name="import"), re_path(r"^import/(\d+)/?$", views.ImportStatus.as_view(), name="import-status"), From 1033d3d0459689b6b6cfbe628edc83cb9ffcf389 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 30 Sep 2021 11:33:04 -0700 Subject: [PATCH 17/19] Updates connector tests --- bookwyrm/connectors/settings.py | 2 +- bookwyrm/management/commands/initdb.py | 1 - bookwyrm/tests/connectors/test_connector_manager.py | 12 +++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bookwyrm/connectors/settings.py b/bookwyrm/connectors/settings.py index 4cc98da7f..927e39b26 100644 --- a/bookwyrm/connectors/settings.py +++ b/bookwyrm/connectors/settings.py @@ -1,3 +1,3 @@ """ settings book data connectors """ -CONNECTORS = ["openlibrary", "inventaire", "self_connector", "bookwyrm_connector"] +CONNECTORS = ["openlibrary", "inventaire", "bookwyrm_connector"] diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py index 9f2f29cda..d0ab648e0 100644 --- a/bookwyrm/management/commands/initdb.py +++ b/bookwyrm/management/commands/initdb.py @@ -4,7 +4,6 @@ from django.contrib.auth.models import Group, Permission from django.contrib.contenttypes.models import ContentType from bookwyrm.models import Connector, FederatedServer, SiteSettings, User -from bookwyrm.settings import DOMAIN def init_groups(): diff --git a/bookwyrm/tests/connectors/test_connector_manager.py b/bookwyrm/tests/connectors/test_connector_manager.py index 186f918bf..d6d82fc8c 100644 --- a/bookwyrm/tests/connectors/test_connector_manager.py +++ b/bookwyrm/tests/connectors/test_connector_manager.py @@ -50,17 +50,18 @@ class ConnectorManager(TestCase): self.assertIsInstance(connectors[0], BookWyrmConnector) @responses.activate - def test_search(self): + def test_search_plaintext(self): """search all connectors""" responses.add( responses.GET, "http://fake.ciom/search/Example?min_confidence=0.1", - json={}, + json=[{"title": "Hello", "key": "https://www.example.com/search/1"}], ) results = connector_manager.search("Example") self.assertEqual(len(results), 1) self.assertEqual(len(results[0]["results"]), 1) - self.assertEqual(results[0]["results"][0].title, "Example Edition") + self.assertEqual(results[0]["connector"].identifier, "test_connector_remote") + self.assertEqual(results[0]["results"][0].title, "Hello") def test_search_empty_query(self): """don't panic on empty queries""" @@ -73,12 +74,13 @@ class ConnectorManager(TestCase): responses.add( responses.GET, "http://fake.ciom/isbn/0000000000", - json={}, + json=[{"title": "Hello", "key": "https://www.example.com/search/1"}], ) results = connector_manager.search("0000000000") self.assertEqual(len(results), 1) self.assertEqual(len(results[0]["results"]), 1) - self.assertEqual(results[0]["results"][0].title, "Example Edition") + self.assertEqual(results[0]["connector"].identifier, "test_connector_remote") + self.assertEqual(results[0]["results"][0].title, "Hello") def test_first_search_result(self): """only get one search result""" From 92f9319fe1d8f9ddbc37335c1494c52ee86e20cc Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 30 Sep 2021 11:46:17 -0700 Subject: [PATCH 18/19] Updates tests that reference self_connector --- .../tests/importers/test_goodreads_import.py | 23 +++++-------------- .../importers/test_librarything_import.py | 12 ---------- bookwyrm/tests/models/test_import_model.py | 2 +- bookwyrm/tests/views/test_get_started.py | 3 --- bookwyrm/tests/views/test_isbn.py | 5 +--- 5 files changed, 8 insertions(+), 37 deletions(-) diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index c332bf693..d2b0ea7d3 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -12,7 +12,6 @@ import responses from bookwyrm import models from bookwyrm.importers import GoodreadsImporter from bookwyrm.importers.importer import import_data, handle_imported_book -from bookwyrm.settings import DOMAIN def make_date(*args): @@ -39,16 +38,6 @@ class GoodreadsImport(TestCase): "mouse", "mouse@mouse.mouse", "password", local=True ) - models.Connector.objects.create( - identifier=DOMAIN, - name="Local", - connector_file="self_connector", - base_url="https://%s" % DOMAIN, - books_url="https://%s/book" % DOMAIN, - covers_url="https://%s/images/covers" % DOMAIN, - search_url="https://%s/search?q=" % DOMAIN, - priority=1, - ) work = models.Work.objects.create(title="Test Work") self.book = models.Edition.objects.create( title="Example Edition", @@ -124,7 +113,7 @@ class GoodreadsImport(TestCase): import_job = models.ImportJob.objects.create(user=self.user) datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") - csv_file = open(datafile, "r") + csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding for index, entry in enumerate(list(csv.DictReader(csv_file))): entry = self.importer.parse_fields(entry) import_item = models.ImportItem.objects.create( @@ -161,7 +150,7 @@ class GoodreadsImport(TestCase): import_job = models.ImportJob.objects.create(user=self.user) datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") - csv_file = open(datafile, "r") + csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding for index, entry in enumerate(list(csv.DictReader(csv_file))): entry = self.importer.parse_fields(entry) import_item = models.ImportItem.objects.create( @@ -191,7 +180,7 @@ class GoodreadsImport(TestCase): shelf = self.user.shelf_set.filter(identifier="read").first() import_job = models.ImportJob.objects.create(user=self.user) datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") - csv_file = open(datafile, "r") + csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding for index, entry in enumerate(list(csv.DictReader(csv_file))): entry = self.importer.parse_fields(entry) import_item = models.ImportItem.objects.create( @@ -223,7 +212,7 @@ class GoodreadsImport(TestCase): """goodreads review import""" import_job = models.ImportJob.objects.create(user=self.user) datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") - csv_file = open(datafile, "r") + csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding entry = list(csv.DictReader(csv_file))[2] entry = self.importer.parse_fields(entry) import_item = models.ImportItem.objects.create( @@ -247,7 +236,7 @@ class GoodreadsImport(TestCase): datafile = pathlib.Path(__file__).parent.joinpath( "../data/goodreads-rating.csv" ) - csv_file = open(datafile, "r") + csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding entry = list(csv.DictReader(csv_file))[2] entry = self.importer.parse_fields(entry) import_item = models.ImportItem.objects.create( @@ -268,7 +257,7 @@ class GoodreadsImport(TestCase): """goodreads review import""" import_job = models.ImportJob.objects.create(user=self.user) datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") - csv_file = open(datafile, "r") + csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding entry = list(csv.DictReader(csv_file))[2] entry = self.importer.parse_fields(entry) import_item = models.ImportItem.objects.create( diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index 7a1c5d510..ab92c11b1 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -11,7 +11,6 @@ import responses from bookwyrm import models from bookwyrm.importers import LibrarythingImporter from bookwyrm.importers.importer import import_data, handle_imported_book -from bookwyrm.settings import DOMAIN def make_date(*args): @@ -39,17 +38,6 @@ class LibrarythingImport(TestCase): self.user = models.User.objects.create_user( "mmai", "mmai@mmai.mmai", "password", local=True ) - - models.Connector.objects.create( - identifier=DOMAIN, - name="Local", - connector_file="self_connector", - base_url="https://%s" % DOMAIN, - books_url="https://%s/book" % DOMAIN, - covers_url="https://%s/images/covers" % DOMAIN, - search_url="https://%s/search?q=" % DOMAIN, - priority=1, - ) work = models.Work.objects.create(title="Test Work") self.book = models.Edition.objects.create( title="Example Edition", diff --git a/bookwyrm/tests/models/test_import_model.py b/bookwyrm/tests/models/test_import_model.py index caf610349..0e5d6760a 100644 --- a/bookwyrm/tests/models/test_import_model.py +++ b/bookwyrm/tests/models/test_import_model.py @@ -9,8 +9,8 @@ from django.test import TestCase import responses from bookwyrm import models +from bookwyrm.book_search import SearchResult from bookwyrm.connectors import connector_manager -from bookwyrm.connectors.abstract_connector import SearchResult class ImportJob(TestCase): diff --git a/bookwyrm/tests/views/test_get_started.py b/bookwyrm/tests/views/test_get_started.py index cc53986fa..ff441b578 100644 --- a/bookwyrm/tests/views/test_get_started.py +++ b/bookwyrm/tests/views/test_get_started.py @@ -29,9 +29,6 @@ class GetStartedViews(TestCase): title="Example Edition", remote_id="https://example.com/book/1", ) - models.Connector.objects.create( - identifier="self", connector_file="self_connector" - ) models.SiteSettings.objects.create() def test_profile_view(self, *_): diff --git a/bookwyrm/tests/views/test_isbn.py b/bookwyrm/tests/views/test_isbn.py index 05a109568..bdf72f753 100644 --- a/bookwyrm/tests/views/test_isbn.py +++ b/bookwyrm/tests/views/test_isbn.py @@ -34,9 +34,6 @@ class IsbnViews(TestCase): remote_id="https://example.com/book/1", parent_work=self.work, ) - models.Connector.objects.create( - identifier="self", connector_file="self_connector" - ) models.SiteSettings.objects.create() def test_isbn_json_response(self): @@ -51,4 +48,4 @@ class IsbnViews(TestCase): data = json.loads(response.content) self.assertEqual(len(data), 1) self.assertEqual(data[0]["title"], "Test Book") - self.assertEqual(data[0]["key"], "https://%s/book/%d" % (DOMAIN, self.book.id)) + self.assertEqual(data[0]["key"], f"https://{DOMAIN}/book/{self.book.id}") From 3f44389c6bb17156294f958223e449d435fb27d8 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 30 Sep 2021 13:03:04 -0700 Subject: [PATCH 19/19] Adds new test file for search --- .../test_abstract_minimal_connector.py | 14 --- .../connectors/test_connector_manager.py | 2 +- bookwyrm/tests/test_book_search.py | 118 ++++++++++++++++++ 3 files changed, 119 insertions(+), 15 deletions(-) create mode 100644 bookwyrm/tests/test_book_search.py diff --git a/bookwyrm/tests/connectors/test_abstract_minimal_connector.py b/bookwyrm/tests/connectors/test_abstract_minimal_connector.py index 589072123..a90ce0c7e 100644 --- a/bookwyrm/tests/connectors/test_abstract_minimal_connector.py +++ b/bookwyrm/tests/connectors/test_abstract_minimal_connector.py @@ -3,7 +3,6 @@ from django.test import TestCase import responses from bookwyrm import models -from bookwyrm.book_search import SearchResult from bookwyrm.connectors import abstract_connector from bookwyrm.connectors.abstract_connector import Mapping @@ -94,19 +93,6 @@ class AbstractConnector(TestCase): results = self.test_connector.isbn_search("123456") self.assertEqual(len(results), 10) - def test_search_result(self): - """a class that stores info about a search result""" - result = SearchResult( - title="Title", - key="https://example.com/book/1", - author="Author Name", - year="1850", - connector=self.test_connector, - ) - # there's really not much to test here, it's just a dataclass - self.assertEqual(result.confidence, 1) - self.assertEqual(result.title, "Title") - def test_create_mapping(self): """maps remote fields for book data to bookwyrm activitypub fields""" mapping = Mapping("isbn") diff --git a/bookwyrm/tests/connectors/test_connector_manager.py b/bookwyrm/tests/connectors/test_connector_manager.py index d6d82fc8c..c88a8036a 100644 --- a/bookwyrm/tests/connectors/test_connector_manager.py +++ b/bookwyrm/tests/connectors/test_connector_manager.py @@ -14,7 +14,7 @@ class ConnectorManager(TestCase): """we'll need some books and a connector info entry""" self.work = models.Work.objects.create(title="Example Work") - self.edition = models.Edition.objects.create( + models.Edition.objects.create( title="Example Edition", parent_work=self.work, isbn_10="0000000000" ) self.edition = models.Edition.objects.create( diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py new file mode 100644 index 000000000..4b9a06811 --- /dev/null +++ b/bookwyrm/tests/test_book_search.py @@ -0,0 +1,118 @@ +""" test searching for books """ +import datetime +from django.test import TestCase +from django.utils import timezone + +from bookwyrm import book_search, models +from bookwyrm.connectors.abstract_connector import AbstractMinimalConnector + + +class BookSearch(TestCase): + """look for some books""" + + def setUp(self): + """we need basic test data and mocks""" + self.work = models.Work.objects.create(title="Example Work") + + self.first_edition = models.Edition.objects.create( + title="Example Edition", + parent_work=self.work, + isbn_10="0000000000", + physical_format="Paperback", + published_date=datetime.datetime(2019, 4, 9, 0, 0, tzinfo=timezone.utc), + ) + self.second_edition = models.Edition.objects.create( + title="Another Edition", + parent_work=self.work, + isbn_10="1111111111", + openlibrary_key="hello", + ) + + def test_search(self): + """search for a book in the db""" + # title/author + results = book_search.search("Example") + self.assertEqual(len(results), 1) + self.assertEqual(results[0], self.first_edition) + + # isbn + results = book_search.search("0000000000") + self.assertEqual(len(results), 1) + self.assertEqual(results[0], self.first_edition) + + # identifier + results = book_search.search("hello") + self.assertEqual(len(results), 1) + self.assertEqual(results[0], self.second_edition) + + def test_isbn_search(self): + """test isbn search""" + results = book_search.isbn_search("0000000000") + self.assertEqual(len(results), 1) + self.assertEqual(results[0], self.first_edition) + + def test_search_identifiers(self): + """search by unique identifiers""" + results = book_search.search_identifiers("hello") + self.assertEqual(len(results), 1) + self.assertEqual(results[0], self.second_edition) + + def test_search_title_author(self): + """search by unique identifiers""" + results = book_search.search_title_author("Another", min_confidence=0) + self.assertEqual(len(results), 1) + self.assertEqual(results[0], self.second_edition) + + def test_format_search_result(self): + """format a search result""" + result = book_search.format_search_result(self.first_edition) + self.assertEqual(result["title"], "Example Edition") + self.assertEqual(result["key"], self.first_edition.remote_id) + self.assertEqual(result["year"], 2019) + + result = book_search.format_search_result(self.second_edition) + self.assertEqual(result["title"], "Another Edition") + self.assertEqual(result["key"], self.second_edition.remote_id) + self.assertIsNone(result["year"]) + + def test_search_result(self): + """a class that stores info about a search result""" + models.Connector.objects.create( + identifier="example.com", + connector_file="openlibrary", + base_url="https://example.com", + books_url="https://example.com/books", + covers_url="https://example.com/covers", + search_url="https://example.com/search?q=", + isbn_search_url="https://example.com/isbn?q=", + ) + + class TestConnector(AbstractMinimalConnector): + """nothing added here""" + + def format_search_result(self, search_result): + return search_result + + def get_or_create_book(self, remote_id): + pass + + def parse_search_data(self, data): + return data + + def format_isbn_search_result(self, search_result): + return search_result + + def parse_isbn_search_data(self, data): + return data + + test_connector = TestConnector("example.com") + result = book_search.SearchResult( + title="Title", + key="https://example.com/book/1", + author="Author Name", + year="1850", + connector=test_connector, + ) + # there's really not much to test here, it's just a dataclass + self.assertEqual(result.confidence, 1) + self.assertEqual(result.title, "Title")