Merge pull request #160 from mouse-reeve/search-controls

Separate out local and remote search results

fedireads/books_manager.py

@@ -30,48 +30,60 @@ def load_more_data(book_id):
 
 
 def search(query):
-    ''' try an external datasource for books '''
+    ''' find books based on arbitary keywords '''
-    self = self_connector()
+    results = []
-    results = self.search(query)
+    dedup_slug = lambda r: '%s/%s/%s' % (r.title, r.author, r.year)
-    if len(results) >= 10:
+    result_index = set()
-        return results
+    for connector in get_connectors():
+        result_set = connector.search(query)
 
-    connector = get_connector()
+        result_set = [r for r in result_set \
-    external_results = connector.search(query)
+                if dedup_slug(r) not in result_index]
-    dedupe_slug = lambda r: '%s %s %s' % (r.title, r.author, r.year)
+        # `|=` concats two sets. WE ARE GETTING FANCY HERE
-    result_index = [dedupe_slug(r) for r in results]
+        result_index |= set(dedup_slug(r) for r in result_set)
-    for result in external_results:
+        results.append({
-        if dedupe_slug(result) in result_index:
+            'connector': connector,
-            continue
+            'results': result_set,
-        results.append(result)
+        })
 
     return results
 
+
+def first_search_result(query):
+    ''' search until you find a result that fits '''
+    for connector in get_connectors():
+        result = connector.search(query)
+        if result:
+            return result[0]
+    return None
+
+
 def update_book(book):
     ''' re-sync with the original data source '''
     connector = get_connector(book)
     connector.update_book(book)
 
 
-def self_connector():
+def get_connectors():
-    ''' load the connector for the local database '''
+    ''' load all connectors '''
-    return get_connector(self=True)
+    connectors_info = models.Connector.objects.order_by('priority').all()
+    return [load_connector(c) for c in connectors_info]
 
 
-def get_connector(book=None, self=False):
+def get_connector(book=None):
     ''' pick a book data connector '''
     if book and book.connector:
         connector_info = book.connector
-    elif self:
-        connector_info = models.Connector.objects.filter(
-            connector_file='self_connector'
-        ).first()
     else:
         # only select from external connectors
-        connector_info = models.Connector.objects.exclude(
+        connector_info = models.Connector.objects.filter(
-            connector_file='self_connector'
+            local=False
-        ).first()
+        ).order_by('priority').first()
+    return load_connector(connector_info)
 
+
+def load_connector(connector_info):
+    ''' instantiate the connector class '''
     connector = importlib.import_module(
         'fedireads.connectors.%s' % connector_info.connector_file
     )

fedireads/connectors/abstract_connector.py

@@ -14,11 +14,14 @@ class AbstractConnector(ABC):
         info = models.Connector.objects.get(identifier=identifier)
         self.connector = info
 
-        self.url = info.base_url
+        self.base_url = info.base_url
+        self.books_url = info.books_url
         self.covers_url = info.covers_url
         self.search_url = info.search_url
         self.key_name = info.key_name
         self.max_query_count = info.max_query_count
+        self.name = info.name
+        self.local = info.local
 
 
     def is_available(self):

fedireads/connectors/openlibrary.py

@@ -166,7 +166,7 @@ class Connector(AbstractConnector):
 
     def load_book_data(self, olkey):
         ''' query openlibrary for data on a book '''
-        response = requests.get('%s/works/%s.json' % (self.url, olkey))
+        response = requests.get('%s/works/%s.json' % (self.books_url, olkey))
         if not response.ok:
             response.raise_for_status()
         data = response.json()
@@ -176,7 +176,7 @@ class Connector(AbstractConnector):
     def load_edition_data(self, olkey):
         ''' query openlibrary for editions of a work '''
         response = requests.get(
-            '%s/works/%s/editions.json' % (self.url, olkey))
+            '%s/works/%s/editions.json' % (self.books_url, olkey))
         if not response.ok:
             response.raise_for_status()
         data = response.json()
@@ -209,7 +209,7 @@ class Connector(AbstractConnector):
         except models.Author.DoesNotExist:
             pass
 
-        response = requests.get('%s/authors/%s.json' % (self.url, olkey))
+        response = requests.get('%s/authors/%s.json' % (self.base_url, olkey))
         if not response.ok:
             response.raise_for_status()
 

fedireads/migrations/0036_auto_20200503_2007.py

@@ -0,0 +1,39 @@
+# Generated by Django 3.0.3 on 2020-05-03 20:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fedireads', '0035_auto_20200429_1708'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='connector',
+            name='books_url',
+            field=models.CharField(default='https://openlibrary.org', max_length=255),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='connector',
+            name='local',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='connector',
+            name='name',
+            field=models.CharField(max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='connector',
+            name='priority',
+            field=models.IntegerField(default=2),
+        ),
+        migrations.AlterField(
+            model_name='connector',
+            name='connector_file',
+            field=models.CharField(choices=[('openlibrary', 'Openlibrary'), ('self_connector', 'Self Connector'), ('fedireads_connector', 'Fedireads Connector')], max_length=255),
+        ),
+    ]

fedireads/models/book.py

@@ -16,14 +16,17 @@ ConnectorFiles = models.TextChoices('ConnectorFiles', CONNECTORS)
 class Connector(FedireadsModel):
     ''' book data source connectors '''
     identifier = models.CharField(max_length=255, unique=True)
+    priority = models.IntegerField(default=2)
+    name = models.CharField(max_length=255, null=True)
+    local = models.BooleanField(default=False)
     connector_file = models.CharField(
         max_length=255,
-        default='openlibrary',
         choices=ConnectorFiles.choices
     )
     api_key = models.CharField(max_length=255, null=True)
 
     base_url = models.CharField(max_length=255)
+    books_url = models.CharField(max_length=255)
     covers_url = models.CharField(max_length=255)
     search_url = models.CharField(max_length=255, null=True)
 

fedireads/models/import_job.py

@@ -53,23 +53,15 @@ class ImportItem(models.Model):
     def resolve(self):
         ''' try various ways to lookup a book '''
         self.book = (
-            self.get_book_from_db_isbn() or
             self.get_book_from_isbn() or
             self.get_book_from_title_author()
         )
 
-    def get_book_from_db_isbn(self):
-        ''' see if we already know about the book '''
-        try:
-            return Edition.objects.filter(isbn_13=self.isbn).first()
-        except Edition.DoesNotExist:
-            return None
-
     def get_book_from_isbn(self):
         ''' search by isbn '''
-        search_results = books_manager.search(self.isbn)
+        search_result = books_manager.first_search_result(self.isbn)
-        if search_results:
+        if search_result:
-            return books_manager.get_or_create_book(search_results[0].key)
+            return books_manager.get_or_create_book(search_result.key)
 
     def get_book_from_title_author(self):
         ''' search by title and author '''
@@ -77,9 +69,9 @@ class ImportItem(models.Model):
             self.data['Title'],
             self.data['Author']
         )
-        search_results = books_manager.search(search_term)
+        search_result = books_manager.first_search_result(search_term)
-        if search_results:
+        if search_result:
-            return books_manager.get_or_create_book(search_results[0].key)
+            return books_manager.get_or_create_book(search_result.key)
 
     @property
     def isbn(self):

fedireads/static/format.css

@@ -45,6 +45,10 @@ h3 small {
     font-weight: normal;
 }
 
+section {
+    margin-bottom: 1em;
+}
+
 
 /* fixed display top bar */
 body {

fedireads/templates/book_results.html

@@ -1,13 +1,21 @@
 {% extends 'layout.html' %}
 {% block content %}
-<div id="content">
+<div class="content-container">
-    <div>
+    <h2>Search results</h2>
-        <h1>Search results</h1>
+    {% for result_set in results %}
-        {% for result in results %}
+    <section>
+        {% if not result_set.connector.local %}
+        <h3>
+            Results from <a href="{{ result_set.connector.base_url }}" target="_blank">{{ result_set.connector.name }}</a>
+        </h3>
+        {% endif %}
+
+        {% for result in result_set.results %}
         <div>
             <a href="/book/{{ result.key }}">{{ result.title }}</a> by {{ result.author }} ({{ result.year }})
         </div>
         {% endfor %}
-    </div>
+    </section>
+    {% endfor %}
 </div>
 {% endblock %}

init_db.py

@@ -13,8 +13,10 @@ User.objects.get(id=1).followers.add(User.objects.get(id=2))
 
 Connector.objects.create(
     identifier='openlibrary.org',
+    name='OpenLibrary',
     connector_file='openlibrary',
     base_url='https://openlibrary.org',
+    books_url='https://openlibrary.org',
     covers_url='https://covers.openlibrary.org',
     search_url='https://openlibrary.org/search?q=',
     key_name='openlibrary_key',
@@ -22,11 +24,15 @@ Connector.objects.create(
 
 Connector.objects.create(
     identifier=DOMAIN,
+    name='Local',
+    local=True,
     connector_file='self_connector',
-    base_url='https://%s/book' % DOMAIN,
+    base_url='https://%s' % DOMAIN,
+    books_url='https://%s/book' % DOMAIN,
     covers_url='https://%s/images/covers' % DOMAIN,
     search_url='https://%s/search?q=' % DOMAIN,
     key_name='openlibrary_key',
+    priority=1,
 )