Merge pull request #111 from mouse-reeve/book-datasources

Adds book data source connector database table

fedireads/books_manager.py

@@ -1,11 +1,37 @@
 ''' select and call a connector for whatever book task needs doing '''
-from fedireads.connectors import OpenLibraryConnector
+import importlib
+
+from fedireads import models
+
 
-openlibrary = OpenLibraryConnector()
 def get_or_create_book(key):
     ''' pull up a book record by whatever means possible '''
-    return openlibrary.get_or_create_book(key)
+    try:
+        book = models.Book.objects.select_subclasses().get(
+            fedireads_key=key
+        )
+        return book
+    except models.Book.DoesNotExist:
+        pass
+
+    connector = get_connector()
+    return connector.get_or_create_book(key)
+
 
 def search(query):
     ''' ya '''
-    return openlibrary.search(query)
+    connector = get_connector()
+    return connector.search(query)
+
+
+def get_connector(book=None):
+    ''' pick a book data connector '''
+    if book and book.connector:
+        connector_info = book.connector
+    else:
+        connector_info = models.Connector.objects.first()
+
+    connector = importlib.import_module(
+        'fedireads.connectors.%s' % connector_info.connector_file
+    )
+    return connector.Connector(connector_info.identifier)

fedireads/connectors/__init__.py

@@ -1,3 +1,2 @@
 ''' bring connectors into the namespace '''
 from .settings import CONNECTORS
-from .openlibrary import OpenLibraryConnector

fedireads/connectors/abstract_connector.py

@@ -1,28 +1,31 @@
 ''' functionality outline for a book data connector '''
 from abc import ABC, abstractmethod
 
-from fedireads.connectors import CONNECTORS
+from fedireads import models
 
 
 class AbstractConnector(ABC):
     ''' generic book data connector '''
 
-    def __init__(self, connector_name):
+    def __init__(self, identifier):
         # load connector settings
-        settings = CONNECTORS.get(connector_name)
+        info = models.Connector.objects.get(identifier=identifier)
-        if not settings:
+        self.model = info
-            raise ValueError('No connector with name "%s"' % connector_name)
 
-        try:
+        self.url = info.base_url
-            self.url = settings['BASE_URL']
+        self.covers_url = info.covers_url
-            self.covers_url = settings['COVERS_URL']
+        self.search_url = info.search_url
-            self.db_field = settings['DB_KEY_FIELD']
+        self.key_name = info.key_name
-            self.key_name = settings['KEY_NAME']
+        self.max_query_count = info.max_query_count
-        except KeyError:
-            raise KeyError('Invalid connector settings')
-        # TODO: politeness settings
 
 
+    def is_available(self):
+        ''' check if you're allowed to use this connector '''
+        if self.model.max_query_count is not None:
+            if self.model.query_count >= self.model.max_query_count:
+                return False
+        return True
+
     @abstractmethod
     def search(self, query):
         ''' free text search '''
@@ -61,4 +64,5 @@ class SearchResult(object):
         self.raw_data = raw_data
 
     def __repr__(self):
-        return "<SearchResult key={!r} title={!r} author={!r}>".format(self.key, self.title, self.author)
+        return "<SearchResult key={!r} title={!r} author={!r}>".format(
+            self.key, self.title, self.author)

fedireads/connectors/openlibrary.py

@@ -8,10 +8,10 @@ from fedireads import models
 from .abstract_connector import AbstractConnector, SearchResult
 
 
-class OpenLibraryConnector(AbstractConnector):
+class Connector(AbstractConnector):
     ''' instantiate a connector for OL '''
-    def __init__(self):
+    def __init__(self, identifier):
-        super().__init__('openlibrary')
+        super().__init__(identifier)
 
 
     def search(self, query):
@@ -76,6 +76,7 @@ class OpenLibraryConnector(AbstractConnector):
             key = data.get('works')[0]['key']
             key = key.split('/')[-1]
             work = self.get_or_create_book(key)
+
             book.parent_work = work
 
         # we also need to know the author get the cover
@@ -138,3 +139,4 @@ class OpenLibraryConnector(AbstractConnector):
 
     def update_book(self, book_obj):
         pass
+

fedireads/connectors/settings.py

@@ -1,28 +1,3 @@
 ''' settings book data connectors '''
-CONNECTORS = {
-    'openlibrary': {
-        'KEY_NAME': 'olkey',
-        'DB_KEY_FIELD': 'openlibrary_key',
-        'POLITENESS_DELAY': 0,
-        'MAX_DAILY_QUERIES': -1,
-        'BASE_URL': 'https://openlibrary.org',
-        'COVERS_URL': 'https://covers.openlibrary.org',
-    },
-}
 
-''' not implemented yet:
+CONNECTORS = ['openlibrary', 'fedireads_connector']
-    'librarything': {
-        'KEY_NAME': 'ltkey',
-        'DB_KEY_FIELD': 'librarything_key',
-        'POLITENESS_DELAY': 1,
-        'MAX_DAILY_QUERIES': 1000,
-        'BASE_URL': 'https://librarything.com',
-    },
-    'worldcat': {
-        'KEY_NAME': 'ocn',
-        'DB_KEY_FIELD': 'oclc_number',
-        'POLITENESS_DELAY': 0,
-        'MAX_DAILY_QUERIES': -1,
-        'BASE_URL': 'https://worldcat.org',
-    },
-'''

fedireads/migrations/0020_auto_20200327_2335.py

@@ -0,0 +1,58 @@
+# Generated by Django 3.0.3 on 2020-03-27 23:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+import fedireads.models.book
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fedireads', '0019_comment'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Connector',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_date', models.DateTimeField(auto_now_add=True)),
+                ('updated_date', models.DateTimeField(auto_now=True)),
+                ('identifier', models.CharField(max_length=255, unique=True)),
+                ('connector_file', models.CharField(choices=[('openlibrary', 'Openlibrary'), ('fedireads', 'Fedireads')], default='openlibrary', max_length=255)),
+                ('is_self', models.BooleanField(default=False)),
+                ('api_key', models.CharField(max_length=255, null=True)),
+                ('base_url', models.CharField(max_length=255)),
+                ('covers_url', models.CharField(max_length=255)),
+                ('search_url', models.CharField(max_length=255, null=True)),
+                ('key_name', models.CharField(max_length=255)),
+                ('politeness_delay', models.IntegerField(null=True)),
+                ('max_query_count', models.IntegerField(null=True)),
+                ('query_count', models.IntegerField(default=0)),
+                ('query_count_expiry', models.DateTimeField(auto_now_add=True)),
+            ],
+        ),
+        migrations.RenameField(
+            model_name='book',
+            old_name='local_key',
+            new_name='fedireads_key',
+        ),
+        migrations.RenameField(
+            model_name='book',
+            old_name='origin',
+            new_name='source_url',
+        ),
+        migrations.RemoveField(
+            model_name='book',
+            name='local_edits',
+        ),
+        migrations.AddConstraint(
+            model_name='connector',
+            constraint=models.CheckConstraint(check=models.Q(connector_file__in=fedireads.models.book.ConnectorFiles), name='connector_file_valid'),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='connector',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='fedireads.Connector'),
+        ),
+    ]

fedireads/models/__init__.py

@@ -1,5 +1,5 @@
 ''' bring all the models into the app namespace '''
-from .book import Book, Work, Edition, Author
+from .book import Connector, Book, Work, Edition, Author
 from .shelf import Shelf, ShelfBook
 from .status import Status, Review, Comment, Favorite, Tag, Notification
 from .user import User, FederatedServer, UserFollows, UserFollowRequest, \

fedireads/models/book.py

@@ -1,26 +1,66 @@
 ''' database schema for books and shelves '''
 from datetime import datetime
 from django.db import models
+from model_utils.managers import InheritanceManager
 from uuid import uuid4
 
 from fedireads.settings import DOMAIN
 from fedireads.utils.fields import JSONField, ArrayField
 from fedireads.utils.models import FedireadsModel
 
+from fedireads.connectors.settings import CONNECTORS
+
+
+ConnectorFiles = models.TextChoices('ConnectorFiles', CONNECTORS)
+class Connector(FedireadsModel):
+    ''' book data source connectors '''
+    identifier = models.CharField(max_length=255, unique=True)
+    connector_file = models.CharField(
+        max_length=255,
+        default='openlibrary',
+        choices=ConnectorFiles.choices
+    )
+    # is this a connector to your own database, should only be true if
+    # the connector_file is `fedireads`
+    is_self = models.BooleanField(default=False)
+    api_key = models.CharField(max_length=255, null=True)
+
+    base_url = models.CharField(max_length=255)
+    covers_url = models.CharField(max_length=255)
+    search_url = models.CharField(max_length=255, null=True)
+
+    key_name = models.CharField(max_length=255)
+
+    politeness_delay = models.IntegerField(null=True) #seconds
+    max_query_count = models.IntegerField(null=True)
+    # how many queries executed in a unit of time, like a day
+    query_count = models.IntegerField(default=0)
+    # when to reset the query count back to 0 (ie, after 1 day)
+    query_count_expiry = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        constraints = [
+            models.CheckConstraint(
+                check=models.Q(connector_file__in=ConnectorFiles),
+                name='connector_file_valid'
+            )
+        ]
+
 
 class Book(FedireadsModel):
     ''' a generic book, which can mean either an edition or a work '''
     # these identifiers apply to both works and editions
     openlibrary_key = models.CharField(max_length=255, unique=True, null=True)
     librarything_key = models.CharField(max_length=255, unique=True, null=True)
-    local_key = models.CharField(max_length=255, unique=True, default=uuid4)
+    fedireads_key = models.CharField(max_length=255, unique=True, default=uuid4)
     misc_identifiers = JSONField(null=True)
 
     # info about where the data comes from and where/if to sync
-    origin = models.CharField(max_length=255, unique=True, null=True)
+    source_url = models.CharField(max_length=255, unique=True, null=True)
-    local_edits = models.BooleanField(default=False)
     sync = models.BooleanField(default=True)
     last_sync_date = models.DateTimeField(default=datetime.now)
+    connector = models.ForeignKey(
+        'Connector', on_delete=models.PROTECT, null=True)
 
     # TODO: edit history
 
@@ -44,8 +84,8 @@ class Book(FedireadsModel):
         through='ShelfBook',
         through_fields=('book', 'shelf')
     )
-    # TODO: why can't I just call this work????
     parent_work = models.ForeignKey('Work', on_delete=models.PROTECT, null=True)
+    objects = InheritanceManager()
 
     @property
     def absolute_id(self):
@@ -55,7 +95,12 @@ class Book(FedireadsModel):
         return '%s/%s/%s' % (base_path, model_name, self.openlibrary_key)
 
     def __repr__(self):
-        return "<{} key={!r} title={!r} author={!r}>".format(self.__class__, self.openlibrary_key, self.title, self.author)
+        return "<{} key={!r} title={!r} author={!r}>".format(
+            self.__class__,
+            self.openlibrary_key,
+            self.title,
+            self.author
+        )
 
 
 class Work(Book):
@@ -82,6 +127,8 @@ class Author(FedireadsModel):
     name = models.CharField(max_length=255)
     last_name = models.CharField(max_length=255, null=True)
     first_name = models.CharField(max_length=255, null=True)
-    aliases = ArrayField(models.CharField(max_length=255), blank=True, default=list)
+    aliases = ArrayField(
+        models.CharField(max_length=255), blank=True, default=list
+    )
     bio = models.TextField(null=True, blank=True)
 

fedireads/outgoing.py

@@ -232,7 +232,7 @@ def handle_tag(user, book, name):
 
 def handle_untag(user, book, name):
     ''' tag a book '''
-    book = models.Book.objects.get(openlibrary_key=book)
+    book = models.Book.objects.get(fedireads_key=book)
     tag = models.Tag.objects.get(name=name, book=book, user=user)
     tag_activity = activitypub.get_remove_tag(tag)
     tag.delete()

fedireads/templates/book.html

@@ -5,7 +5,7 @@
     <h2><q>{{ book.title }}</q> by
         {% include 'snippets/authors.html' with book=book %}</h2>
     <div>
-        {% if book.parent_work %}<p>Edition of <a href="/book/{{ book.parent_work.openlibrary_key }}">{{ book.parent_work.title }}</a></p>{% endif %}
+        {% if book.parent_work %}<p>Edition of <a href="/book/{{ book.parent_work.fedireads_key }}">{{ book.parent_work.title }}</a></p>{% endif %}
         <div class="book-preview">
 
             {% include 'snippets/book_cover.html' with book=book size=large %}
@@ -32,7 +32,7 @@
     <h2>Leave a review</h2>
     <form class="review-form" name="review" action="/review/" method="post">
         {% csrf_token %}
-        <input type="hidden" name="book" value="{{ book.openlibrary_key }}"></input>
+        <input type="hidden" name="book" value="{{ book.fedireads_key }}"></input>
         {{ review_form.as_p }}
         <button type="submit">Post review</button>
     </form>

fedireads/templates/snippets/authors.html

@@ -1 +1 @@
-<a href="/author/{{ book.authors.first.openlibrary_key }}" class="author">{{ book.authors.first.name }}</a>
+<a href="/author/{{ book.authors.first.fedireads_key }}" class="author">{{ book.authors.first.name }}</a>

fedireads/templates/snippets/book.html

@@ -1,7 +1,7 @@
 {% load fr_display %}
 {% include 'snippets/book_cover.html' with book=book %}
 <p class="title">
-    <a href="/book/{{ book.openlibrary_key }}">{{ book.title }}</a>
+    <a href="/book/{{ book.fedireads_key }}">{{ book.title }}</a>
 </p>
 <p>
     by {% include 'snippets/authors.html' with book=book %}

fedireads/templates/snippets/create_status.html

@@ -4,7 +4,7 @@
 <h2>
     {% include 'snippets/avatar.html' with user=user %}
     Your thoughts on
-    <a href="/book/{{ book.openlibrary_key }}">{{ book.title }}</a>
+    <a href="/book/{{ book.fedireads_key }}">{{ book.title }}</a>
     by {% include 'snippets/authors.html' with book=book %}
 </h2>
 
@@ -24,16 +24,14 @@
     {% include 'snippets/book_cover.html' with book=book %}
     <form class="tab-option-{{ book.id }} review-form" name="review" action="/review/" method="post" id="tab-review-{{ book.id }}">
         {% csrf_token %}
-        {# todo: this shouldn't use the openlibrary key #}
+        <input type="hidden" name="book" value="{{ book.fedireads_key }}"></input>
-        <input type="hidden" name="book" value="{{ book.openlibrary_key }}"></input>
         {{ review_form.as_p }}
         <button type="submit">post review</button>
     </form>
 
     <form class="hidden tab-option-{{ book.id }} review-form" name="comment" action="/comment/" method="post" id="tab-comment-{{ book.id }}">
         {% csrf_token %}
-        {# todo: this shouldn't use the openlibrary key #}
+        <input type="hidden" name="book" value="{{ book.fedireads_key }}"></input>
-        <input type="hidden" name="book" value="{{ book.openlibrary_key }}"></input>
         {{ comment_form.as_p }}
         <button type="submit">post comment</button>
     </form>

fedireads/templates/snippets/shelf.html

@@ -33,7 +33,7 @@
         {% include 'snippets/book_cover.html' with book=book size="small" %}
     </td>
     <td>
-        <a href="/book/{{ book.openlibrary_key }}">{{ book.title }}</a>
+        <a href="/book/{{ book.fedireads_key }}">{{ book.title }}</a>
     </td>
     <td>
         {{ book.authors.first.name }}
@@ -45,7 +45,7 @@
         {{ book.created_date | naturalday }}
     </td>
     <td>
-        <a href="https://openlibrary.org{{ book.key }}" target="_blank">OpenLibrary</a>
+        <a href="https://openlibrary.org/book/{{ book.openlibrary_key }}" target="_blank">OpenLibrary</a>
     </td>
     {% if ratings %}
     <td>

fedireads/templates/snippets/tag.html

@@ -3,14 +3,14 @@
     {% if tag.identifier in user_tags %}
     <form class="tag-form" name="tag" action="/untag/" method="post">
         {% csrf_token %}
-        <input type="hidden" name="book" value="{{ book.openlibrary_key }}"></input>
+        <input type="hidden" name="book" value="{{ book.fedireads_key }}"></input>
         <input type="hidden" name="name" value="{{ tag.name }}"></input>
         <button type="submit">x</button>
     </form>
     {% else %}
     <form class="tag-form" name="tag" action="/tag/" method="post">
         {% csrf_token %}
-        <input type="hidden" name="book" value="{{ book.openlibrary_key }}"></input>
+        <input type="hidden" name="book" value="{{ book.fedireads_key }}"></input>
         <input type="hidden" name="name" value="{{ tag.name }}"></input>
         <button type="submit">+</button>
     </form>

fedireads/urls.py

@@ -56,8 +56,8 @@ urlpatterns = [
     re_path(r'%s/replies\.json$' % status_path, views.replies_page),
 
     # books
-    re_path(r'^book/(?P<book_identifier>\w+)/?$', views.book_page),
+    re_path(r'^book/(?P<book_identifier>[\w\-]+)/?$', views.book_page),
-    re_path(r'^book/(?P<book_identifier>\w+)/(?P<tab>friends|local|federated)?$', views.book_page),
+    re_path(r'^book/(?P<book_identifier>[\w\-]+)/(?P<tab>friends|local|federated)?$', views.book_page),
     re_path(r'^author/(?P<author_identifier>\w+)/?$', views.author_page),
     re_path(r'^tag/(?P<tag_id>.+)/?$', views.tag_page),
     re_path(r'^shelf/%s/(?P<shelf_identifier>[\w-]+)/?$' % username_regex, views.shelf_page),

fedireads/views.py

@@ -100,7 +100,9 @@ def books_page(request):
     ''' discover books '''
     recent_books = models.Book.objects
     if request.user.is_authenticated:
-        recent_books = recent_books.filter(~Q(shelfbook__shelf__user=request.user))
+        recent_books = recent_books.filter(
+            ~Q(shelfbook__shelf__user=request.user)
+        )
     recent_books = recent_books.order_by('-created_date')[:50]
 
     data = {

init_db.py

@@ -1,10 +1,35 @@
-from fedireads.models import User
+''' starter data '''
 from fedireads.books_manager import get_or_create_book
+from fedireads.models import Connector, User
+from fedireads.settings import DOMAIN
 
 User.objects.create_user('mouse', 'mouse.reeve@gmail.com', 'password123')
-User.objects.create_user('rat', 'rat@rat.com', 'ratword', manually_approves_followers=True)
+User.objects.create_user(
+    'rat', 'rat@rat.com', 'ratword',
+    manually_approves_followers=True
+)
 
 User.objects.get(id=1).followers.add(User.objects.get(id=2))
 
+Connector.objects.create(
+    identifier='openlibrary.org',
+    connector_file='openlibrary',
+    base_url='https://openlibrary.org',
+    covers_url='https://covers.openlibrary.org',
+    search_url='https://openlibrary.org/search?q=',
+    key_name='openlibrary_key',
+)
+
+Connector.objects.create(
+    identifier=DOMAIN,
+    connector_file='fedireads_connector',
+    base_url='https://%s/book' % DOMAIN,
+    covers_url='https://%s/images/covers' % DOMAIN,
+    search_url='https://%s/search?q=' % DOMAIN,
+    key_name='openlibrary_key',
+    is_self=True
+)
+
+
 get_or_create_book('OL1715344W')
 get_or_create_book('OL102749W')