Merge pull request #111 from mouse-reeve/book-datasources

Adds book data source connector database table
This commit is contained in:
Mouse Reeve 2020-03-27 18:37:54 -07:00 committed by GitHub
commit 16855228b0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 210 additions and 74 deletions

View file

@ -1,11 +1,37 @@
''' select and call a connector for whatever book task needs doing '''
from fedireads.connectors import OpenLibraryConnector
import importlib
from fedireads import models
openlibrary = OpenLibraryConnector()
def get_or_create_book(key):
''' pull up a book record by whatever means possible '''
return openlibrary.get_or_create_book(key)
try:
book = models.Book.objects.select_subclasses().get(
fedireads_key=key
)
return book
except models.Book.DoesNotExist:
pass
connector = get_connector()
return connector.get_or_create_book(key)
def search(query):
''' ya '''
return openlibrary.search(query)
connector = get_connector()
return connector.search(query)
def get_connector(book=None):
''' pick a book data connector '''
if book and book.connector:
connector_info = book.connector
else:
connector_info = models.Connector.objects.first()
connector = importlib.import_module(
'fedireads.connectors.%s' % connector_info.connector_file
)
return connector.Connector(connector_info.identifier)

View file

@ -1,3 +1,2 @@
''' bring connectors into the namespace '''
from .settings import CONNECTORS
from .openlibrary import OpenLibraryConnector

View file

@ -1,28 +1,31 @@
''' functionality outline for a book data connector '''
from abc import ABC, abstractmethod
from fedireads.connectors import CONNECTORS
from fedireads import models
class AbstractConnector(ABC):
''' generic book data connector '''
def __init__(self, connector_name):
def __init__(self, identifier):
# load connector settings
settings = CONNECTORS.get(connector_name)
if not settings:
raise ValueError('No connector with name "%s"' % connector_name)
info = models.Connector.objects.get(identifier=identifier)
self.model = info
try:
self.url = settings['BASE_URL']
self.covers_url = settings['COVERS_URL']
self.db_field = settings['DB_KEY_FIELD']
self.key_name = settings['KEY_NAME']
except KeyError:
raise KeyError('Invalid connector settings')
# TODO: politeness settings
self.url = info.base_url
self.covers_url = info.covers_url
self.search_url = info.search_url
self.key_name = info.key_name
self.max_query_count = info.max_query_count
def is_available(self):
''' check if you're allowed to use this connector '''
if self.model.max_query_count is not None:
if self.model.query_count >= self.model.max_query_count:
return False
return True
@abstractmethod
def search(self, query):
''' free text search '''
@ -61,4 +64,5 @@ class SearchResult(object):
self.raw_data = raw_data
def __repr__(self):
return "<SearchResult key={!r} title={!r} author={!r}>".format(self.key, self.title, self.author)
return "<SearchResult key={!r} title={!r} author={!r}>".format(
self.key, self.title, self.author)

View file

@ -8,10 +8,10 @@ from fedireads import models
from .abstract_connector import AbstractConnector, SearchResult
class OpenLibraryConnector(AbstractConnector):
class Connector(AbstractConnector):
''' instantiate a connector for OL '''
def __init__(self):
super().__init__('openlibrary')
def __init__(self, identifier):
super().__init__(identifier)
def search(self, query):
@ -76,6 +76,7 @@ class OpenLibraryConnector(AbstractConnector):
key = data.get('works')[0]['key']
key = key.split('/')[-1]
work = self.get_or_create_book(key)
book.parent_work = work
# we also need to know the author get the cover
@ -138,3 +139,4 @@ class OpenLibraryConnector(AbstractConnector):
def update_book(self, book_obj):
pass

View file

@ -1,28 +1,3 @@
''' settings book data connectors '''
CONNECTORS = {
'openlibrary': {
'KEY_NAME': 'olkey',
'DB_KEY_FIELD': 'openlibrary_key',
'POLITENESS_DELAY': 0,
'MAX_DAILY_QUERIES': -1,
'BASE_URL': 'https://openlibrary.org',
'COVERS_URL': 'https://covers.openlibrary.org',
},
}
''' not implemented yet:
'librarything': {
'KEY_NAME': 'ltkey',
'DB_KEY_FIELD': 'librarything_key',
'POLITENESS_DELAY': 1,
'MAX_DAILY_QUERIES': 1000,
'BASE_URL': 'https://librarything.com',
},
'worldcat': {
'KEY_NAME': 'ocn',
'DB_KEY_FIELD': 'oclc_number',
'POLITENESS_DELAY': 0,
'MAX_DAILY_QUERIES': -1,
'BASE_URL': 'https://worldcat.org',
},
'''
CONNECTORS = ['openlibrary', 'fedireads_connector']

View file

@ -0,0 +1,58 @@
# Generated by Django 3.0.3 on 2020-03-27 23:35
from django.db import migrations, models
import django.db.models.deletion
import fedireads.models.book
class Migration(migrations.Migration):
dependencies = [
('fedireads', '0019_comment'),
]
operations = [
migrations.CreateModel(
name='Connector',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_date', models.DateTimeField(auto_now_add=True)),
('updated_date', models.DateTimeField(auto_now=True)),
('identifier', models.CharField(max_length=255, unique=True)),
('connector_file', models.CharField(choices=[('openlibrary', 'Openlibrary'), ('fedireads', 'Fedireads')], default='openlibrary', max_length=255)),
('is_self', models.BooleanField(default=False)),
('api_key', models.CharField(max_length=255, null=True)),
('base_url', models.CharField(max_length=255)),
('covers_url', models.CharField(max_length=255)),
('search_url', models.CharField(max_length=255, null=True)),
('key_name', models.CharField(max_length=255)),
('politeness_delay', models.IntegerField(null=True)),
('max_query_count', models.IntegerField(null=True)),
('query_count', models.IntegerField(default=0)),
('query_count_expiry', models.DateTimeField(auto_now_add=True)),
],
),
migrations.RenameField(
model_name='book',
old_name='local_key',
new_name='fedireads_key',
),
migrations.RenameField(
model_name='book',
old_name='origin',
new_name='source_url',
),
migrations.RemoveField(
model_name='book',
name='local_edits',
),
migrations.AddConstraint(
model_name='connector',
constraint=models.CheckConstraint(check=models.Q(connector_file__in=fedireads.models.book.ConnectorFiles), name='connector_file_valid'),
),
migrations.AddField(
model_name='book',
name='connector',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='fedireads.Connector'),
),
]

View file

@ -1,5 +1,5 @@
''' bring all the models into the app namespace '''
from .book import Book, Work, Edition, Author
from .book import Connector, Book, Work, Edition, Author
from .shelf import Shelf, ShelfBook
from .status import Status, Review, Comment, Favorite, Tag, Notification
from .user import User, FederatedServer, UserFollows, UserFollowRequest, \

View file

@ -1,26 +1,66 @@
''' database schema for books and shelves '''
from datetime import datetime
from django.db import models
from model_utils.managers import InheritanceManager
from uuid import uuid4
from fedireads.settings import DOMAIN
from fedireads.utils.fields import JSONField, ArrayField
from fedireads.utils.models import FedireadsModel
from fedireads.connectors.settings import CONNECTORS
ConnectorFiles = models.TextChoices('ConnectorFiles', CONNECTORS)
class Connector(FedireadsModel):
''' book data source connectors '''
identifier = models.CharField(max_length=255, unique=True)
connector_file = models.CharField(
max_length=255,
default='openlibrary',
choices=ConnectorFiles.choices
)
# is this a connector to your own database, should only be true if
# the connector_file is `fedireads`
is_self = models.BooleanField(default=False)
api_key = models.CharField(max_length=255, null=True)
base_url = models.CharField(max_length=255)
covers_url = models.CharField(max_length=255)
search_url = models.CharField(max_length=255, null=True)
key_name = models.CharField(max_length=255)
politeness_delay = models.IntegerField(null=True) #seconds
max_query_count = models.IntegerField(null=True)
# how many queries executed in a unit of time, like a day
query_count = models.IntegerField(default=0)
# when to reset the query count back to 0 (ie, after 1 day)
query_count_expiry = models.DateTimeField(auto_now_add=True)
class Meta:
constraints = [
models.CheckConstraint(
check=models.Q(connector_file__in=ConnectorFiles),
name='connector_file_valid'
)
]
class Book(FedireadsModel):
''' a generic book, which can mean either an edition or a work '''
# these identifiers apply to both works and editions
openlibrary_key = models.CharField(max_length=255, unique=True, null=True)
librarything_key = models.CharField(max_length=255, unique=True, null=True)
local_key = models.CharField(max_length=255, unique=True, default=uuid4)
fedireads_key = models.CharField(max_length=255, unique=True, default=uuid4)
misc_identifiers = JSONField(null=True)
# info about where the data comes from and where/if to sync
origin = models.CharField(max_length=255, unique=True, null=True)
local_edits = models.BooleanField(default=False)
source_url = models.CharField(max_length=255, unique=True, null=True)
sync = models.BooleanField(default=True)
last_sync_date = models.DateTimeField(default=datetime.now)
connector = models.ForeignKey(
'Connector', on_delete=models.PROTECT, null=True)
# TODO: edit history
@ -44,8 +84,8 @@ class Book(FedireadsModel):
through='ShelfBook',
through_fields=('book', 'shelf')
)
# TODO: why can't I just call this work????
parent_work = models.ForeignKey('Work', on_delete=models.PROTECT, null=True)
objects = InheritanceManager()
@property
def absolute_id(self):
@ -55,7 +95,12 @@ class Book(FedireadsModel):
return '%s/%s/%s' % (base_path, model_name, self.openlibrary_key)
def __repr__(self):
return "<{} key={!r} title={!r} author={!r}>".format(self.__class__, self.openlibrary_key, self.title, self.author)
return "<{} key={!r} title={!r} author={!r}>".format(
self.__class__,
self.openlibrary_key,
self.title,
self.author
)
class Work(Book):
@ -82,6 +127,8 @@ class Author(FedireadsModel):
name = models.CharField(max_length=255)
last_name = models.CharField(max_length=255, null=True)
first_name = models.CharField(max_length=255, null=True)
aliases = ArrayField(models.CharField(max_length=255), blank=True, default=list)
aliases = ArrayField(
models.CharField(max_length=255), blank=True, default=list
)
bio = models.TextField(null=True, blank=True)

View file

@ -232,7 +232,7 @@ def handle_tag(user, book, name):
def handle_untag(user, book, name):
''' tag a book '''
book = models.Book.objects.get(openlibrary_key=book)
book = models.Book.objects.get(fedireads_key=book)
tag = models.Tag.objects.get(name=name, book=book, user=user)
tag_activity = activitypub.get_remove_tag(tag)
tag.delete()

View file

@ -5,7 +5,7 @@
<h2><q>{{ book.title }}</q> by
{% include 'snippets/authors.html' with book=book %}</h2>
<div>
{% if book.parent_work %}<p>Edition of <a href="/book/{{ book.parent_work.openlibrary_key }}">{{ book.parent_work.title }}</a></p>{% endif %}
{% if book.parent_work %}<p>Edition of <a href="/book/{{ book.parent_work.fedireads_key }}">{{ book.parent_work.title }}</a></p>{% endif %}
<div class="book-preview">
{% include 'snippets/book_cover.html' with book=book size=large %}
@ -32,7 +32,7 @@
<h2>Leave a review</h2>
<form class="review-form" name="review" action="/review/" method="post">
{% csrf_token %}
<input type="hidden" name="book" value="{{ book.openlibrary_key }}"></input>
<input type="hidden" name="book" value="{{ book.fedireads_key }}"></input>
{{ review_form.as_p }}
<button type="submit">Post review</button>
</form>

View file

@ -1 +1 @@
<a href="/author/{{ book.authors.first.openlibrary_key }}" class="author">{{ book.authors.first.name }}</a>
<a href="/author/{{ book.authors.first.fedireads_key }}" class="author">{{ book.authors.first.name }}</a>

View file

@ -1,7 +1,7 @@
{% load fr_display %}
{% include 'snippets/book_cover.html' with book=book %}
<p class="title">
<a href="/book/{{ book.openlibrary_key }}">{{ book.title }}</a>
<a href="/book/{{ book.fedireads_key }}">{{ book.title }}</a>
</p>
<p>
by {% include 'snippets/authors.html' with book=book %}

View file

@ -4,7 +4,7 @@
<h2>
{% include 'snippets/avatar.html' with user=user %}
Your thoughts on
<a href="/book/{{ book.openlibrary_key }}">{{ book.title }}</a>
<a href="/book/{{ book.fedireads_key }}">{{ book.title }}</a>
by {% include 'snippets/authors.html' with book=book %}
</h2>
@ -24,16 +24,14 @@
{% include 'snippets/book_cover.html' with book=book %}
<form class="tab-option-{{ book.id }} review-form" name="review" action="/review/" method="post" id="tab-review-{{ book.id }}">
{% csrf_token %}
{# todo: this shouldn't use the openlibrary key #}
<input type="hidden" name="book" value="{{ book.openlibrary_key }}"></input>
<input type="hidden" name="book" value="{{ book.fedireads_key }}"></input>
{{ review_form.as_p }}
<button type="submit">post review</button>
</form>
<form class="hidden tab-option-{{ book.id }} review-form" name="comment" action="/comment/" method="post" id="tab-comment-{{ book.id }}">
{% csrf_token %}
{# todo: this shouldn't use the openlibrary key #}
<input type="hidden" name="book" value="{{ book.openlibrary_key }}"></input>
<input type="hidden" name="book" value="{{ book.fedireads_key }}"></input>
{{ comment_form.as_p }}
<button type="submit">post comment</button>
</form>

View file

@ -33,7 +33,7 @@
{% include 'snippets/book_cover.html' with book=book size="small" %}
</td>
<td>
<a href="/book/{{ book.openlibrary_key }}">{{ book.title }}</a>
<a href="/book/{{ book.fedireads_key }}">{{ book.title }}</a>
</td>
<td>
{{ book.authors.first.name }}
@ -45,7 +45,7 @@
{{ book.created_date | naturalday }}
</td>
<td>
<a href="https://openlibrary.org{{ book.key }}" target="_blank">OpenLibrary</a>
<a href="https://openlibrary.org/book/{{ book.openlibrary_key }}" target="_blank">OpenLibrary</a>
</td>
{% if ratings %}
<td>

View file

@ -3,14 +3,14 @@
{% if tag.identifier in user_tags %}
<form class="tag-form" name="tag" action="/untag/" method="post">
{% csrf_token %}
<input type="hidden" name="book" value="{{ book.openlibrary_key }}"></input>
<input type="hidden" name="book" value="{{ book.fedireads_key }}"></input>
<input type="hidden" name="name" value="{{ tag.name }}"></input>
<button type="submit">x</button>
</form>
{% else %}
<form class="tag-form" name="tag" action="/tag/" method="post">
{% csrf_token %}
<input type="hidden" name="book" value="{{ book.openlibrary_key }}"></input>
<input type="hidden" name="book" value="{{ book.fedireads_key }}"></input>
<input type="hidden" name="name" value="{{ tag.name }}"></input>
<button type="submit">+</button>
</form>

View file

@ -56,8 +56,8 @@ urlpatterns = [
re_path(r'%s/replies\.json$' % status_path, views.replies_page),
# books
re_path(r'^book/(?P<book_identifier>\w+)/?$', views.book_page),
re_path(r'^book/(?P<book_identifier>\w+)/(?P<tab>friends|local|federated)?$', views.book_page),
re_path(r'^book/(?P<book_identifier>[\w\-]+)/?$', views.book_page),
re_path(r'^book/(?P<book_identifier>[\w\-]+)/(?P<tab>friends|local|federated)?$', views.book_page),
re_path(r'^author/(?P<author_identifier>\w+)/?$', views.author_page),
re_path(r'^tag/(?P<tag_id>.+)/?$', views.tag_page),
re_path(r'^shelf/%s/(?P<shelf_identifier>[\w-]+)/?$' % username_regex, views.shelf_page),

View file

@ -100,7 +100,9 @@ def books_page(request):
''' discover books '''
recent_books = models.Book.objects
if request.user.is_authenticated:
recent_books = recent_books.filter(~Q(shelfbook__shelf__user=request.user))
recent_books = recent_books.filter(
~Q(shelfbook__shelf__user=request.user)
)
recent_books = recent_books.order_by('-created_date')[:50]
data = {

View file

@ -1,10 +1,35 @@
from fedireads.models import User
''' starter data '''
from fedireads.books_manager import get_or_create_book
from fedireads.models import Connector, User
from fedireads.settings import DOMAIN
User.objects.create_user('mouse', 'mouse.reeve@gmail.com', 'password123')
User.objects.create_user('rat', 'rat@rat.com', 'ratword', manually_approves_followers=True)
User.objects.create_user(
'rat', 'rat@rat.com', 'ratword',
manually_approves_followers=True
)
User.objects.get(id=1).followers.add(User.objects.get(id=2))
Connector.objects.create(
identifier='openlibrary.org',
connector_file='openlibrary',
base_url='https://openlibrary.org',
covers_url='https://covers.openlibrary.org',
search_url='https://openlibrary.org/search?q=',
key_name='openlibrary_key',
)
Connector.objects.create(
identifier=DOMAIN,
connector_file='fedireads_connector',
base_url='https://%s/book' % DOMAIN,
covers_url='https://%s/images/covers' % DOMAIN,
search_url='https://%s/search?q=' % DOMAIN,
key_name='openlibrary_key',
is_self=True
)
get_or_create_book('OL1715344W')
get_or_create_book('OL102749W')