From 5d2fbb85003e69f6b23063815937b0b2014514da Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 27 Mar 2020 15:25:08 -0700 Subject: [PATCH] Use database as source for initializing connector --- fedireads/books_manager.py | 34 +++++++++++++++---- fedireads/connectors/__init__.py | 1 - fedireads/connectors/abstract_connector.py | 24 ++++++++----- fedireads/connectors/openlibrary.py | 6 ++-- fedireads/connectors/settings.py | 27 ++------------- ...327_2114.py => 0020_auto_20200327_2201.py} | 6 ++-- fedireads/models/book.py | 4 +-- 7 files changed, 56 insertions(+), 46 deletions(-) rename fedireads/migrations/{0020_auto_20200327_2114.py => 0020_auto_20200327_2201.py} (89%) diff --git a/fedireads/books_manager.py b/fedireads/books_manager.py index 15a0517f9..fcaf65704 100644 --- a/fedireads/books_manager.py +++ b/fedireads/books_manager.py @@ -1,15 +1,37 @@ ''' select and call a connector for whatever book task needs doing ''' -from fedireads.connectors import OpenLibraryConnector +import importlib + +from fedireads import models +from fedireads.connectors.settings import CONNECTORS + -openlibrary = OpenLibraryConnector() def get_or_create_book(key): ''' pull up a book record by whatever means possible ''' - return openlibrary.get_or_create_book(key) + try: + book = models.Book.objects.get( + openlibrary_key=key + ).select_subclasses() + return book + except models.Book.DoesNotExist: + pass + + connector = get_connector() + return connector.get_or_create_book(key) + def search(query): ''' ya ''' - return openlibrary.search(query) + connector = get_connector() + return connector.search(query) -def update_book(key): - return openlibrary.update_book(key) +def get_connector(book=None): + ''' pick a book data connector ''' + if book and book.connector: + connector_info = book.connector + else: + connector_info = models.Connector.objects.first() + + classname = CONNECTORS[connector_info.name]['classname'] + connector = importlib.import_module(classname) + return connector.Connector() diff --git a/fedireads/connectors/__init__.py b/fedireads/connectors/__init__.py index 8cee368cd..b525f65f5 100644 --- a/fedireads/connectors/__init__.py +++ b/fedireads/connectors/__init__.py @@ -1,3 +1,2 @@ ''' bring connectors into the namespace ''' from .settings import CONNECTORS -from .openlibrary import OpenLibraryConnector diff --git a/fedireads/connectors/abstract_connector.py b/fedireads/connectors/abstract_connector.py index 6f378a785..7e83b7348 100644 --- a/fedireads/connectors/abstract_connector.py +++ b/fedireads/connectors/abstract_connector.py @@ -1,6 +1,7 @@ ''' functionality outline for a book data connector ''' from abc import ABC, abstractmethod +from fedireads import models from fedireads.connectors import CONNECTORS @@ -13,15 +14,22 @@ class AbstractConnector(ABC): if not settings: raise ValueError('No connector with name "%s"' % connector_name) - try: - self.url = settings['BASE_URL'] - self.covers_url = settings['COVERS_URL'] - self.db_field = settings['DB_KEY_FIELD'] - self.key_name = settings['KEY_NAME'] - except KeyError: - raise KeyError('Invalid connector settings') - # TODO: politeness settings + info = models.Connector.objects.get(name=settings['db_name']) + self.model = info + self.url = info.base_url + self.covers_url = info.covers_url + self.search_url = info.search_url + self.key_name = info.key_name + self.max_query_count = info.max_query_count + + + def is_available(self): + ''' check if you're allowed to use this connector ''' + if self.model.max_query_count is not None: + if self.model.query_count >= self.model.max_query_count: + return False + return True @abstractmethod def search(self, query): diff --git a/fedireads/connectors/openlibrary.py b/fedireads/connectors/openlibrary.py index 1beedb08b..a83a7d630 100644 --- a/fedireads/connectors/openlibrary.py +++ b/fedireads/connectors/openlibrary.py @@ -8,10 +8,10 @@ from fedireads import models from .abstract_connector import AbstractConnector, SearchResult -class OpenLibraryConnector(AbstractConnector): +class Connector(AbstractConnector): ''' instantiate a connector for OL ''' def __init__(self): - super().__init__('openlibrary') + super().__init__('OpenLibrary') def search(self, query): @@ -76,6 +76,8 @@ class OpenLibraryConnector(AbstractConnector): key = data.get('works')[0]['key'] key = key.split('/')[-1] work = self.get_or_create_book(key) + h + book.parent_work = work # we also need to know the author get the cover diff --git a/fedireads/connectors/settings.py b/fedireads/connectors/settings.py index 0b34a22e9..9a0103f00 100644 --- a/fedireads/connectors/settings.py +++ b/fedireads/connectors/settings.py @@ -1,28 +1,7 @@ ''' settings book data connectors ''' CONNECTORS = { - 'openlibrary': { - 'KEY_NAME': 'olkey', - 'DB_KEY_FIELD': 'openlibrary_key', - 'POLITENESS_DELAY': 0, - 'MAX_DAILY_QUERIES': -1, - 'BASE_URL': 'https://openlibrary.org', - 'COVERS_URL': 'https://covers.openlibrary.org', + 'OpenLibrary': { + 'db_name': 'OpenLibrary', + 'classname': 'fedireads.connectors.openlibrary', }, } - -''' not implemented yet: - 'librarything': { - 'KEY_NAME': 'ltkey', - 'DB_KEY_FIELD': 'librarything_key', - 'POLITENESS_DELAY': 1, - 'MAX_DAILY_QUERIES': 1000, - 'BASE_URL': 'https://librarything.com', - }, - 'worldcat': { - 'KEY_NAME': 'ocn', - 'DB_KEY_FIELD': 'oclc_number', - 'POLITENESS_DELAY': 0, - 'MAX_DAILY_QUERIES': -1, - 'BASE_URL': 'https://worldcat.org', - }, -''' diff --git a/fedireads/migrations/0020_auto_20200327_2114.py b/fedireads/migrations/0020_auto_20200327_2201.py similarity index 89% rename from fedireads/migrations/0020_auto_20200327_2114.py rename to fedireads/migrations/0020_auto_20200327_2201.py index b21a20643..89f45fbed 100644 --- a/fedireads/migrations/0020_auto_20200327_2114.py +++ b/fedireads/migrations/0020_auto_20200327_2201.py @@ -1,4 +1,4 @@ -# Generated by Django 3.0.3 on 2020-03-27 21:14 +# Generated by Django 3.0.3 on 2020-03-27 22:01 from django.db import migrations, models import django.db.models.deletion @@ -17,12 +17,12 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created_date', models.DateTimeField(auto_now_add=True)), ('updated_date', models.DateTimeField(auto_now=True)), - ('name', models.CharField(max_length=255)), + ('name', models.CharField(max_length=255, unique=True)), ('api_key', models.CharField(max_length=255, null=True)), ('base_url', models.CharField(max_length=255)), ('covers_url', models.CharField(max_length=255)), ('search_url', models.CharField(max_length=255, null=True)), - ('key_name', models.CharField(max_length=255)), + ('key_name', models.CharField(max_length=255, unique=True)), ('politeness_delay', models.IntegerField(null=True)), ('max_query_count', models.IntegerField(null=True)), ('query_count', models.IntegerField(default=0)), diff --git a/fedireads/models/book.py b/fedireads/models/book.py index 42f8742a7..94e522b21 100644 --- a/fedireads/models/book.py +++ b/fedireads/models/book.py @@ -10,14 +10,14 @@ from fedireads.utils.models import FedireadsModel class Connector(FedireadsModel): ''' book data source connectors ''' - name = models.CharField(max_length=255) + name = models.CharField(max_length=255, unique=True) api_key = models.CharField(max_length=255, null=True) base_url = models.CharField(max_length=255) covers_url = models.CharField(max_length=255) search_url = models.CharField(max_length=255, null=True) - key_name = models.CharField(max_length=255) + key_name = models.CharField(max_length=255, unique=True) politeness_delay = models.IntegerField(null=True) #seconds max_query_count = models.IntegerField(null=True)