From e169565e006c44458db1a3bbaefb760f485f6c2a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 2 Jan 2021 07:45:45 -0800 Subject: [PATCH] Makes expanding book data fully part of the connector --- bookwyrm/books_manager.py | 20 +----------- bookwyrm/connectors/__init__.py | 2 +- bookwyrm/connectors/abstract_connector.py | 38 ++++++++++++++++++----- bookwyrm/view_actions.py | 2 -- celerywyrm/celery.py | 3 +- 5 files changed, 34 insertions(+), 31 deletions(-) diff --git a/bookwyrm/books_manager.py b/bookwyrm/books_manager.py index bc1fa7231..70386ba83 100644 --- a/bookwyrm/books_manager.py +++ b/bookwyrm/books_manager.py @@ -1,12 +1,10 @@ ''' select and call a connector for whatever book task needs doing ''' -import importlib from urllib.parse import urlparse from requests import HTTPError from bookwyrm import models -from bookwyrm.connectors import ConnectorException -from bookwyrm.tasks import app +from bookwyrm.connectors import ConnectorException, load_connector def get_edition(book_id): @@ -40,14 +38,6 @@ def get_or_create_connector(remote_id): return load_connector(connector_info) -@app.task -def load_more_data(book_id): - ''' background the work of getting all 10,000 editions of LoTR ''' - book = models.Book.objects.select_subclasses().get(id=book_id) - connector = load_connector(book.connector) - connector.expand_book_data(book) - - def search(query, min_confidence=0.1): ''' find books based on arbitary keywords ''' results = [] @@ -90,11 +80,3 @@ def get_connectors(): ''' load all connectors ''' for info in models.Connector.objects.order_by('priority').all(): yield load_connector(info) - - -def load_connector(connector_info): - ''' instantiate the connector class ''' - connector = importlib.import_module( - 'bookwyrm.connectors.%s' % connector_info.connector_file - ) - return connector.Connector(connector_info.identifier) diff --git a/bookwyrm/connectors/__init__.py b/bookwyrm/connectors/__init__.py index 4eb91de41..f6a558f36 100644 --- a/bookwyrm/connectors/__init__.py +++ b/bookwyrm/connectors/__init__.py @@ -1,4 +1,4 @@ ''' bring connectors into the namespace ''' from .settings import CONNECTORS -from .abstract_connector import ConnectorException +from .abstract_connector import ConnectorException, load_connector from .abstract_connector import get_data, get_image diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 3461e80f5..c1f5a3216 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -1,6 +1,7 @@ ''' functionality outline for a book data connector ''' from abc import ABC, abstractmethod from dataclasses import asdict, dataclass +import importlib import logging from urllib3.exceptions import RequestError @@ -10,6 +11,7 @@ from requests import HTTPError from requests.exceptions import SSLError from bookwyrm import activitypub, models, settings +from bookwyrm.tasks import app logger = logging.getLogger(__name__) @@ -90,7 +92,6 @@ class AbstractConnector(AbstractMinimalConnector): return True - @transaction.atomic def get_or_create_book(self, remote_id): ''' translate arbitrary json into an Activitypub dataclass ''' # first, check if we have the origin_id saved @@ -123,13 +124,17 @@ class AbstractConnector(AbstractMinimalConnector): if not work_data or not edition_data: raise ConnectorException('Unable to load book data: %s' % remote_id) - # create activitypub object - work_activity = activitypub.Work(**work_data) - # this will dedupe automatically - work = work_activity.to_model(models.Work) - for author in self.get_authors_from_data(data): - work.authors.add(author) - return self.create_edition_from_data(work, edition_data) + with transaction.atomic(): + # create activitypub object + work_activity = activitypub.Work(**work_data) + # this will dedupe automatically + work = work_activity.to_model(models.Work) + for author in self.get_authors_from_data(data): + work.authors.add(author) + + edition = self.create_edition_from_data(work, edition_data) + load_more_data.delay(self.connector.id, work.id) + return edition def create_edition_from_data(self, work, edition_data): @@ -187,6 +192,23 @@ class AbstractConnector(AbstractMinimalConnector): ''' get more info on a book ''' +@app.task +def load_more_data(connector_id, book_id): + ''' background the work of getting all 10,000 editions of LoTR ''' + connector_info = models.Connector.objects.get(id=connector_id) + connector = load_connector(connector_info) + book = models.Book.objects.select_subclasses().get(id=book_id) + connector.expand_book_data(book) + + +def load_connector(connector_info): + ''' instantiate the connector class ''' + connector = importlib.import_module( + 'bookwyrm.connectors.%s' % connector_info.connector_file + ) + return connector.Connector(connector_info.identifier) + + def dict_from_mappings(data, mappings): ''' create a dict in Activitypub format, using mappings supplies by the subclass ''' diff --git a/bookwyrm/view_actions.py b/bookwyrm/view_actions.py index 8a686babf..44e71e553 100644 --- a/bookwyrm/view_actions.py +++ b/bookwyrm/view_actions.py @@ -212,8 +212,6 @@ def resolve_book(request): remote_id = request.POST.get('remote_id') connector = books_manager.get_or_create_connector(remote_id) book = connector.get_or_create_book(remote_id) - if book.connector: - books_manager.load_more_data.delay(book.id) return redirect('/book/%d' % book.id) diff --git a/celerywyrm/celery.py b/celerywyrm/celery.py index efa081ee8..5a53dab5b 100644 --- a/celerywyrm/celery.py +++ b/celerywyrm/celery.py @@ -20,8 +20,9 @@ app.config_from_object('django.conf:settings', namespace='CELERY') # Load task modules from all registered Django app configs. app.autodiscover_tasks() app.autodiscover_tasks(['bookwyrm'], related_name='activitypub.base_activity') -app.autodiscover_tasks(['bookwyrm'], related_name='books_manager') app.autodiscover_tasks(['bookwyrm'], related_name='broadcast') +app.autodiscover_tasks( + ['bookwyrm'], related_name='connectors.abstract_connector') app.autodiscover_tasks(['bookwyrm'], related_name='emailing') app.autodiscover_tasks(['bookwyrm'], related_name='goodreads_import') app.autodiscover_tasks(['bookwyrm'], related_name='incoming')