Makes expanding book data fully part of the connector

bookwyrm/books_manager.py

@@ -1,12 +1,10 @@
 ''' select and call a connector for whatever book task needs doing '''
-import importlib
 from urllib.parse import urlparse
 
 from requests import HTTPError
 
 from bookwyrm import models
-from bookwyrm.connectors import ConnectorException
+from bookwyrm.connectors import ConnectorException, load_connector
-from bookwyrm.tasks import app
 
 
 def get_edition(book_id):
@@ -40,14 +38,6 @@ def get_or_create_connector(remote_id):
     return load_connector(connector_info)
 
 
-@app.task
-def load_more_data(book_id):
-    ''' background the work of getting all 10,000 editions of LoTR '''
-    book = models.Book.objects.select_subclasses().get(id=book_id)
-    connector = load_connector(book.connector)
-    connector.expand_book_data(book)
-
-
 def search(query, min_confidence=0.1):
     ''' find books based on arbitary keywords '''
     results = []
@@ -90,11 +80,3 @@ def get_connectors():
     ''' load all connectors '''
     for info in models.Connector.objects.order_by('priority').all():
         yield load_connector(info)
-
-
-def load_connector(connector_info):
-    ''' instantiate the connector class '''
-    connector = importlib.import_module(
-        'bookwyrm.connectors.%s' % connector_info.connector_file
-    )
-    return connector.Connector(connector_info.identifier)

bookwyrm/connectors/__init__.py

@@ -1,4 +1,4 @@
 ''' bring connectors into the namespace '''
 from .settings import CONNECTORS
-from .abstract_connector import ConnectorException
+from .abstract_connector import ConnectorException, load_connector
 from .abstract_connector import get_data, get_image

bookwyrm/connectors/abstract_connector.py

@@ -1,6 +1,7 @@
 ''' functionality outline for a book data connector '''
 from abc import ABC, abstractmethod
 from dataclasses import asdict, dataclass
+import importlib
 import logging
 from urllib3.exceptions import RequestError
 
@@ -10,6 +11,7 @@ from requests import HTTPError
 from requests.exceptions import SSLError
 
 from bookwyrm import activitypub, models, settings
+from bookwyrm.tasks import app
 
 
 logger = logging.getLogger(__name__)
@@ -90,7 +92,6 @@ class AbstractConnector(AbstractMinimalConnector):
         return True
 
 
-    @transaction.atomic
     def get_or_create_book(self, remote_id):
         ''' translate arbitrary json into an Activitypub dataclass '''
         # first, check if we have the origin_id saved
@@ -123,13 +124,17 @@ class AbstractConnector(AbstractMinimalConnector):
         if not work_data or not edition_data:
             raise ConnectorException('Unable to load book data: %s' % remote_id)
 
-        # create activitypub object
+        with transaction.atomic():
-        work_activity = activitypub.Work(**work_data)
+            # create activitypub object
-        # this will dedupe automatically
+            work_activity = activitypub.Work(**work_data)
-        work = work_activity.to_model(models.Work)
+            # this will dedupe automatically
-        for author in self.get_authors_from_data(data):
+            work = work_activity.to_model(models.Work)
-            work.authors.add(author)
+            for author in self.get_authors_from_data(data):
-        return self.create_edition_from_data(work, edition_data)
+                work.authors.add(author)
+
+            edition = self.create_edition_from_data(work, edition_data)
+        load_more_data.delay(self.connector.id, work.id)
+        return edition
 
 
     def create_edition_from_data(self, work, edition_data):
@@ -187,6 +192,23 @@ class AbstractConnector(AbstractMinimalConnector):
         ''' get more info on a book '''
 
 
+@app.task
+def load_more_data(connector_id, book_id):
+    ''' background the work of getting all 10,000 editions of LoTR '''
+    connector_info = models.Connector.objects.get(id=connector_id)
+    connector = load_connector(connector_info)
+    book = models.Book.objects.select_subclasses().get(id=book_id)
+    connector.expand_book_data(book)
+
+
+def load_connector(connector_info):
+    ''' instantiate the connector class '''
+    connector = importlib.import_module(
+        'bookwyrm.connectors.%s' % connector_info.connector_file
+    )
+    return connector.Connector(connector_info.identifier)
+
+
 def dict_from_mappings(data, mappings):
     ''' create a dict in Activitypub format, using mappings supplies by
     the subclass '''

bookwyrm/view_actions.py

@@ -212,8 +212,6 @@ def resolve_book(request):
     remote_id = request.POST.get('remote_id')
     connector = books_manager.get_or_create_connector(remote_id)
     book = connector.get_or_create_book(remote_id)
-    if book.connector:
-        books_manager.load_more_data.delay(book.id)
 
     return redirect('/book/%d' % book.id)
 

celerywyrm/celery.py

@@ -20,8 +20,9 @@ app.config_from_object('django.conf:settings', namespace='CELERY')
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks()
 app.autodiscover_tasks(['bookwyrm'], related_name='activitypub.base_activity')
-app.autodiscover_tasks(['bookwyrm'], related_name='books_manager')
 app.autodiscover_tasks(['bookwyrm'], related_name='broadcast')
+app.autodiscover_tasks(
+    ['bookwyrm'], related_name='connectors.abstract_connector')
 app.autodiscover_tasks(['bookwyrm'], related_name='emailing')
 app.autodiscover_tasks(['bookwyrm'], related_name='goodreads_import')
 app.autodiscover_tasks(['bookwyrm'], related_name='incoming')