Split expand book data task into per-edition tasks

Loading every edition in one task takes ages, and produces a large task
that clogs up the queue. This will create more, smaller tasks that will
finish more quickly.

bookwyrm/connectors/connector_manager.py

@@ -152,6 +152,15 @@ def load_more_data(connector_id, book_id):
     connector.expand_book_data(book)
 
 
+@app.task(queue="low_priority")
+def create_edition_task(connector_id, work_id, data):
+    """separate task for each of the 10,000 editions of LoTR"""
+    connector_info = models.Connector.objects.get(id=connector_id)
+    connector = load_connector(connector_info)
+    work = models.Work.objects.select_subclasses().get(id=work_id)
+    connector.create_edition_from_data(work, data)
+
+
 def load_connector(connector_info):
     """instantiate the connector class"""
     connector = importlib.import_module(

bookwyrm/connectors/inventaire.py

@@ -5,7 +5,7 @@ from bookwyrm import models
 from bookwyrm.book_search import SearchResult
 from .abstract_connector import AbstractConnector, Mapping
 from .abstract_connector import get_data
-from .connector_manager import ConnectorException
+from .connector_manager import ConnectorException, create_edition_task
 
 
 class Connector(AbstractConnector):
@@ -156,12 +156,16 @@ class Connector(AbstractConnector):
 
         for edition_uri in edition_options.get("uris"):
             remote_id = self.get_remote_id(edition_uri)
+            create_edition_task.delay(self.connector.id, work.id, remote_id)
+
+    def create_edition_from_data(self, work, edition_data, instance=None):
+        """pass in the url as data and then call the version in abstract connector"""
         try:
-                data = self.get_book_data(remote_id)
+            data = self.get_book_data(edition_data)
         except ConnectorException:
             # who, indeed, knows
-                continue
+            return
-            self.create_edition_from_data(work, data)
+        super().create_edition_from_data(work, data, instance=instance)
 
     def get_cover_url(self, cover_blob, *_):
         """format the relative cover url into an absolute one:

bookwyrm/connectors/openlibrary.py

@@ -5,7 +5,7 @@ from bookwyrm import models
 from bookwyrm.book_search import SearchResult
 from .abstract_connector import AbstractConnector, Mapping
 from .abstract_connector import get_data, infer_physical_format, unique_physical_format
-from .connector_manager import ConnectorException
+from .connector_manager import ConnectorException, create_edition_task
 from .openlibrary_languages import languages
 
 
@@ -204,7 +204,7 @@ class Connector(AbstractConnector):
             # does this edition have ANY interesting data?
             if ignore_edition(edition_data):
                 continue
-            self.create_edition_from_data(work, edition_data)
+            create_edition_task.delay(self.connector.id, work.id, edition_data)
 
 
 def ignore_edition(edition_data):