move loading editions to task

fedireads/books_manager.py

@@ -2,6 +2,7 @@
 import importlib
 
 from fedireads import models
+from fedireads.tasks import app
 
 
 def get_or_create_book(key):
@@ -15,7 +16,18 @@ def get_or_create_book(key):
         pass
 
     connector = get_connector()
-    return connector.get_or_create_book(key)
+    book = connector.get_or_create_book(key)
+    load_more_data.delay(book.id)
+    return book
+
+
+@app.task
+def load_more_data(book_id):
+    ''' background the work of getting all 10,000 editions of LoTR '''
+    book = models.Book.objects.select_subclasses().get(id=book_id)
+    connector = get_connector(book)
+    connector.expand_book_data(book)
+
 
 
 def search(query):

fedireads/connectors/abstract_connector.py

@@ -43,6 +43,12 @@ class AbstractConnector(ABC):
         pass
 
 
+    @abstractmethod
+    def expand_book_data(self, book):
+        ''' get more info on a book '''
+        pass
+
+
     @abstractmethod
     def get_or_create_author(self, book_id):
         ''' request and format a book given an identifier '''

fedireads/connectors/openlibrary.py

@@ -76,7 +76,7 @@ class Connector(AbstractConnector):
         return self.update_from_data(book, data)
 
 
-    def update_from_data(self, book, data, work=None):
+    def update_from_data(self, book, data):
         ''' update a book from a json blob '''
         mappings = {
             'publish_date': ('published_date', get_date),
@@ -98,13 +98,11 @@ class Connector(AbstractConnector):
         book.save()
 
         # this book sure as heck better be an edition
-        if not work:
-            work = None
-            if data.get('works'):
-                key = data.get('works')[0]['key']
-                key = key.split('/')[-1]
-                work = self.get_or_create_book(key)
-        book.parent_work = work
+        if data.get('works'):
+            key = data.get('works')[0]['key']
+            key = key.split('/')[-1]
+            work = self.get_or_create_book(key)
+            book.parent_work = work
 
         if isinstance(book, models.Work):
             # load editions of a work
@@ -124,25 +122,48 @@ class Connector(AbstractConnector):
         return book
 
 
-    def get_editions_of_work(self, work):
+    def expand_book_data(self, book):
+        work = book
+        if isinstance(book, models.Edition):
+            work = book.parent_work
+        self.get_editions_of_work(work, default_only=False)
+
+
+    def get_editions_of_work(self, work, default_only=True):
         ''' get all editions of a work '''
         response = requests.get(
             '%s/works/%s/editions.json' % (self.url, work.openlibrary_key))
         edition_data = response.json()
-        for data in edition_data.get('entries', []):
+
+        options = edition_data.get('entries', [])
+        if default_only:
+            options = [e for e in options if e.get('cover')] or options
+            options = [e for e in options if \
+                '/languages/eng' in str(e.get('languages'))] or options
+            formats = ['paperback', 'hardcover', 'mass market paperback']
+            options = [e for e in options if \
+                str(e.get('physical_format')).lower() in formats] or options
+            options = sorted(
+                options,
+                key=lambda e: e.get('publish_date', '3000')
+            )
+
+            if not options:
+                return
+            options = options[:1]
+
+        for data in options:
             try:
                 olkey = data['key'].split('/')[-1]
             except KeyError:
                 # bad data I guess?
-                continue
+                return
 
             try:
                 models.Edition.objects.get(openlibrary_key=olkey)
-                continue
             except ObjectDoesNotExist:
                 book = models.Edition.objects.create(openlibrary_key=olkey)
-                self.update_from_data(book, data, work=work)
-        set_default_edition(work)
+                self.update_from_data(book, data)
 
 
     def get_or_create_author(self, olkey):

fr_celery/celery.py

@@ -20,4 +20,5 @@ app.config_from_object('django.conf:settings', namespace='CELERY')
 app.autodiscover_tasks()
 app.autodiscover_tasks(['fedireads'], related_name='incoming')
 app.autodiscover_tasks(['fedireads'], related_name='broadcast')
+app.autodiscover_tasks(['fedireads'], related_name='books_manager')