Use many small tasks instead of one big task

bookwyrm/importers/importer.py

@@ -102,39 +102,40 @@ class Importer:
 
     def start_import(self, job):
         """initalizes a csv import job"""
-        result = import_data.delay(self.service, job.id)
+        result = start_import_task.delay(self.service, job.id)
         job.task_id = result.id
         job.save()
 
 
 @app.task(queue="low_priority")
-def import_data(source, job_id):
+def start_import_task(source, job_id):
-    """does the actual lookup work in a celery task"""
+    """trigger the child tasks for each row"""
     job = ImportJob.objects.get(id=job_id)
-    try:
+    # these are sub-tasks so that one big task doesn't use up all the memory in celery
-        for item in job.items.all():
+    for item in job.items.values("id").all():
+        import_item_task.delay(source, item.id)
+
+
+@app.task(queue="low_priority")
+def import_item_task(source, item_id):
+    """resolve a row into a book"""
+    item = models.ImportItem.objets.get(id=item_id)
     try:
         item.resolve()
     except Exception as err:  # pylint: disable=broad-except
         logger.exception(err)
         item.fail_reason = _("Error loading book")
         item.save()
-                continue
+        return
-
-            if item.book or item.book_guess:
-                item.save()
 
     if item.book:
+        job = item.job
         # shelves book and handles reviews
-                handle_imported_book(
+        handle_imported_book(source, job.user, item, job.include_reviews, job.privacy)
-                    source, job.user, item, job.include_reviews, job.privacy
-                )
     else:
         item.fail_reason = _("Could not find a match for book")
+
     item.save()
-    finally:
-        job.complete = True
-        job.save()
 
 
 def handle_imported_book(source, user, item, include_reviews, privacy):