Check if import is stopped and create import status field

This commit is contained in:
Mouse Reeve 2022-11-05 13:29:08 -07:00
parent 3c2f2c10bf
commit 24e5ea3295
3 changed files with 55 additions and 2 deletions

View file

@ -132,13 +132,19 @@ def start_import_task(job_id):
job = ImportJob.objects.get(id=job_id) job = ImportJob.objects.get(id=job_id)
# these are sub-tasks so that one big task doesn't use up all the memory in celery # these are sub-tasks so that one big task doesn't use up all the memory in celery
for item in job.items.values_list("id", flat=True).all(): for item in job.items.values_list("id", flat=True).all():
import_item_task.delay(item) task = import_item_task.delay(item)
item.task_id = task.id
item.save()
@app.task(queue="low_priority") @app.task(queue="low_priority")
def import_item_task(item_id): def import_item_task(item_id):
"""resolve a row into a book""" """resolve a row into a book"""
item = models.ImportItem.objects.get(id=item_id) item = models.ImportItem.objects.get(id=item_id)
# make sure the job has not been stopped
if item.job.complete:
return
try: try:
item.resolve() item.resolve()
except Exception as err: # pylint: disable=broad-except except Exception as err: # pylint: disable=broad-except

View file

@ -0,0 +1,33 @@
# Generated by Django 3.2.15 on 2022-11-05 20:30
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0159_auto_20220924_0634"),
]
operations = [
migrations.AddField(
model_name="importitem",
name="task_id",
field=models.CharField(blank=True, max_length=200, null=True),
),
migrations.AddField(
model_name="importjob",
name="status",
field=models.CharField(
choices=[
("pending", "Pending"),
("active", "Active"),
("complete", "Complete"),
("stopped", "Stopped"),
],
default="pending",
max_length=50,
null=True,
),
),
]

View file

@ -5,6 +5,7 @@ import dateutil.parser
from django.db import models from django.db import models
from django.utils import timezone from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from bookwyrm.connectors import connector_manager from bookwyrm.connectors import connector_manager
from bookwyrm.models import ReadThrough, User, Book, Edition from bookwyrm.models import ReadThrough, User, Book, Edition
@ -31,6 +32,14 @@ def construct_search_term(title, author):
return " ".join([title, author]) return " ".join([title, author])
ImportStatuses = [
("pending", _("Pending")),
("active", _("Active")),
("complete", _("Complete")),
("stopped", _("Stopped")),
]
class ImportJob(models.Model): class ImportJob(models.Model):
"""entry for a specific request for book data import""" """entry for a specific request for book data import"""
@ -39,11 +48,15 @@ class ImportJob(models.Model):
updated_date = models.DateTimeField(default=timezone.now) updated_date = models.DateTimeField(default=timezone.now)
include_reviews = models.BooleanField(default=True) include_reviews = models.BooleanField(default=True)
mappings = models.JSONField() mappings = models.JSONField()
complete = models.BooleanField(default=False)
source = models.CharField(max_length=100) source = models.CharField(max_length=100)
privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels) privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels)
retry = models.BooleanField(default=False) retry = models.BooleanField(default=False)
complete = models.BooleanField(default=False)
status = models.CharField(
max_length=50, choices=ImportStatuses, default="pending", null=True
)
@property @property
def pending_items(self): def pending_items(self):
"""items that haven't been processed yet""" """items that haven't been processed yet"""
@ -95,6 +108,7 @@ class ImportItem(models.Model):
linked_review = models.ForeignKey( linked_review = models.ForeignKey(
"Review", on_delete=models.SET_NULL, null=True, blank=True "Review", on_delete=models.SET_NULL, null=True, blank=True
) )
task_id = models.CharField(max_length=200, null=True, blank=True)
def update_job(self): def update_job(self):
"""let the job know when the items get work done""" """let the job know when the items get work done"""