mirrors/bookwyrm
1
0
Fork 1
mirror of https://github.com/bookwyrm-social/bookwyrm.git synced 2024-11-26 03:21:05 +00:00
Code Issues Projects Releases Wiki Activity

Fixed Import limit for retries

Browse source
This commit is contained in:
Giebisch 2023-01-17 00:59:51 +01:00
parent d0adb370cd
commit 5faf7f5e87
2 changed files with 29 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
bookwyrm/importers/importer.py
View file

@ -51,20 +51,8 @@ class Importer:
source=self.service,
)
site_settings = SiteSettings.objects.get()
import_size_limit = site_settings.import_size_limit
import_limit_reset = site_settings.import_limit_reset
enforce_limit = import_size_limit and import_limit_reset
if enforce_limit:
time_range = timezone.now() - timedelta(days=import_limit_reset)
import_jobs = ImportJob.objects.filter(
user=user, created_date__gte=time_range
)
# pylint: disable=consider-using-generator
imported_books = sum([job.successful_item_count for job in import_jobs])
allowed_imports = import_size_limit - imported_books
if allowed_imports <= 0:
enforce_limit, allowed_imports = self.get_import_limit(user)
if enforce_limit and allowed_imports <= 0:
job.complete_job()
return job
for index, entry in rows:
@ -119,6 +107,24 @@ class Importer:
"""use the dataclass to create the formatted row of data"""
return {k: entry.get(v) for k, v in mappings.items()}
def get_import_limit(self, user): # pylint: disable=no-self-use
"""check if import limit is set and return how many imports are left"""
site_settings = SiteSettings.objects.get()
import_size_limit = site_settings.import_size_limit
import_limit_reset = site_settings.import_limit_reset
enforce_limit = import_size_limit and import_limit_reset
allowed_imports = 0
if enforce_limit:
time_range = timezone.now() - timedelta(days=import_limit_reset)
import_jobs = ImportJob.objects.filter(
user=user, created_date__gte=time_range
)
# pylint: disable=consider-using-generator
imported_books = sum([job.successful_item_count for job in import_jobs])
allowed_imports = import_size_limit - imported_books
return enforce_limit, allowed_imports
def create_retry_job(self, user, original_job, items):
"""retry items that didn't import"""
job = ImportJob.objects.create(
@ -130,7 +136,13 @@ class Importer:
mappings=original_job.mappings,
retry=True,
)
for item in items:
enforce_limit, allowed_imports = self.get_import_limit(user)
if enforce_limit and allowed_imports <= 0:
job.complete_job()
return job
for index, item in enumerate(items):
if enforce_limit and index >= allowed_imports:
break
# this will re-normalize the raw data
self.create_item(job, item.index, item.data)
return job

1
bookwyrm/migrations/0174_merge_20230111_1523.py
View file

@ -6,7 +6,6 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0173_auto_20221228_1436"),
("bookwyrm", "0173_merge_20230102_1444"),
]