- remove test export files
- check in emblackened files
This commit is contained in:
Hugh Rundle 2024-01-14 12:19:59 +11:00
parent cbd08127ef
commit 62cc6c298f
No known key found for this signature in database
GPG key ID: A7E35779918253F9
6 changed files with 108 additions and 56 deletions

View file

@ -9,45 +9,84 @@ import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('bookwyrm', '0191_merge_20240102_0326'),
("bookwyrm", "0191_merge_20240102_0326"),
]
operations = [
migrations.AddField(
model_name='bookwyrmexportjob',
name='export_json',
field=models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, null=True),
model_name="bookwyrmexportjob",
name="export_json",
field=models.JSONField(
encoder=django.core.serializers.json.DjangoJSONEncoder, null=True
),
),
migrations.AddField(
model_name='bookwyrmexportjob',
name='json_completed',
model_name="bookwyrmexportjob",
name="json_completed",
field=models.BooleanField(default=False),
),
migrations.AlterField(
model_name='bookwyrmexportjob',
name='export_data',
field=models.FileField(null=True, storage=bookwyrm.storage_backends.ExportsFileStorage, upload_to=''),
model_name="bookwyrmexportjob",
name="export_data",
field=models.FileField(
null=True,
storage=bookwyrm.storage_backends.ExportsFileStorage,
upload_to="",
),
),
migrations.CreateModel(
name='AddFileToTar',
name="AddFileToTar",
fields=[
('childjob_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='bookwyrm.childjob')),
('parent_export_job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='child_edition_export_jobs', to='bookwyrm.bookwyrmexportjob')),
(
"childjob_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="bookwyrm.childjob",
),
),
(
"parent_export_job",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="child_edition_export_jobs",
to="bookwyrm.bookwyrmexportjob",
),
),
],
options={
'abstract': False,
"abstract": False,
},
bases=('bookwyrm.childjob',),
bases=("bookwyrm.childjob",),
),
migrations.CreateModel(
name='AddBookToUserExportJob',
name="AddBookToUserExportJob",
fields=[
('childjob_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='bookwyrm.childjob')),
('edition', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='bookwyrm.edition')),
(
"childjob_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="bookwyrm.childjob",
),
),
(
"edition",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="bookwyrm.edition",
),
),
],
options={
'abstract': False,
"abstract": False,
},
bases=('bookwyrm.childjob',),
bases=("bookwyrm.childjob",),
),
]

View file

@ -24,6 +24,7 @@ from bookwyrm.utils.tar import BookwyrmTarFile
logger = logging.getLogger(__name__)
class BookwyrmExportJob(ParentJob):
"""entry for a specific request to export a bookwyrm user"""
@ -32,11 +33,12 @@ class BookwyrmExportJob(ParentJob):
else:
storage = storage_backends.ExportsFileStorage
export_data = FileField(null=True, storage=storage) # use custom storage backend here
export_data = FileField(
null=True, storage=storage
) # use custom storage backend here
export_json = JSONField(null=True, encoder=DjangoJSONEncoder)
json_completed = BooleanField(default=False)
def start_job(self):
"""Start the job"""
@ -44,7 +46,6 @@ class BookwyrmExportJob(ParentJob):
self.task_id = task.id
self.save(update_fields=["task_id"])
def notify_child_job_complete(self):
"""let the job know when the items get work done"""
@ -63,8 +64,7 @@ class BookwyrmExportJob(ParentJob):
# add json file to tarfile
tar_job = AddFileToTar.objects.create(
parent_job=self,
parent_export_job=self
parent_job=self, parent_export_job=self
)
tar_job.start_job()
@ -116,7 +116,9 @@ class AddBookToUserExportJob(ChildJob):
# ListItems include "notes" and "approved" so we need them
# even though we know it's this book
book["lists"] = []
list_items = ListItem.objects.filter(book=self.edition, user=self.parent_job.user).distinct()
list_items = ListItem.objects.filter(
book=self.edition, user=self.parent_job.user
).distinct()
for item in list_items:
list_info = item.book_list.to_activity()
@ -133,16 +135,18 @@ class AddBookToUserExportJob(ChildJob):
for status in ["comments", "quotations", "reviews"]:
book[status] = []
comments = Comment.objects.filter(user=self.parent_job.user, book=self.edition).all()
comments = Comment.objects.filter(
user=self.parent_job.user, book=self.edition
).all()
for status in comments:
obj = status.to_activity()
obj["progress"] = status.progress
obj["progress_mode"] = status.progress_mode
book["comments"].append(obj)
quotes = Quotation.objects.filter(user=self.parent_job.user, book=self.edition).all()
quotes = Quotation.objects.filter(
user=self.parent_job.user, book=self.edition
).all()
for status in quotes:
obj = status.to_activity()
obj["position"] = status.position
@ -150,15 +154,18 @@ class AddBookToUserExportJob(ChildJob):
obj["position_mode"] = status.position_mode
book["quotations"].append(obj)
reviews = Review.objects.filter(user=self.parent_job.user, book=self.edition).all()
reviews = Review.objects.filter(
user=self.parent_job.user, book=self.edition
).all()
for status in reviews:
obj = status.to_activity()
book["reviews"].append(obj)
# readthroughs can't be serialized to activity
book_readthroughs = (
ReadThrough.objects.filter(user=self.parent_job.user, book=self.edition).distinct().values()
ReadThrough.objects.filter(user=self.parent_job.user, book=self.edition)
.distinct()
.values()
)
book["readthroughs"] = list(book_readthroughs)
@ -167,7 +174,9 @@ class AddBookToUserExportJob(ChildJob):
self.complete_job()
except Exception as err: # pylint: disable=broad-except
logger.exception("AddBookToUserExportJob %s Failed with error: %s", self.id, err)
logger.exception(
"AddBookToUserExportJob %s Failed with error: %s", self.id, err
)
self.set_status("failed")
@ -178,7 +187,6 @@ class AddFileToTar(ChildJob):
BookwyrmExportJob, on_delete=CASCADE, related_name="child_edition_export_jobs"
) # TODO: do we actually need this? Does self.parent_job.export_data work?
def start_job(self):
"""Start the job"""
@ -188,7 +196,7 @@ class AddFileToTar(ChildJob):
# but Hugh couldn't make that work
try:
task_id=self.parent_export_job.task_id
task_id = self.parent_export_job.task_id
export_data = self.parent_export_job.export_data
export_json = self.parent_export_job.export_json
json_data = DjangoJSONEncoder().encode(export_json)
@ -198,27 +206,19 @@ class AddFileToTar(ChildJob):
if settings.USE_S3:
s3_job = S3Tar(
settings.AWS_STORAGE_BUCKET_NAME,
f"exports/{str(self.parent_export_job.task_id)}.tar.gz"
f"exports/{str(self.parent_export_job.task_id)}.tar.gz",
)
# TODO: either encrypt the file or we will need to get it to the user
# from this secure part of the bucket
export_data.save("archive.json", ContentFile(json_data.encode("utf-8")))
s3_job.add_file(
f"exports/{export_data.name}"
)
s3_job.add_file(
f"images/{user.avatar.name}",
folder="avatar"
)
s3_job.add_file(f"exports/{export_data.name}")
s3_job.add_file(f"images/{user.avatar.name}", folder="avatar")
for book in editions:
if getattr(book, "cover", False):
cover_name = f"images/{book.cover.name}"
s3_job.add_file(
cover_name,
folder="covers"
)
s3_job.add_file(cover_name, folder="covers")
s3_job.tar()
# delete export json as soon as it's tarred
@ -228,7 +228,7 @@ class AddFileToTar(ChildJob):
else:
# TODO: is the export_data file open to the world?
logger.info( "export file URL: %s",export_data.url)
logger.info("export file URL: %s", export_data.url)
export_data.open("wb")
with BookwyrmTarFile.open(mode="w:gz", fileobj=export_data) as tar:
@ -237,7 +237,9 @@ class AddFileToTar(ChildJob):
# Add avatar image if present
if getattr(user, "avatar", False):
tar.add_image(user.avatar, filename="avatar", directory=f"avatar/") # TODO: does this work?
tar.add_image(
user.avatar, filename="avatar", directory=f"avatar/"
) # TODO: does this work?
for book in editions:
if getattr(book, "cover", False):
@ -245,7 +247,6 @@ class AddFileToTar(ChildJob):
export_data.close()
self.complete_job()
except Exception as err: # pylint: disable=broad-except
@ -277,6 +278,7 @@ def start_export_task(**kwargs):
logger.exception("User Export Job %s Failed with error: %s", job.id, err)
job.set_status("failed")
@app.task(queue=IMPORTS, base=ParentTask)
def export_saved_lists_task(**kwargs):
"""add user saved lists to export JSON"""
@ -381,16 +383,23 @@ def trigger_books_jobs(**kwargs):
for edition in editions:
try:
edition_job = AddBookToUserExportJob.objects.create(edition=edition, parent_job=job)
edition_job = AddBookToUserExportJob.objects.create(
edition=edition, parent_job=job
)
edition_job.start_job()
except Exception as err: # pylint: disable=broad-except
logger.exception("AddBookToUserExportJob %s Failed with error: %s", edition_job.id, err)
logger.exception(
"AddBookToUserExportJob %s Failed with error: %s",
edition_job.id,
err,
)
edition_job.set_status("failed")
except Exception as err: # pylint: disable=broad-except
logger.exception("trigger_books_jobs %s Failed with error: %s", job.id, err)
job.set_status("failed")
def get_books_for_user(user):
"""Get all the books and editions related to a user"""

View file

@ -442,4 +442,6 @@ if HTTP_X_FORWARDED_PROTO:
# Do not change this setting unless you already have an existing
# user with the same username - in which case you should change it!
INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"
DATA_UPLOAD_MAX_MEMORY_SIZE = (1024**2 * 20) # 20MB TEMPORARY FIX WHILST WORKING ON THIS
DATA_UPLOAD_MAX_MEMORY_SIZE = (
1024**2 * 20
) # 20MB TEMPORARY FIX WHILST WORKING ON THIS

View file

@ -63,12 +63,14 @@ class AzureImagesStorage(AzureStorage): # pylint: disable=abstract-method
location = "images"
overwrite_files = False
class ExportsFileStorage(FileSystemStorage): # pylint: disable=abstract-method
"""Storage class for exports contents with local files"""
location = "exports"
overwrite_files = False
class ExportsS3Storage(S3Boto3Storage): # pylint: disable=abstract-method
"""Storage class for exports contents with S3"""