- remove test export files
- check in emblackened files
This commit is contained in:
Hugh Rundle 2024-01-14 12:19:59 +11:00
parent cbd08127ef
commit 62cc6c298f
No known key found for this signature in database
GPG key ID: A7E35779918253F9
6 changed files with 108 additions and 56 deletions

View file

@ -9,45 +9,84 @@ import django.db.models.deletion
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('bookwyrm', '0191_merge_20240102_0326'), ("bookwyrm", "0191_merge_20240102_0326"),
] ]
operations = [ operations = [
migrations.AddField( migrations.AddField(
model_name='bookwyrmexportjob', model_name="bookwyrmexportjob",
name='export_json', name="export_json",
field=models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, null=True), field=models.JSONField(
encoder=django.core.serializers.json.DjangoJSONEncoder, null=True
),
), ),
migrations.AddField( migrations.AddField(
model_name='bookwyrmexportjob', model_name="bookwyrmexportjob",
name='json_completed', name="json_completed",
field=models.BooleanField(default=False), field=models.BooleanField(default=False),
), ),
migrations.AlterField( migrations.AlterField(
model_name='bookwyrmexportjob', model_name="bookwyrmexportjob",
name='export_data', name="export_data",
field=models.FileField(null=True, storage=bookwyrm.storage_backends.ExportsFileStorage, upload_to=''), field=models.FileField(
null=True,
storage=bookwyrm.storage_backends.ExportsFileStorage,
upload_to="",
),
), ),
migrations.CreateModel( migrations.CreateModel(
name='AddFileToTar', name="AddFileToTar",
fields=[ fields=[
('childjob_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='bookwyrm.childjob')), (
('parent_export_job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='child_edition_export_jobs', to='bookwyrm.bookwyrmexportjob')), "childjob_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="bookwyrm.childjob",
),
),
(
"parent_export_job",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="child_edition_export_jobs",
to="bookwyrm.bookwyrmexportjob",
),
),
], ],
options={ options={
'abstract': False, "abstract": False,
}, },
bases=('bookwyrm.childjob',), bases=("bookwyrm.childjob",),
), ),
migrations.CreateModel( migrations.CreateModel(
name='AddBookToUserExportJob', name="AddBookToUserExportJob",
fields=[ fields=[
('childjob_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='bookwyrm.childjob')), (
('edition', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='bookwyrm.edition')), "childjob_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="bookwyrm.childjob",
),
),
(
"edition",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="bookwyrm.edition",
),
),
], ],
options={ options={
'abstract': False, "abstract": False,
}, },
bases=('bookwyrm.childjob',), bases=("bookwyrm.childjob",),
), ),
] ]

View file

@ -24,6 +24,7 @@ from bookwyrm.utils.tar import BookwyrmTarFile
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class BookwyrmExportJob(ParentJob): class BookwyrmExportJob(ParentJob):
"""entry for a specific request to export a bookwyrm user""" """entry for a specific request to export a bookwyrm user"""
@ -32,11 +33,12 @@ class BookwyrmExportJob(ParentJob):
else: else:
storage = storage_backends.ExportsFileStorage storage = storage_backends.ExportsFileStorage
export_data = FileField(null=True, storage=storage) # use custom storage backend here export_data = FileField(
null=True, storage=storage
) # use custom storage backend here
export_json = JSONField(null=True, encoder=DjangoJSONEncoder) export_json = JSONField(null=True, encoder=DjangoJSONEncoder)
json_completed = BooleanField(default=False) json_completed = BooleanField(default=False)
def start_job(self): def start_job(self):
"""Start the job""" """Start the job"""
@ -44,7 +46,6 @@ class BookwyrmExportJob(ParentJob):
self.task_id = task.id self.task_id = task.id
self.save(update_fields=["task_id"]) self.save(update_fields=["task_id"])
def notify_child_job_complete(self): def notify_child_job_complete(self):
"""let the job know when the items get work done""" """let the job know when the items get work done"""
@ -63,8 +64,7 @@ class BookwyrmExportJob(ParentJob):
# add json file to tarfile # add json file to tarfile
tar_job = AddFileToTar.objects.create( tar_job = AddFileToTar.objects.create(
parent_job=self, parent_job=self, parent_export_job=self
parent_export_job=self
) )
tar_job.start_job() tar_job.start_job()
@ -116,7 +116,9 @@ class AddBookToUserExportJob(ChildJob):
# ListItems include "notes" and "approved" so we need them # ListItems include "notes" and "approved" so we need them
# even though we know it's this book # even though we know it's this book
book["lists"] = [] book["lists"] = []
list_items = ListItem.objects.filter(book=self.edition, user=self.parent_job.user).distinct() list_items = ListItem.objects.filter(
book=self.edition, user=self.parent_job.user
).distinct()
for item in list_items: for item in list_items:
list_info = item.book_list.to_activity() list_info = item.book_list.to_activity()
@ -133,16 +135,18 @@ class AddBookToUserExportJob(ChildJob):
for status in ["comments", "quotations", "reviews"]: for status in ["comments", "quotations", "reviews"]:
book[status] = [] book[status] = []
comments = Comment.objects.filter(
comments = Comment.objects.filter(user=self.parent_job.user, book=self.edition).all() user=self.parent_job.user, book=self.edition
).all()
for status in comments: for status in comments:
obj = status.to_activity() obj = status.to_activity()
obj["progress"] = status.progress obj["progress"] = status.progress
obj["progress_mode"] = status.progress_mode obj["progress_mode"] = status.progress_mode
book["comments"].append(obj) book["comments"].append(obj)
quotes = Quotation.objects.filter(
quotes = Quotation.objects.filter(user=self.parent_job.user, book=self.edition).all() user=self.parent_job.user, book=self.edition
).all()
for status in quotes: for status in quotes:
obj = status.to_activity() obj = status.to_activity()
obj["position"] = status.position obj["position"] = status.position
@ -150,15 +154,18 @@ class AddBookToUserExportJob(ChildJob):
obj["position_mode"] = status.position_mode obj["position_mode"] = status.position_mode
book["quotations"].append(obj) book["quotations"].append(obj)
reviews = Review.objects.filter(
reviews = Review.objects.filter(user=self.parent_job.user, book=self.edition).all() user=self.parent_job.user, book=self.edition
).all()
for status in reviews: for status in reviews:
obj = status.to_activity() obj = status.to_activity()
book["reviews"].append(obj) book["reviews"].append(obj)
# readthroughs can't be serialized to activity # readthroughs can't be serialized to activity
book_readthroughs = ( book_readthroughs = (
ReadThrough.objects.filter(user=self.parent_job.user, book=self.edition).distinct().values() ReadThrough.objects.filter(user=self.parent_job.user, book=self.edition)
.distinct()
.values()
) )
book["readthroughs"] = list(book_readthroughs) book["readthroughs"] = list(book_readthroughs)
@ -167,7 +174,9 @@ class AddBookToUserExportJob(ChildJob):
self.complete_job() self.complete_job()
except Exception as err: # pylint: disable=broad-except except Exception as err: # pylint: disable=broad-except
logger.exception("AddBookToUserExportJob %s Failed with error: %s", self.id, err) logger.exception(
"AddBookToUserExportJob %s Failed with error: %s", self.id, err
)
self.set_status("failed") self.set_status("failed")
@ -178,7 +187,6 @@ class AddFileToTar(ChildJob):
BookwyrmExportJob, on_delete=CASCADE, related_name="child_edition_export_jobs" BookwyrmExportJob, on_delete=CASCADE, related_name="child_edition_export_jobs"
) # TODO: do we actually need this? Does self.parent_job.export_data work? ) # TODO: do we actually need this? Does self.parent_job.export_data work?
def start_job(self): def start_job(self):
"""Start the job""" """Start the job"""
@ -198,27 +206,19 @@ class AddFileToTar(ChildJob):
if settings.USE_S3: if settings.USE_S3:
s3_job = S3Tar( s3_job = S3Tar(
settings.AWS_STORAGE_BUCKET_NAME, settings.AWS_STORAGE_BUCKET_NAME,
f"exports/{str(self.parent_export_job.task_id)}.tar.gz" f"exports/{str(self.parent_export_job.task_id)}.tar.gz",
) )
# TODO: either encrypt the file or we will need to get it to the user # TODO: either encrypt the file or we will need to get it to the user
# from this secure part of the bucket # from this secure part of the bucket
export_data.save("archive.json", ContentFile(json_data.encode("utf-8"))) export_data.save("archive.json", ContentFile(json_data.encode("utf-8")))
s3_job.add_file( s3_job.add_file(f"exports/{export_data.name}")
f"exports/{export_data.name}" s3_job.add_file(f"images/{user.avatar.name}", folder="avatar")
)
s3_job.add_file(
f"images/{user.avatar.name}",
folder="avatar"
)
for book in editions: for book in editions:
if getattr(book, "cover", False): if getattr(book, "cover", False):
cover_name = f"images/{book.cover.name}" cover_name = f"images/{book.cover.name}"
s3_job.add_file( s3_job.add_file(cover_name, folder="covers")
cover_name,
folder="covers"
)
s3_job.tar() s3_job.tar()
# delete export json as soon as it's tarred # delete export json as soon as it's tarred
@ -237,7 +237,9 @@ class AddFileToTar(ChildJob):
# Add avatar image if present # Add avatar image if present
if getattr(user, "avatar", False): if getattr(user, "avatar", False):
tar.add_image(user.avatar, filename="avatar", directory=f"avatar/") # TODO: does this work? tar.add_image(
user.avatar, filename="avatar", directory=f"avatar/"
) # TODO: does this work?
for book in editions: for book in editions:
if getattr(book, "cover", False): if getattr(book, "cover", False):
@ -245,7 +247,6 @@ class AddFileToTar(ChildJob):
export_data.close() export_data.close()
self.complete_job() self.complete_job()
except Exception as err: # pylint: disable=broad-except except Exception as err: # pylint: disable=broad-except
@ -277,6 +278,7 @@ def start_export_task(**kwargs):
logger.exception("User Export Job %s Failed with error: %s", job.id, err) logger.exception("User Export Job %s Failed with error: %s", job.id, err)
job.set_status("failed") job.set_status("failed")
@app.task(queue=IMPORTS, base=ParentTask) @app.task(queue=IMPORTS, base=ParentTask)
def export_saved_lists_task(**kwargs): def export_saved_lists_task(**kwargs):
"""add user saved lists to export JSON""" """add user saved lists to export JSON"""
@ -381,16 +383,23 @@ def trigger_books_jobs(**kwargs):
for edition in editions: for edition in editions:
try: try:
edition_job = AddBookToUserExportJob.objects.create(edition=edition, parent_job=job) edition_job = AddBookToUserExportJob.objects.create(
edition=edition, parent_job=job
)
edition_job.start_job() edition_job.start_job()
except Exception as err: # pylint: disable=broad-except except Exception as err: # pylint: disable=broad-except
logger.exception("AddBookToUserExportJob %s Failed with error: %s", edition_job.id, err) logger.exception(
"AddBookToUserExportJob %s Failed with error: %s",
edition_job.id,
err,
)
edition_job.set_status("failed") edition_job.set_status("failed")
except Exception as err: # pylint: disable=broad-except except Exception as err: # pylint: disable=broad-except
logger.exception("trigger_books_jobs %s Failed with error: %s", job.id, err) logger.exception("trigger_books_jobs %s Failed with error: %s", job.id, err)
job.set_status("failed") job.set_status("failed")
def get_books_for_user(user): def get_books_for_user(user):
"""Get all the books and editions related to a user""" """Get all the books and editions related to a user"""

View file

@ -442,4 +442,6 @@ if HTTP_X_FORWARDED_PROTO:
# Do not change this setting unless you already have an existing # Do not change this setting unless you already have an existing
# user with the same username - in which case you should change it! # user with the same username - in which case you should change it!
INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"
DATA_UPLOAD_MAX_MEMORY_SIZE = (1024**2 * 20) # 20MB TEMPORARY FIX WHILST WORKING ON THIS DATA_UPLOAD_MAX_MEMORY_SIZE = (
1024**2 * 20
) # 20MB TEMPORARY FIX WHILST WORKING ON THIS

View file

@ -63,12 +63,14 @@ class AzureImagesStorage(AzureStorage): # pylint: disable=abstract-method
location = "images" location = "images"
overwrite_files = False overwrite_files = False
class ExportsFileStorage(FileSystemStorage): # pylint: disable=abstract-method class ExportsFileStorage(FileSystemStorage): # pylint: disable=abstract-method
"""Storage class for exports contents with local files""" """Storage class for exports contents with local files"""
location = "exports" location = "exports"
overwrite_files = False overwrite_files = False
class ExportsS3Storage(S3Boto3Storage): # pylint: disable=abstract-method class ExportsS3Storage(S3Boto3Storage): # pylint: disable=abstract-method
"""Storage class for exports contents with S3""" """Storage class for exports contents with S3"""