Merge BookwyrmExportJob export_data field back into one with dynamic storage backend

bookwyrm/migrations/0198_alter_bookwyrmexportjob_export_data.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.25 on 2024-03-26 11:37
+
+import bookwyrm.models.bookwyrm_export_job
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0197_merge_20240324_0235"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="bookwyrmexportjob",
+            name="export_data",
+            field=models.FileField(
+                null=True,
+                storage=bookwyrm.models.bookwyrm_export_job.select_exports_storage,
+                upload_to="",
+            ),
+        ),
+    ]

bookwyrm/migrations/0198_export_job_separate_file_fields.py

@@ -1,28 +0,0 @@
-# Generated by Django 3.2.25 on 2024-03-24 11:20
-
-import bookwyrm.storage_backends
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("bookwyrm", "0197_merge_20240324_0235"),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name="bookwyrmexportjob",
-            old_name="export_data",
-            new_name="export_data_file",
-        ),
-        migrations.AddField(
-            model_name="bookwyrmexportjob",
-            name="export_data_s3",
-            field=models.FileField(
-                null=True,
-                storage=bookwyrm.storage_backends.ExportsS3Storage,
-                upload_to="",
-            ),
-        ),
-    ]

bookwyrm/models/bookwyrm_export_job.py

@@ -12,8 +12,9 @@ from django.db.models import Q
 from django.core.serializers.json import DjangoJSONEncoder
 from django.core.files.base import ContentFile
 from django.utils import timezone
+from django.utils.module_loading import import_string
 
-from bookwyrm import settings, storage_backends
+from bookwyrm import settings
 
 from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, List, ListItem
 from bookwyrm.models import Review, Comment, Quotation
@@ -34,33 +35,19 @@ class BookwyrmAwsSession(BotoSession):
         return super().client("s3", *args, **kwargs)
 
 
+def select_exports_storage():
+    """callable to allow for dependency on runtime configuration"""
+    cls = import_string(settings.EXPORTS_STORAGE)
+    return cls()
+
+
 class BookwyrmExportJob(ParentJob):
     """entry for a specific request to export a bookwyrm user"""
 
-    # Only one of these fields is used, dependent on the configuration.
+    export_data = FileField(null=True, storage=select_exports_storage)
-    export_data_file = FileField(null=True, storage=storage_backends.ExportsFileStorage)
-    export_data_s3 = FileField(null=True, storage=storage_backends.ExportsS3Storage)
-
     export_json = JSONField(null=True, encoder=DjangoJSONEncoder)
     json_completed = BooleanField(default=False)
 
-    @property
-    def export_data(self):
-        """returns the file field of the configured storage backend"""
-        # TODO: We could check whether a field for a different backend is
-        # filled, to support migrating to a different backend.
-        if settings.USE_S3:
-            return self.export_data_s3
-        return self.export_data_file
-
-    @export_data.setter
-    def export_data(self, value):
-        """sets the file field of the configured storage backend"""
-        if settings.USE_S3:
-            self.export_data_s3 = value
-        else:
-            self.export_data_file = value
-
     def start_job(self):
         """Start the job"""
 
@@ -265,15 +252,15 @@ class AddFileToTar(ChildJob):
 
                 # Create archive and store file name
                 s3_tar.tar()
-                export_job.export_data_s3 = s3_archive_path
+                export_job.export_data = s3_archive_path
-                export_job.save()
+                export_job.save(update_fields=["export_data"])
 
                 # Delete temporary files
                 S3Boto3Storage.delete(storage, export_json_tmp_file)
 
             else:
-                export_job.export_data_file = f"{export_task_id}.tar.gz"
+                export_job.export_data = f"{export_task_id}.tar.gz"
-                with export_job.export_data_file.open("wb") as tar_file:
+                with export_job.export_data.open("wb") as tar_file:
                     with BookwyrmTarFile.open(mode="w:gz", fileobj=tar_file) as tar:
                         # save json file
                         tar.write_bytes(export_json_bytes)
@@ -285,7 +272,7 @@ class AddFileToTar(ChildJob):
                         for edition in editions:
                             if edition.cover:
                                 tar.add_image(edition.cover, directory="images/")
-                export_job.save()
+                export_job.save(update_fields=["export_data"])
 
             self.complete_job()
 

bookwyrm/settings.py

@@ -390,16 +390,20 @@ if USE_S3:
     # S3 Static settings
     STATIC_LOCATION = "static"
     STATIC_URL = f"{PROTOCOL}://{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/"
+    STATIC_FULL_URL = STATIC_URL
     STATICFILES_STORAGE = "bookwyrm.storage_backends.StaticStorage"
     # S3 Media settings
     MEDIA_LOCATION = "images"
     MEDIA_URL = f"{PROTOCOL}://{AWS_S3_CUSTOM_DOMAIN}/{MEDIA_LOCATION}/"
     MEDIA_FULL_URL = MEDIA_URL
-    STATIC_FULL_URL = STATIC_URL
     DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
+    # S3 Exports settings
+    EXPORTS_STORAGE = "bookwyrm.storage_backends.ExportsS3Storage"
+    # Content Security Policy
     CSP_DEFAULT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
     CSP_SCRIPT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
 elif USE_AZURE:
+    # Azure settings
     AZURE_ACCOUNT_NAME = env("AZURE_ACCOUNT_NAME")
     AZURE_ACCOUNT_KEY = env("AZURE_ACCOUNT_KEY")
     AZURE_CONTAINER = env("AZURE_CONTAINER")
@@ -409,6 +413,7 @@ elif USE_AZURE:
     STATIC_URL = (
         f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{STATIC_LOCATION}/"
     )
+    STATIC_FULL_URL = STATIC_URL
     STATICFILES_STORAGE = "bookwyrm.storage_backends.AzureStaticStorage"
     # Azure Media settings
     MEDIA_LOCATION = "images"
@@ -416,15 +421,24 @@ elif USE_AZURE:
         f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{MEDIA_LOCATION}/"
     )
     MEDIA_FULL_URL = MEDIA_URL
-    STATIC_FULL_URL = STATIC_URL
     DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.AzureImagesStorage"
+    # Azure Exports settings
+    EXPORTS_STORAGE = None  # not implemented yet
+    # Content Security Policy
     CSP_DEFAULT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
     CSP_SCRIPT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
 else:
+    # Static settings
     STATIC_URL = "/static/"
+    STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}"
+    STATICFILES_STORAGE = "django.contrib.staticfiles.storage.StaticFilesStorage"
+    # Media settings
     MEDIA_URL = "/images/"
     MEDIA_FULL_URL = f"{PROTOCOL}://{DOMAIN}{MEDIA_URL}"
-    STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}"
+    DEFAULT_FILE_STORAGE = "django.core.files.storage.FileSystemStorage"
+    # Exports settings
+    EXPORTS_STORAGE = "bookwyrm.storage_backends.ExportsFileStorage"
+    # Content Security Policy
     CSP_DEFAULT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS
     CSP_SCRIPT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS