Use S3 as static and media storage

TODO

- [ ] Write some documentation for non-AWS, S3-compatible services
- [ ] Make a management command to move existing media to the new S3 bucket
- [ ] See if Nginx can reverse-proxy the S3 bucket in order to keep the files accessible through the domain name

.env.dev.example

@@ -50,3 +50,13 @@ EMAIL_USE_SSL=false
 
 # Set this to true when initializing certbot for domain, false when not
 CERTBOT_INIT=false
+
+# S3 configuration
+# Commented are example values if you use Scaleway instead of AWS
+USE_S3=false
+# AWS_ACCESS_KEY_ID=
+# AWS_SECRET_ACCESS_KEY=
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"

.env.prod.example

@@ -50,3 +50,13 @@ EMAIL_USE_SSL=false
 
 # Set this to true when initializing certbot for domain, false when not
 CERTBOT_INIT=false
+
+# S3 configuration
+# Commented are example values if you use Scaleway instead of AWS
+USE_S3=false
+# AWS_ACCESS_KEY_ID=
+# AWS_SECRET_ACCESS_KEY=
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"

bookwyrm/settings.py

@@ -67,6 +67,7 @@ INSTALLED_APPS = [
     "django_rename_app",
     "bookwyrm",
     "celery",
+    "storages",
 ]
 
 MIDDLEWARE = [
@@ -174,17 +175,47 @@ USE_L10N = True
 USE_TZ = True
 
 
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/3.2/howto/static-files/
-
-PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
-STATIC_URL = "/static/"
-STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
-MEDIA_URL = "/images/"
-MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
-
 USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % (
     requests.utils.default_user_agent(),
     VERSION,
     DOMAIN,
 )
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.2/howto/static-files/
+
+PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Storage
+
+USE_S3 = env.bool('USE_S3', False)
+
+if USE_S3:
+    # AWS settings
+    AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID")
+    AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
+    AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
+    AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN")
+    AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME")
+    AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL")
+    AWS_DEFAULT_ACL = "public-read"
+    AWS_S3_OBJECT_PARAMETERS = {'CacheControl': 'max-age=86400'}
+    # S3 Static settings
+    STATIC_LOCATION = 'static'
+    STATIC_URL = 'https://%s/%s/' % (AWS_S3_CUSTOM_DOMAIN, STATIC_LOCATION)
+    STATICFILES_STORAGE = 'bookwyrm.storage_backends.StaticStorage'
+    # S3 Media settings
+    MEDIA_LOCATION = 'images'
+    MEDIA_URL = 'https://%s/%s/' % (AWS_S3_CUSTOM_DOMAIN, MEDIA_LOCATION)
+    DEFAULT_FILE_STORAGE = 'bookwyrm.storage_backends.ImagesStorage'
+    # I don't know if it's used, but the site crashes without it
+    MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
+else:
+    STATIC_URL = "/static/"
+    STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
+    MEDIA_URL = "/images/"
+    MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
+
+STATICFILES_DIRS = [
+    os.path.join(BASE_DIR, 'static'),
+]

bookwyrm/storage_backends.py

@@ -0,0 +1,11 @@
+from storages.backends.s3boto3 import S3Boto3Storage
+
+class StaticStorage(S3Boto3Storage):
+    location = 'static'
+    default_acl = 'public-read'
+
+
+class ImagesStorage(S3Boto3Storage):
+    location = 'images'
+    default_acl = 'public-read'
+    file_overwrite = False

requirements.txt

@@ -13,6 +13,8 @@ requests==2.22.0
 responses==0.10.14
 django-rename-app==0.1.2
 pytz>=2021.1
+boto3==1.17.88
+django-storages==1.11.1
 
 # Dev
 black==21.4b0