From cba34780d8464ed77fd28b25537971ffa3d80fee Mon Sep 17 00:00:00 2001
From: Joachim <joachim.robert@protonmail.com>
Date: Mon, 7 Jun 2021 18:12:48 +0200
Subject: [PATCH] Use S3 as static and media storage

TODO

- [ ] Write some documentation for non-AWS, S3-compatible services
- [ ] Make a management command to move existing media to the new S3 bucket
- [ ] See if Nginx can reverse-proxy the S3 bucket in order to keep the files accessible through the domain name
---
 .env.dev.example             | 10 ++++++++
 .env.prod.example            | 10 ++++++++
 bookwyrm/settings.py         | 49 +++++++++++++++++++++++++++++-------
 bookwyrm/storage_backends.py | 11 ++++++++
 requirements.txt             |  2 ++
 5 files changed, 73 insertions(+), 9 deletions(-)
 create mode 100644 bookwyrm/storage_backends.py

diff --git a/.env.dev.example b/.env.dev.example
index 03e6ef25..f974e73e 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -50,3 +50,13 @@ EMAIL_USE_SSL=false
 
 # Set this to true when initializing certbot for domain, false when not
 CERTBOT_INIT=false
+
+# S3 configuration
+# Commented are example values if you use Scaleway instead of AWS
+USE_S3=false
+# AWS_ACCESS_KEY_ID=
+# AWS_SECRET_ACCESS_KEY=
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
diff --git a/.env.prod.example b/.env.prod.example
index dd15fbb7..97dce4d4 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -50,3 +50,13 @@ EMAIL_USE_SSL=false
 
 # Set this to true when initializing certbot for domain, false when not
 CERTBOT_INIT=false
+
+# S3 configuration
+# Commented are example values if you use Scaleway instead of AWS
+USE_S3=false
+# AWS_ACCESS_KEY_ID=
+# AWS_SECRET_ACCESS_KEY=
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py
index 48e02b67..c6c393f7 100644
--- a/bookwyrm/settings.py
+++ b/bookwyrm/settings.py
@@ -67,6 +67,7 @@ INSTALLED_APPS = [
     "django_rename_app",
     "bookwyrm",
     "celery",
+    "storages",
 ]
 
 MIDDLEWARE = [
@@ -174,17 +175,47 @@ USE_L10N = True
 USE_TZ = True
 
 
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/3.2/howto/static-files/
-
-PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
-STATIC_URL = "/static/"
-STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
-MEDIA_URL = "/images/"
-MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
-
 USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % (
     requests.utils.default_user_agent(),
     VERSION,
     DOMAIN,
 )
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.2/howto/static-files/
+
+PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Storage
+
+USE_S3 = env.bool('USE_S3', False)
+
+if USE_S3:
+    # AWS settings
+    AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID")
+    AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
+    AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
+    AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN")
+    AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME")
+    AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL")
+    AWS_DEFAULT_ACL = "public-read"
+    AWS_S3_OBJECT_PARAMETERS = {'CacheControl': 'max-age=86400'}
+    # S3 Static settings
+    STATIC_LOCATION = 'static'
+    STATIC_URL = 'https://%s/%s/' % (AWS_S3_CUSTOM_DOMAIN, STATIC_LOCATION)
+    STATICFILES_STORAGE = 'bookwyrm.storage_backends.StaticStorage'
+    # S3 Media settings
+    MEDIA_LOCATION = 'images'
+    MEDIA_URL = 'https://%s/%s/' % (AWS_S3_CUSTOM_DOMAIN, MEDIA_LOCATION)
+    DEFAULT_FILE_STORAGE = 'bookwyrm.storage_backends.ImagesStorage'
+    # I don't know if it's used, but the site crashes without it
+    MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
+else:
+    STATIC_URL = "/static/"
+    STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
+    MEDIA_URL = "/images/"
+    MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
+
+STATICFILES_DIRS = [
+    os.path.join(BASE_DIR, 'static'),
+]
diff --git a/bookwyrm/storage_backends.py b/bookwyrm/storage_backends.py
new file mode 100644
index 00000000..50afaebc
--- /dev/null
+++ b/bookwyrm/storage_backends.py
@@ -0,0 +1,11 @@
+from storages.backends.s3boto3 import S3Boto3Storage
+
+class StaticStorage(S3Boto3Storage):
+    location = 'static'
+    default_acl = 'public-read'
+
+
+class ImagesStorage(S3Boto3Storage):
+    location = 'images'
+    default_acl = 'public-read'
+    file_overwrite = False
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 3e50a034..1d0c794f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,6 +13,8 @@ requests==2.22.0
 responses==0.10.14
 django-rename-app==0.1.2
 pytz>=2021.1
+boto3==1.17.88
+django-storages==1.11.1
 
 # Dev
 black==21.4b0