From cba34780d8464ed77fd28b25537971ffa3d80fee Mon Sep 17 00:00:00 2001 From: Joachim Date: Mon, 7 Jun 2021 18:12:48 +0200 Subject: [PATCH] Use S3 as static and media storage TODO - [ ] Write some documentation for non-AWS, S3-compatible services - [ ] Make a management command to move existing media to the new S3 bucket - [ ] See if Nginx can reverse-proxy the S3 bucket in order to keep the files accessible through the domain name --- .env.dev.example | 10 ++++++++ .env.prod.example | 10 ++++++++ bookwyrm/settings.py | 49 +++++++++++++++++++++++++++++------- bookwyrm/storage_backends.py | 11 ++++++++ requirements.txt | 2 ++ 5 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 bookwyrm/storage_backends.py diff --git a/.env.dev.example b/.env.dev.example index 03e6ef25..f974e73e 100644 --- a/.env.dev.example +++ b/.env.dev.example @@ -50,3 +50,13 @@ EMAIL_USE_SSL=false # Set this to true when initializing certbot for domain, false when not CERTBOT_INIT=false + +# S3 configuration +# Commented are example values if you use Scaleway instead of AWS +USE_S3=false +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# AWS_STORAGE_BUCKET_NAME= # "example-bucket-name" +# AWS_S3_CUSTOM_DOMAIN=None # "example-bucket-name.s3.fr-par.scw.cloud" +# AWS_S3_REGION_NAME=None # "fr-par" +# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud" diff --git a/.env.prod.example b/.env.prod.example index dd15fbb7..97dce4d4 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -50,3 +50,13 @@ EMAIL_USE_SSL=false # Set this to true when initializing certbot for domain, false when not CERTBOT_INIT=false + +# S3 configuration +# Commented are example values if you use Scaleway instead of AWS +USE_S3=false +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# AWS_STORAGE_BUCKET_NAME= # "example-bucket-name" +# AWS_S3_CUSTOM_DOMAIN=None # "example-bucket-name.s3.fr-par.scw.cloud" +# AWS_S3_REGION_NAME=None # "fr-par" +# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud" diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 48e02b67..c6c393f7 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -67,6 +67,7 @@ INSTALLED_APPS = [ "django_rename_app", "bookwyrm", "celery", + "storages", ] MIDDLEWARE = [ @@ -174,17 +175,47 @@ USE_L10N = True USE_TZ = True -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.2/howto/static-files/ - -PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) -STATIC_URL = "/static/" -STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static")) -MEDIA_URL = "/images/" -MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images")) - USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % ( requests.utils.default_user_agent(), VERSION, DOMAIN, ) + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ + +PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) + +# Storage + +USE_S3 = env.bool('USE_S3', False) + +if USE_S3: + # AWS settings + AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID") + AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY") + AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME") + AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN") + AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME") + AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL") + AWS_DEFAULT_ACL = "public-read" + AWS_S3_OBJECT_PARAMETERS = {'CacheControl': 'max-age=86400'} + # S3 Static settings + STATIC_LOCATION = 'static' + STATIC_URL = 'https://%s/%s/' % (AWS_S3_CUSTOM_DOMAIN, STATIC_LOCATION) + STATICFILES_STORAGE = 'bookwyrm.storage_backends.StaticStorage' + # S3 Media settings + MEDIA_LOCATION = 'images' + MEDIA_URL = 'https://%s/%s/' % (AWS_S3_CUSTOM_DOMAIN, MEDIA_LOCATION) + DEFAULT_FILE_STORAGE = 'bookwyrm.storage_backends.ImagesStorage' + # I don't know if it's used, but the site crashes without it + MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images")) +else: + STATIC_URL = "/static/" + STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static")) + MEDIA_URL = "/images/" + MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images")) + +STATICFILES_DIRS = [ + os.path.join(BASE_DIR, 'static'), +] diff --git a/bookwyrm/storage_backends.py b/bookwyrm/storage_backends.py new file mode 100644 index 00000000..50afaebc --- /dev/null +++ b/bookwyrm/storage_backends.py @@ -0,0 +1,11 @@ +from storages.backends.s3boto3 import S3Boto3Storage + +class StaticStorage(S3Boto3Storage): + location = 'static' + default_acl = 'public-read' + + +class ImagesStorage(S3Boto3Storage): + location = 'images' + default_acl = 'public-read' + file_overwrite = False \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3e50a034..1d0c794f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,8 @@ requests==2.22.0 responses==0.10.14 django-rename-app==0.1.2 pytz>=2021.1 +boto3==1.17.88 +django-storages==1.11.1 # Dev black==21.4b0