From cba34780d8464ed77fd28b25537971ffa3d80fee Mon Sep 17 00:00:00 2001 From: Joachim Date: Mon, 7 Jun 2021 18:12:48 +0200 Subject: [PATCH 01/74] Use S3 as static and media storage TODO - [ ] Write some documentation for non-AWS, S3-compatible services - [ ] Make a management command to move existing media to the new S3 bucket - [ ] See if Nginx can reverse-proxy the S3 bucket in order to keep the files accessible through the domain name --- .env.dev.example | 10 ++++++++ .env.prod.example | 10 ++++++++ bookwyrm/settings.py | 49 +++++++++++++++++++++++++++++------- bookwyrm/storage_backends.py | 11 ++++++++ requirements.txt | 2 ++ 5 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 bookwyrm/storage_backends.py diff --git a/.env.dev.example b/.env.dev.example index 03e6ef258..f974e73ec 100644 --- a/.env.dev.example +++ b/.env.dev.example @@ -50,3 +50,13 @@ EMAIL_USE_SSL=false # Set this to true when initializing certbot for domain, false when not CERTBOT_INIT=false + +# S3 configuration +# Commented are example values if you use Scaleway instead of AWS +USE_S3=false +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# AWS_STORAGE_BUCKET_NAME= # "example-bucket-name" +# AWS_S3_CUSTOM_DOMAIN=None # "example-bucket-name.s3.fr-par.scw.cloud" +# AWS_S3_REGION_NAME=None # "fr-par" +# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud" diff --git a/.env.prod.example b/.env.prod.example index dd15fbb79..97dce4d47 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -50,3 +50,13 @@ EMAIL_USE_SSL=false # Set this to true when initializing certbot for domain, false when not CERTBOT_INIT=false + +# S3 configuration +# Commented are example values if you use Scaleway instead of AWS +USE_S3=false +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# AWS_STORAGE_BUCKET_NAME= # "example-bucket-name" +# AWS_S3_CUSTOM_DOMAIN=None # "example-bucket-name.s3.fr-par.scw.cloud" +# AWS_S3_REGION_NAME=None # "fr-par" +# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud" diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 48e02b67b..c6c393f71 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -67,6 +67,7 @@ INSTALLED_APPS = [ "django_rename_app", "bookwyrm", "celery", + "storages", ] MIDDLEWARE = [ @@ -174,17 +175,47 @@ USE_L10N = True USE_TZ = True -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.2/howto/static-files/ - -PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) -STATIC_URL = "/static/" -STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static")) -MEDIA_URL = "/images/" -MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images")) - USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % ( requests.utils.default_user_agent(), VERSION, DOMAIN, ) + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ + +PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) + +# Storage + +USE_S3 = env.bool('USE_S3', False) + +if USE_S3: + # AWS settings + AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID") + AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY") + AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME") + AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN") + AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME") + AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL") + AWS_DEFAULT_ACL = "public-read" + AWS_S3_OBJECT_PARAMETERS = {'CacheControl': 'max-age=86400'} + # S3 Static settings + STATIC_LOCATION = 'static' + STATIC_URL = 'https://%s/%s/' % (AWS_S3_CUSTOM_DOMAIN, STATIC_LOCATION) + STATICFILES_STORAGE = 'bookwyrm.storage_backends.StaticStorage' + # S3 Media settings + MEDIA_LOCATION = 'images' + MEDIA_URL = 'https://%s/%s/' % (AWS_S3_CUSTOM_DOMAIN, MEDIA_LOCATION) + DEFAULT_FILE_STORAGE = 'bookwyrm.storage_backends.ImagesStorage' + # I don't know if it's used, but the site crashes without it + MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images")) +else: + STATIC_URL = "/static/" + STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static")) + MEDIA_URL = "/images/" + MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images")) + +STATICFILES_DIRS = [ + os.path.join(BASE_DIR, 'static'), +] diff --git a/bookwyrm/storage_backends.py b/bookwyrm/storage_backends.py new file mode 100644 index 000000000..50afaebce --- /dev/null +++ b/bookwyrm/storage_backends.py @@ -0,0 +1,11 @@ +from storages.backends.s3boto3 import S3Boto3Storage + +class StaticStorage(S3Boto3Storage): + location = 'static' + default_acl = 'public-read' + + +class ImagesStorage(S3Boto3Storage): + location = 'images' + default_acl = 'public-read' + file_overwrite = False \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3e50a0341..1d0c794f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,8 @@ requests==2.22.0 responses==0.10.14 django-rename-app==0.1.2 pytz>=2021.1 +boto3==1.17.88 +django-storages==1.11.1 # Dev black==21.4b0 From b448d4051424ddabafb9188eb9e42788c241b22f Mon Sep 17 00:00:00 2001 From: Joachim Date: Mon, 7 Jun 2021 18:13:24 +0200 Subject: [PATCH 02/74] Use the right snippets on Templates --- bookwyrm/templates/book/book.html | 8 +++++-- bookwyrm/templates/feed/feed_layout.html | 3 ++- bookwyrm/templates/get_started/layout.html | 3 ++- bookwyrm/templates/import_status.html | 3 ++- bookwyrm/templates/layout.html | 22 ++++++++++--------- bookwyrm/templates/snippets/about.html | 4 +++- bookwyrm/templates/snippets/avatar.html | 4 +++- bookwyrm/templates/snippets/book_cover.html | 5 +++-- .../snippets/status/content_status.html | 5 +++-- .../snippets/status/status_header.html | 3 ++- 10 files changed, 38 insertions(+), 22 deletions(-) diff --git a/bookwyrm/templates/book/book.html b/bookwyrm/templates/book/book.html index af2230200..0f0655517 100644 --- a/bookwyrm/templates/book/book.html +++ b/bookwyrm/templates/book/book.html @@ -1,5 +1,9 @@ {% extends 'layout.html' %} -{% load i18n %}{% load bookwyrm_tags %}{% load humanize %}{% load utilities %} +{% load i18n %} +{% load bookwyrm_tags %} +{% load humanize %} +{% load utilities %} +{% load static %} {% block title %}{{ book|book_title }}{% endblock %} @@ -309,5 +313,5 @@ {% endblock %} {% block scripts %} - + {% endblock %} diff --git a/bookwyrm/templates/feed/feed_layout.html b/bookwyrm/templates/feed/feed_layout.html index 3c6b65a1b..07784418a 100644 --- a/bookwyrm/templates/feed/feed_layout.html +++ b/bookwyrm/templates/feed/feed_layout.html @@ -1,5 +1,6 @@ {% extends 'layout.html' %} {% load i18n %} +{% load static %} {% block title %}{% trans "Updates" %}{% endblock %} @@ -103,5 +104,5 @@ {% endblock %} {% block scripts %} - + {% endblock %} diff --git a/bookwyrm/templates/get_started/layout.html b/bookwyrm/templates/get_started/layout.html index 63c042895..08a2e7c54 100644 --- a/bookwyrm/templates/get_started/layout.html +++ b/bookwyrm/templates/get_started/layout.html @@ -1,5 +1,6 @@ {% extends 'layout.html' %} {% load i18n %} +{% load static %} {% block title %}{% trans "Welcome" %}{% endblock %} @@ -9,7 +10,7 @@
- +

{% blocktrans %}Welcome to {{ site_name }}!{% endblocktrans %} diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html index ff80ad71c..b9d19318c 100644 --- a/bookwyrm/templates/import_status.html +++ b/bookwyrm/templates/import_status.html @@ -1,6 +1,7 @@ {% extends 'layout.html' %} {% load i18n %} {% load humanize %} +{% load static %} {% block title %}{% trans "Import Status" %}{% endblock %} @@ -151,5 +152,5 @@ {% endspaceless %}{% endblock %} {% block scripts %} - + {% endblock %} diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html index 56530a841..bf509d495 100644 --- a/bookwyrm/templates/layout.html +++ b/bookwyrm/templates/layout.html @@ -1,14 +1,16 @@ -{% load layout %}{% load i18n %} +{% load layout %} +{% load i18n %} +{% load static %} {% block title %}BookWyrm{% endblock %} | {{ site.name }} - - - + + + - + @@ -16,8 +18,8 @@ - - + + @@ -25,7 +27,7 @@
- +
@@ -237,8 +239,8 @@ - - + + {% block scripts %}{% endblock %} diff --git a/bookwyrm/templates/snippets/about.html b/bookwyrm/templates/snippets/about.html index 820fb5759..a2b31b185 100644 --- a/bookwyrm/templates/snippets/about.html +++ b/bookwyrm/templates/snippets/about.html @@ -1,7 +1,9 @@ +{% load static %} +
- BookWyrm logo + BookWyrm logo
diff --git a/bookwyrm/templates/snippets/avatar.html b/bookwyrm/templates/snippets/avatar.html index bb37c46d4..5f28cd4be 100644 --- a/bookwyrm/templates/snippets/avatar.html +++ b/bookwyrm/templates/snippets/avatar.html @@ -1,2 +1,4 @@ -{{ user.alt_text }} +{% load static %} + +{{ user.alt_text }} diff --git a/bookwyrm/templates/snippets/book_cover.html b/bookwyrm/templates/snippets/book_cover.html index 16ab305e5..aa315e360 100644 --- a/bookwyrm/templates/snippets/book_cover.html +++ b/bookwyrm/templates/snippets/book_cover.html @@ -1,6 +1,7 @@ {% spaceless %} {% load i18n %} +{% load static %}
diff --git a/bookwyrm/templates/snippets/status/content_status.html b/bookwyrm/templates/snippets/status/content_status.html index 39ef9173c..c33b34526 100644 --- a/bookwyrm/templates/snippets/status/content_status.html +++ b/bookwyrm/templates/snippets/status/content_status.html @@ -1,6 +1,7 @@ {% load bookwyrm_tags %} {% load markdown %} {% load i18n %} +{% load static %} {% with status_type=status.status_type %}
{{ attachment.caption }}