Merge pull request #1163 from joachimesque/storage-s3

Use S3 as static and media storage
2024-11-19 16:11:05 +00:00 · 2021-08-02 08:29:09 -06:00 · 2021-08-02 08:29:09 -06:00 · b40b8c2d0e
commit b40b8c2d0e
parent d76106ded5 d70469d2e0
19 changed files with 196 additions and 46 deletions
--- a/.env.dev.example
+++ b/.env.dev.example
@ -3,6 +3,7 @@ SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"

 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG=true
+USE_HTTPS=false

 DOMAIN=your.domain.here
 #EMAIL=your@email.here
@ -42,6 +43,21 @@ EMAIL_HOST_PASSWORD=emailpassword123
 EMAIL_USE_TLS=true
 EMAIL_USE_SSL=false

+# S3 configuration
+USE_S3=false
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+# Commented are example values if you use a non-AWS, S3-compatible service
+# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
+# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
+
 # Preview image generation can be computing and storage intensive
 # ENABLE_PREVIEW_IMAGES=True

--- a/.env.prod.example
+++ b/.env.prod.example
@ -3,6 +3,7 @@ SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"

 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG=false
+USE_HTTPS=true

 DOMAIN=your.domain.here
 EMAIL=your@email.here
@ -42,6 +43,21 @@ EMAIL_HOST_PASSWORD=emailpassword123
 EMAIL_USE_TLS=true
 EMAIL_USE_SSL=false

+# S3 configuration
+USE_S3=false
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+# Commented are example values if you use a non-AWS, S3-compatible service
+# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
+# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
+
 # Preview image generation can be computing and storage intensive
 # ENABLE_PREVIEW_IMAGES=True

--- a/bookwyrm/context_processors.py
+++ b/bookwyrm/context_processors.py
@ -11,10 +11,7 @@ def site_settings(request):  # pylint: disable=unused-argument
    return {
        "site": models.SiteSettings.objects.get(),
        "active_announcements": models.Announcement.active_announcements(),
-        "static_url": settings.STATIC_URL,
-        "media_url": settings.MEDIA_URL,
-        "static_path": settings.STATIC_PATH,
-        "media_path": settings.MEDIA_PATH,
+        "media_full_url": settings.MEDIA_FULL_URL,
        "preview_images_enabled": settings.ENABLE_PREVIEW_IMAGES,
        "request_protocol": request_protocol,
    }
--- a/bookwyrm/preview_images.py
+++ b/bookwyrm/preview_images.py
@ -11,6 +11,7 @@ from PIL import Image, ImageDraw, ImageFont, ImageOps, ImageColor

 from django.core.files.base import ContentFile
 from django.core.files.uploadedfile import InMemoryUploadedFile
+from django.core.files.storage import default_storage
 from django.db.models import Avg

 from bookwyrm import models, settings
@ -319,9 +320,9 @@ def save_and_cleanup(image, instance=None):

    try:
        try:
-            old_path = instance.preview_image.path
+            old_path = instance.preview_image.name
        except ValueError:
-            old_path = ""
+            old_path = None

        # Save
        image.save(image_buffer, format="jpeg", quality=75)
@ -342,8 +343,8 @@ def save_and_cleanup(image, instance=None):
            instance.save()

        # Clean up old file after saving
-        if os.path.exists(old_path):
-            os.remove(old_path)
+        if old_path and default_storage.exists(old_path):
+            default_storage.delete(old_path)

    finally:
        image_buffer.close()
--- a/bookwyrm/settings.py
+++ b/bookwyrm/settings.py
@ -58,6 +58,7 @@ SECRET_KEY = env("SECRET_KEY")

 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = env.bool("DEBUG", True)
+USE_HTTPS = env.bool("USE_HTTPS", False)

 ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", ["*"])

@ -74,6 +75,7 @@ INSTALLED_APPS = [
    "django_rename_app",
    "bookwyrm",
    "celery",
+    "storages",
 ]

 MIDDLEWARE = [
@ -179,19 +181,51 @@ USE_L10N = True
 USE_TZ = True


-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/3.2/howto/static-files/
-
-PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
-STATIC_URL = "/static/"
-STATIC_PATH = "%s/%s" % (DOMAIN, env("STATIC_ROOT", "static"))
-STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
-MEDIA_URL = "/images/"
-MEDIA_PATH = "%s/%s" % (DOMAIN, env("MEDIA_ROOT", "images"))
-MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
-
 USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % (
    requests.utils.default_user_agent(),
    VERSION,
    DOMAIN,
 )
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.2/howto/static-files/
+
+PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Storage
+
+PROTOCOL = "http"
+if USE_HTTPS:
+    PROTOCOL = "https"
+
+USE_S3 = env.bool("USE_S3", False)
+
+if USE_S3:
+    # AWS settings
+    AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID")
+    AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
+    AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
+    AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN")
+    AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME", "")
+    AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL")
+    AWS_DEFAULT_ACL = "public-read"
+    AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"}
+    # S3 Static settings
+    STATIC_LOCATION = "static"
+    STATIC_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, STATIC_LOCATION)
+    STATICFILES_STORAGE = "bookwyrm.storage_backends.StaticStorage"
+    # S3 Media settings
+    MEDIA_LOCATION = "images"
+    MEDIA_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, MEDIA_LOCATION)
+    MEDIA_FULL_URL = MEDIA_URL
+    DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
+    # I don't know if it's used, but the site crashes without it
+    STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
+    MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
+else:
+    STATIC_URL = "/static/"
+    STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
+    MEDIA_URL = "/images/"
+    MEDIA_FULL_URL = "%s://%s%s" % (PROTOCOL, DOMAIN, MEDIA_URL)
+    MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
--- a/bookwyrm/storage_backends.py
+++ b/bookwyrm/storage_backends.py
@ -0,0 +1,17 @@
+"""Handles backends for storages"""
+from storages.backends.s3boto3 import S3Boto3Storage
+
+
+class StaticStorage(S3Boto3Storage):  # pylint: disable=abstract-method
+    """Storage class for Static contents"""
+
+    location = "static"
+    default_acl = "public-read"
+
+
+class ImagesStorage(S3Boto3Storage):  # pylint: disable=abstract-method
+    """Storage class for Image files"""
+
+    location = "images"
+    default_acl = "public-read"
+    file_overwrite = False
--- a/bookwyrm/templates/book/book.html
+++ b/bookwyrm/templates/book/book.html
@ -1,5 +1,10 @@
 {% extends 'layout.html' %}
-{% load i18n %}{% load bookwyrm_tags %}{% load humanize %}{% load utilities %}{% load layout %}
+{% load i18n %}
+{% load bookwyrm_tags %}
+{% load humanize %}
+{% load utilities %}
+{% load static %}
+{% load layout %}

 {% block title %}{{ book|book_title }}{% endblock %}

@ -321,5 +326,5 @@
 {% endblock %}

 {% block scripts %}
-<script src="/static/js/vendor/tabs.js"></script>
+<script src="{% static "js/vendor/tabs.js" %}"></script>
 {% endblock %}
--- a/bookwyrm/templates/feed/layout.html
+++ b/bookwyrm/templates/feed/layout.html
@ -1,5 +1,6 @@
 {% extends 'layout.html' %}
 {% load i18n %}
+{% load static %}

 {% block title %}{% trans "Updates" %}{% endblock %}

@ -105,5 +106,5 @@
 {% endblock %}

 {% block scripts %}
-<script src="/static/js/vendor/tabs.js"></script>
+<script src="{% static "js/vendor/tabs.js" %}"></script>
 {% endblock %}
--- a/bookwyrm/templates/get_started/layout.html
+++ b/bookwyrm/templates/get_started/layout.html
@ -1,5 +1,6 @@
 {% extends 'layout.html' %}
 {% load i18n %}
+{% load static %}

 {% block title %}{% trans "Welcome" %}{% endblock %}

@ -9,7 +10,7 @@
    <div class="modal-background"></div>
    <div class="modal-card is-fullwidth">
        <header class="modal-card-head">
-            <img class="image logo mr-2" src="{% if site.logo_small %}/images/{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" aria-hidden="true">
+            <img class="image logo mr-2" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" aria-hidden="true">
            <h1 class="modal-card-title" id="get-started-header">
                {% blocktrans %}Welcome to {{ site_name }}!{% endblocktrans %}
                <span class="subtitle is-block">
--- a/bookwyrm/templates/import_status.html
+++ b/bookwyrm/templates/import_status.html
@ -1,6 +1,7 @@
 {% extends 'layout.html' %}
 {% load i18n %}
 {% load humanize %}
+{% load static %}

 {% block title %}{% trans "Import Status" %}{% endblock %}

@ -156,5 +157,5 @@
 {% endspaceless %}{% endblock %}

 {% block scripts %}
-<script src="/static/js/check_all.js"></script>
+<script src="{% static "js/check_all.js" %}"></script>
 {% endblock %}
--- a/bookwyrm/templates/layout.html
+++ b/bookwyrm/templates/layout.html
@ -1,14 +1,16 @@
-{% load layout %}{% load i18n %}
+{% load layout %}
+{% load i18n %}
+{% load static %}
 <!DOCTYPE html>
 <html lang="{% get_lang %}">
 <head>
    <title>{% block title %}BookWyrm{% endblock %} | {{ site.name }}</title>
    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="stylesheet" href="/static/css/vendor/bulma.min.css">
-    <link rel="stylesheet" href="/static/css/vendor/icons.css">
-    <link rel="stylesheet" href="/static/css/bookwyrm.css">
+    <link rel="stylesheet" href="{% static "css/vendor/bulma.min.css" %}">
+    <link rel="stylesheet" href="{% static "css/vendor/icons.css" %}">
+    <link rel="stylesheet" href="{% static "css/bookwyrm.css" %}">

-    <link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{{ media_url }}{{ site.favicon }}{% else %}/static/images/favicon.ico{% endif %}">
+    <link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{% get_media_prefix %}{{ site.favicon }}{% else %}{% static "images/favicon.ico" %}{% endif %}">

    {% if preview_images_enabled is True %}
    <meta name="twitter:card" content="summary_large_image">
@ -30,7 +32,7 @@
    <div class="container">
        <div class="navbar-brand">
            <a class="navbar-item" href="/">
-                <img class="image logo" src="{% if site.logo_small %}{{ media_url }}{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" alt="Home page">
+                <img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="Home page">
            </a>
            <form class="navbar-item column" action="/search/">
                <div class="field has-addons">
@ -242,8 +244,8 @@
 <script>
    var csrf_token = '{{ csrf_token }}';
 </script>
-<script src="/static/js/bookwyrm.js"></script>
-<script src="/static/js/localstorage.js"></script>
+<script src="{% static "js/bookwyrm.js" %}"></script>
+<script src="{% static "js/localstorage.js" %}"></script>
 {% block scripts %}{% endblock %}
 </body>
 </html>
--- a/bookwyrm/templates/snippets/about.html
+++ b/bookwyrm/templates/snippets/about.html
@ -1,7 +1,9 @@
+{% load static %}
+
 <div class="columns">
    <div class="column is-narrow is-hidden-mobile">
        <figure class="block is-w-xl">
-            <img src="{% if site.logo %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}" alt="BookWyrm logo">
+            <img src="{% if site.logo %}/images/{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}" alt="BookWyrm logo">
        </figure>
    </div>
    <div class="content">
--- a/bookwyrm/templates/snippets/avatar.html
+++ b/bookwyrm/templates/snippets/avatar.html
@ -1,2 +1,4 @@
-<img class="avatar image {% if large %}is-96x96{% elif medium %}is-48x48{% else %}is-32x32{% endif %}" src="{% if user.avatar %}/images/{{ user.avatar }}{% else %}/static/images/default_avi.jpg{% endif %}" {% if ariaHide %}aria-hidden="true"{% endif %} alt="{{ user.alt_text }}">
+{% load static %}
+
+<img class="avatar image {% if large %}is-96x96{% elif medium %}is-48x48{% else %}is-32x32{% endif %}" src="{% if user.avatar %}{% get_media_prefix %}{{ user.avatar }}{% else %}{% static "images/default_avi.jpg" %}{% endif %}" {% if ariaHide %}aria-hidden="true"{% endif %} alt="{{ user.alt_text }}">

--- a/bookwyrm/templates/snippets/book_cover.html
+++ b/bookwyrm/templates/snippets/book_cover.html
@ -1,6 +1,7 @@
 {% spaceless %}

 {% load i18n %}
+{% load static %}

 <figure
    class="
@ -20,14 +21,14 @@
        class="book-cover"

        {% if book.cover %}
-            src="{% if img_path is None %}/images/{% else %}{{ img_path }}{% endif %}{{ book.cover }}"
+            src="{% if img_path is None %}{% get_media_prefix %}{% else %}{{ img_path }}{% endif %}{{ book.cover }}"
            itemprop="thumbnailUrl"

            {% if book.alt_text %}
                alt="{{ book.alt_text }}"
            {% endif %}
        {% else %}
-            src="/static/images/no_cover.jpg"
+            src="{% static "images/no_cover.jpg" %}"
            alt="{% trans "No cover" %}"
        {% endif %}
    >
--- a/bookwyrm/templates/snippets/opengraph_images.html
+++ b/bookwyrm/templates/snippets/opengraph_images.html
@ -1,12 +1,14 @@
+{% load static %}
+
 {% if preview_images_enabled is True %}
    {% if image %}
-        <meta name="twitter:image" content="{{ request.scheme }}://{{ media_path }}{{ image }}">
-        <meta name="og:image" content="{{ request.scheme }}://{{ media_path }}{{ image }}">
+        <meta name="twitter:image" content="{{ media_full_url }}{{ image }}">
+        <meta name="og:image" content="{{ media_full_url }}{{ image }}">
    {% else %}
-        <meta name="twitter:image" content="{{ request.scheme }}://{{ media_path }}{{ site.preview_image }}">
-        <meta name="og:image" content="{{ request.scheme }}://{{ media_path }}{{ site.preview_image }}">
+        <meta name="twitter:image" content="{{ media_full_url }}{{ site.preview_image }}">
+        <meta name="og:image" content="{{ media_full_url }}{{ site.preview_image }}">
    {% endif %}
 {% else %}
-    <meta name="twitter:image" content="{{ request.scheme }}://{% if site.logo %}{{ media_path }}{{ site.logo }}{% else %}{{ static_path }}/images/logo.png{% endif %}">
-    <meta name="og:image" content="{{ request.scheme }}://{% if site.logo %}{{ media_path }}{{ site.logo }}{% else %}{{ static_path }}/images/logo.png{% endif %}">
+    <meta name="twitter:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
+    <meta name="og:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
 {% endif %}
--- a/bookwyrm/templates/snippets/status/content_status.html
+++ b/bookwyrm/templates/snippets/status/content_status.html
@ -1,6 +1,7 @@
 {% load bookwyrm_tags %}
 {% load markdown %}
 {% load i18n %}
+{% load static %}

 {% with status_type=status.status_type %}
 <div
@ -111,12 +112,12 @@
                                <div class="column is-narrow">
                                    <figure class="image is-128x128">
                                        <a
-                                            href="/images/{{ attachment.image }}"
+                                            href="{% get_media_prefix %}{{ attachment.image }}"
                                            target="_blank"
                                            aria-label="{% trans 'Open image in new window' %}"
                                        >
                                            <img
-                                                src="/images/{{ attachment.image }}"
+                                                src="{% get_media_prefix %}{{ attachment.image }}"

                                                {% if attachment.caption %}
                                                    alt="{{ attachment.caption }}"
--- a/bookwyrm/templates/snippets/status/status_header.html
+++ b/bookwyrm/templates/snippets/status/status_header.html
@ -2,6 +2,7 @@
 {% load status_display %}
 {% load i18n %}
 {% load humanize %}
+{% load static %}

 <div class="media">
    <figure class="media-left" aria-hidden="true">
@ -18,7 +19,7 @@
                itemtype="https://schema.org/Person"
            >
                {% if status.user.avatar %}
-                    <meta itemprop="image" content="/images/{{ status.user.avatar }}">
+                    <meta itemprop="image" content="{% get_media_prefix %}{{ status.user.avatar }}">
                {% endif %}

                <a
--- a/50
+++ b/50
@ -38,6 +38,17 @@ function makeitblack {
    docker-compose run --rm web black celerywyrm bookwyrm
 }

+function awscommand {
+    # expose env vars
+    export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+    export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+    export AWS_DEFAULT_REGION=${AWS_S3_REGION_NAME}
+    # first arg is mountpoint, second is the whole aws command
+    docker run --rm -it -v $1\
+        -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION\
+        amazon/aws-cli $2
+}
+
 CMD=$1
 if [ -n "$CMD" ]; then
    shift
@ -115,6 +126,19 @@ case "$CMD" in
    generate_preview_images)
        runweb python manage.py generate_preview_images $@
        ;;
+    copy_media_to_s3)
+        awscommand "bookwyrm_media_volume:/images"\
+            "s3 cp /images s3://${AWS_STORAGE_BUCKET_NAME}/images\
+            --endpoint-url ${AWS_S3_ENDPOINT_URL}\
+            --recursive --acl public-read"
+        ;;
+    set_cors_to_s3)
+        awscommand "$(pwd):/bw"\
+            "s3api put-bucket-cors\
+            --bucket ${AWS_STORAGE_BUCKET_NAME}\
+            --endpoint-url ${AWS_S3_ENDPOINT_URL}\
+            --cors-configuration file:///bw/$@"
+        ;;
    runweb)
        runweb "$@"
        ;;
@ -123,6 +147,30 @@ case "$CMD" in
        ;;
    *)
        set +x # No need to echo echo
-        echo "Unrecognised command. Try: build, clean, up, initdb, resetdb, makemigrations, migrate, bash, shell, dbshell, restart_celery, test, pytest, test_report, black, populate_streams, generate_preview_images"
+        echo "Unrecognised command. Try:"
+        echo "    up [container]"
+        echo "    run"
+        echo "    initdb"
+        echo "    resetdb"
+        echo "    makemigrations [migration]"
+        echo "    migrate [migration]"
+        echo "    bash"
+        echo "    shell"
+        echo "    dbshell"
+        echo "    restart_celery"
+        echo "    test [path]"
+        echo "    pytest [path]"
+        echo "    collectstatic"
+        echo "    makemessages [locale]"
+        echo "    compilemessages [locale]"
+        echo "    build"
+        echo "    clean"
+        echo "    black"
+        echo "    populate_streams"
+        echo "    generate_preview_images [--all]"
+        echo "    copy_media_to_s3"
+        echo "    set_cors_to_s3 [cors file]"
+        echo "    runweb [command]"
+        echo "    rundb [command]"
        ;;
 esac
--- a/requirements.txt
+++ b/requirements.txt
@ -14,6 +14,8 @@ requests==2.22.0
 responses==0.10.14
 django-rename-app==0.1.2
 pytz>=2021.1
+boto3==1.17.88
+django-storages==1.11.1

 # Dev
 black==21.4b0