Merge pull request #1163 from joachimesque/storage-s3

Use S3 as static and media storage

.env.dev.example

@@ -3,6 +3,7 @@ SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG=true
+USE_HTTPS=false
 
 DOMAIN=your.domain.here
 #EMAIL=your@email.here
@@ -42,6 +43,21 @@ EMAIL_HOST_PASSWORD=emailpassword123
 EMAIL_USE_TLS=true
 EMAIL_USE_SSL=false
 
+# S3 configuration
+USE_S3=false
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+# Commented are example values if you use a non-AWS, S3-compatible service
+# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
+# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
+
 # Preview image generation can be computing and storage intensive
 # ENABLE_PREVIEW_IMAGES=True
 

.env.prod.example

@@ -3,6 +3,7 @@ SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG=false
+USE_HTTPS=true
 
 DOMAIN=your.domain.here
 EMAIL=your@email.here
@@ -42,6 +43,21 @@ EMAIL_HOST_PASSWORD=emailpassword123
 EMAIL_USE_TLS=true
 EMAIL_USE_SSL=false
 
+# S3 configuration
+USE_S3=false
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+# Commented are example values if you use a non-AWS, S3-compatible service
+# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
+# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
+
 # Preview image generation can be computing and storage intensive
 # ENABLE_PREVIEW_IMAGES=True
 

bookwyrm/context_processors.py

@@ -11,10 +11,7 @@ def site_settings(request):  # pylint: disable=unused-argument
     return {
         "site": models.SiteSettings.objects.get(),
         "active_announcements": models.Announcement.active_announcements(),
-        "static_url": settings.STATIC_URL,
+        "media_full_url": settings.MEDIA_FULL_URL,
-        "media_url": settings.MEDIA_URL,
-        "static_path": settings.STATIC_PATH,
-        "media_path": settings.MEDIA_PATH,
         "preview_images_enabled": settings.ENABLE_PREVIEW_IMAGES,
         "request_protocol": request_protocol,
     }

bookwyrm/preview_images.py

@@ -11,6 +11,7 @@ from PIL import Image, ImageDraw, ImageFont, ImageOps, ImageColor
 
 from django.core.files.base import ContentFile
 from django.core.files.uploadedfile import InMemoryUploadedFile
+from django.core.files.storage import default_storage
 from django.db.models import Avg
 
 from bookwyrm import models, settings
@@ -319,9 +320,9 @@ def save_and_cleanup(image, instance=None):
 
     try:
         try:
-            old_path = instance.preview_image.path
+            old_path = instance.preview_image.name
         except ValueError:
-            old_path = ""
+            old_path = None
 
         # Save
         image.save(image_buffer, format="jpeg", quality=75)
@@ -342,8 +343,8 @@ def save_and_cleanup(image, instance=None):
             instance.save()
 
         # Clean up old file after saving
-        if os.path.exists(old_path):
+        if old_path and default_storage.exists(old_path):
-            os.remove(old_path)
+            default_storage.delete(old_path)
 
     finally:
         image_buffer.close()

bookwyrm/settings.py

@@ -58,6 +58,7 @@ SECRET_KEY = env("SECRET_KEY")
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = env.bool("DEBUG", True)
+USE_HTTPS = env.bool("USE_HTTPS", False)
 
 ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", ["*"])
 
@@ -74,6 +75,7 @@ INSTALLED_APPS = [
     "django_rename_app",
     "bookwyrm",
     "celery",
+    "storages",
 ]
 
 MIDDLEWARE = [
@@ -179,19 +181,51 @@ USE_L10N = True
 USE_TZ = True
 
 
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/3.2/howto/static-files/
-
-PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
-STATIC_URL = "/static/"
-STATIC_PATH = "%s/%s" % (DOMAIN, env("STATIC_ROOT", "static"))
-STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
-MEDIA_URL = "/images/"
-MEDIA_PATH = "%s/%s" % (DOMAIN, env("MEDIA_ROOT", "images"))
-MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
-
 USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % (
     requests.utils.default_user_agent(),
     VERSION,
     DOMAIN,
 )
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.2/howto/static-files/
+
+PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Storage
+
+PROTOCOL = "http"
+if USE_HTTPS:
+    PROTOCOL = "https"
+
+USE_S3 = env.bool("USE_S3", False)
+
+if USE_S3:
+    # AWS settings
+    AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID")
+    AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
+    AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
+    AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN")
+    AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME", "")
+    AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL")
+    AWS_DEFAULT_ACL = "public-read"
+    AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"}
+    # S3 Static settings
+    STATIC_LOCATION = "static"
+    STATIC_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, STATIC_LOCATION)
+    STATICFILES_STORAGE = "bookwyrm.storage_backends.StaticStorage"
+    # S3 Media settings
+    MEDIA_LOCATION = "images"
+    MEDIA_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, MEDIA_LOCATION)
+    MEDIA_FULL_URL = MEDIA_URL
+    DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
+    # I don't know if it's used, but the site crashes without it
+    STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
+    MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
+else:
+    STATIC_URL = "/static/"
+    STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
+    MEDIA_URL = "/images/"
+    MEDIA_FULL_URL = "%s://%s%s" % (PROTOCOL, DOMAIN, MEDIA_URL)
+    MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))

bookwyrm/storage_backends.py

@@ -0,0 +1,17 @@
+"""Handles backends for storages"""
+from storages.backends.s3boto3 import S3Boto3Storage
+
+
+class StaticStorage(S3Boto3Storage):  # pylint: disable=abstract-method
+    """Storage class for Static contents"""
+
+    location = "static"
+    default_acl = "public-read"
+
+
+class ImagesStorage(S3Boto3Storage):  # pylint: disable=abstract-method
+    """Storage class for Image files"""
+
+    location = "images"
+    default_acl = "public-read"
+    file_overwrite = False

bookwyrm/templates/book/book.html

@@ -1,5 +1,10 @@
 {% extends 'layout.html' %}
-{% load i18n %}{% load bookwyrm_tags %}{% load humanize %}{% load utilities %}{% load layout %}
+{% load i18n %}
+{% load bookwyrm_tags %}
+{% load humanize %}
+{% load utilities %}
+{% load static %}
+{% load layout %}
 
 {% block title %}{{ book|book_title }}{% endblock %}
 
@@ -321,5 +326,5 @@
 {% endblock %}
 
 {% block scripts %}
-<script src="/static/js/vendor/tabs.js"></script>
+<script src="{% static "js/vendor/tabs.js" %}"></script>
 {% endblock %}

bookwyrm/templates/feed/layout.html

@@ -1,5 +1,6 @@
 {% extends 'layout.html' %}
 {% load i18n %}
+{% load static %}
 
 {% block title %}{% trans "Updates" %}{% endblock %}
 
@@ -105,5 +106,5 @@
 {% endblock %}
 
 {% block scripts %}
-<script src="/static/js/vendor/tabs.js"></script>
+<script src="{% static "js/vendor/tabs.js" %}"></script>
 {% endblock %}

bookwyrm/templates/get_started/layout.html

@@ -1,5 +1,6 @@
 {% extends 'layout.html' %}
 {% load i18n %}
+{% load static %}
 
 {% block title %}{% trans "Welcome" %}{% endblock %}
 
@@ -9,7 +10,7 @@
     <div class="modal-background"></div>
     <div class="modal-card is-fullwidth">
         <header class="modal-card-head">
-            <img class="image logo mr-2" src="{% if site.logo_small %}/images/{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" aria-hidden="true">
+            <img class="image logo mr-2" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" aria-hidden="true">
             <h1 class="modal-card-title" id="get-started-header">
                 {% blocktrans %}Welcome to {{ site_name }}!{% endblocktrans %}
                 <span class="subtitle is-block">

bookwyrm/templates/import_status.html

@@ -1,6 +1,7 @@
 {% extends 'layout.html' %}
 {% load i18n %}
 {% load humanize %}
+{% load static %}
 
 {% block title %}{% trans "Import Status" %}{% endblock %}
 
@@ -156,5 +157,5 @@
 {% endspaceless %}{% endblock %}
 
 {% block scripts %}
-<script src="/static/js/check_all.js"></script>
+<script src="{% static "js/check_all.js" %}"></script>
 {% endblock %}

bookwyrm/templates/layout.html

@@ -1,14 +1,16 @@
-{% load layout %}{% load i18n %}
+{% load layout %}
+{% load i18n %}
+{% load static %}
 <!DOCTYPE html>
 <html lang="{% get_lang %}">
 <head>
     <title>{% block title %}BookWyrm{% endblock %} | {{ site.name }}</title>
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="stylesheet" href="/static/css/vendor/bulma.min.css">
+    <link rel="stylesheet" href="{% static "css/vendor/bulma.min.css" %}">
-    <link rel="stylesheet" href="/static/css/vendor/icons.css">
+    <link rel="stylesheet" href="{% static "css/vendor/icons.css" %}">
-    <link rel="stylesheet" href="/static/css/bookwyrm.css">
+    <link rel="stylesheet" href="{% static "css/bookwyrm.css" %}">
 
-    <link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{{ media_url }}{{ site.favicon }}{% else %}/static/images/favicon.ico{% endif %}">
+    <link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{% get_media_prefix %}{{ site.favicon }}{% else %}{% static "images/favicon.ico" %}{% endif %}">
 
     {% if preview_images_enabled is True %}
     <meta name="twitter:card" content="summary_large_image">
@@ -30,7 +32,7 @@
     <div class="container">
         <div class="navbar-brand">
             <a class="navbar-item" href="/">
-                <img class="image logo" src="{% if site.logo_small %}{{ media_url }}{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" alt="Home page">
+                <img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="Home page">
             </a>
             <form class="navbar-item column" action="/search/">
                 <div class="field has-addons">
@@ -242,8 +244,8 @@
 <script>
     var csrf_token = '{{ csrf_token }}';
 </script>
-<script src="/static/js/bookwyrm.js"></script>
+<script src="{% static "js/bookwyrm.js" %}"></script>
-<script src="/static/js/localstorage.js"></script>
+<script src="{% static "js/localstorage.js" %}"></script>
 {% block scripts %}{% endblock %}
 </body>
 </html>

bookwyrm/templates/snippets/about.html

@@ -1,7 +1,9 @@
+{% load static %}
+
 <div class="columns">
     <div class="column is-narrow is-hidden-mobile">
         <figure class="block is-w-xl">
-            <img src="{% if site.logo %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}" alt="BookWyrm logo">
+            <img src="{% if site.logo %}/images/{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}" alt="BookWyrm logo">
         </figure>
     </div>
     <div class="content">

bookwyrm/templates/snippets/avatar.html

@@ -1,2 +1,4 @@
-<img class="avatar image {% if large %}is-96x96{% elif medium %}is-48x48{% else %}is-32x32{% endif %}" src="{% if user.avatar %}/images/{{ user.avatar }}{% else %}/static/images/default_avi.jpg{% endif %}" {% if ariaHide %}aria-hidden="true"{% endif %} alt="{{ user.alt_text }}">
+{% load static %}
+
+<img class="avatar image {% if large %}is-96x96{% elif medium %}is-48x48{% else %}is-32x32{% endif %}" src="{% if user.avatar %}{% get_media_prefix %}{{ user.avatar }}{% else %}{% static "images/default_avi.jpg" %}{% endif %}" {% if ariaHide %}aria-hidden="true"{% endif %} alt="{{ user.alt_text }}">
 

bookwyrm/templates/snippets/book_cover.html

@@ -1,6 +1,7 @@
 {% spaceless %}
 
 {% load i18n %}
+{% load static %}
 
 <figure
     class="
@@ -20,14 +21,14 @@
         class="book-cover"
 
         {% if book.cover %}
-            src="{% if img_path is None %}/images/{% else %}{{ img_path }}{% endif %}{{ book.cover }}"
+            src="{% if img_path is None %}{% get_media_prefix %}{% else %}{{ img_path }}{% endif %}{{ book.cover }}"
             itemprop="thumbnailUrl"
 
             {% if book.alt_text %}
                 alt="{{ book.alt_text }}"
             {% endif %}
         {% else %}
-            src="/static/images/no_cover.jpg"
+            src="{% static "images/no_cover.jpg" %}"
             alt="{% trans "No cover" %}"
         {% endif %}
     >

bookwyrm/templates/snippets/opengraph_images.html

@@ -1,12 +1,14 @@
+{% load static %}
+
 {% if preview_images_enabled is True %}
     {% if image %}
-        <meta name="twitter:image" content="{{ request.scheme }}://{{ media_path }}{{ image }}">
+        <meta name="twitter:image" content="{{ media_full_url }}{{ image }}">
-        <meta name="og:image" content="{{ request.scheme }}://{{ media_path }}{{ image }}">
+        <meta name="og:image" content="{{ media_full_url }}{{ image }}">
     {% else %}
-        <meta name="twitter:image" content="{{ request.scheme }}://{{ media_path }}{{ site.preview_image }}">
+        <meta name="twitter:image" content="{{ media_full_url }}{{ site.preview_image }}">
-        <meta name="og:image" content="{{ request.scheme }}://{{ media_path }}{{ site.preview_image }}">
+        <meta name="og:image" content="{{ media_full_url }}{{ site.preview_image }}">
     {% endif %}
 {% else %}
-    <meta name="twitter:image" content="{{ request.scheme }}://{% if site.logo %}{{ media_path }}{{ site.logo }}{% else %}{{ static_path }}/images/logo.png{% endif %}">
+    <meta name="twitter:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
-    <meta name="og:image" content="{{ request.scheme }}://{% if site.logo %}{{ media_path }}{{ site.logo }}{% else %}{{ static_path }}/images/logo.png{% endif %}">
+    <meta name="og:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
 {% endif %}

bookwyrm/templates/snippets/status/content_status.html

@@ -1,6 +1,7 @@
 {% load bookwyrm_tags %}
 {% load markdown %}
 {% load i18n %}
+{% load static %}
 
 {% with status_type=status.status_type %}
 <div
@@ -111,12 +112,12 @@
                                 <div class="column is-narrow">
                                     <figure class="image is-128x128">
                                         <a
-                                            href="/images/{{ attachment.image }}"
+                                            href="{% get_media_prefix %}{{ attachment.image }}"
                                             target="_blank"
                                             aria-label="{% trans 'Open image in new window' %}"
                                         >
                                             <img
-                                                src="/images/{{ attachment.image }}"
+                                                src="{% get_media_prefix %}{{ attachment.image }}"
 
                                                 {% if attachment.caption %}
                                                     alt="{{ attachment.caption }}"

bookwyrm/templates/snippets/status/status_header.html

@@ -2,6 +2,7 @@
 {% load status_display %}
 {% load i18n %}
 {% load humanize %}
+{% load static %}
 
 <div class="media">
     <figure class="media-left" aria-hidden="true">
@@ -18,7 +19,7 @@
                 itemtype="https://schema.org/Person"
             >
                 {% if status.user.avatar %}
-                    <meta itemprop="image" content="/images/{{ status.user.avatar }}">
+                    <meta itemprop="image" content="{% get_media_prefix %}{{ status.user.avatar }}">
                 {% endif %}
 
                 <a

bw-dev

@@ -38,6 +38,17 @@ function makeitblack {
     docker-compose run --rm web black celerywyrm bookwyrm
 }
 
+function awscommand {
+    # expose env vars
+    export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+    export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+    export AWS_DEFAULT_REGION=${AWS_S3_REGION_NAME}
+    # first arg is mountpoint, second is the whole aws command
+    docker run --rm -it -v $1\
+        -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION\
+        amazon/aws-cli $2
+}
+
 CMD=$1
 if [ -n "$CMD" ]; then
     shift
@@ -115,6 +126,19 @@ case "$CMD" in
     generate_preview_images)
         runweb python manage.py generate_preview_images $@
         ;;
+    copy_media_to_s3)
+        awscommand "bookwyrm_media_volume:/images"\
+            "s3 cp /images s3://${AWS_STORAGE_BUCKET_NAME}/images\
+            --endpoint-url ${AWS_S3_ENDPOINT_URL}\
+            --recursive --acl public-read"
+        ;;
+    set_cors_to_s3)
+        awscommand "$(pwd):/bw"\
+            "s3api put-bucket-cors\
+            --bucket ${AWS_STORAGE_BUCKET_NAME}\
+            --endpoint-url ${AWS_S3_ENDPOINT_URL}\
+            --cors-configuration file:///bw/$@"
+        ;;
     runweb)
         runweb "$@"
         ;;
@@ -123,6 +147,30 @@ case "$CMD" in
         ;;
     *)
         set +x # No need to echo echo
-        echo "Unrecognised command. Try: build, clean, up, initdb, resetdb, makemigrations, migrate, bash, shell, dbshell, restart_celery, test, pytest, test_report, black, populate_streams, generate_preview_images"
+        echo "Unrecognised command. Try:"
+        echo "    up [container]"
+        echo "    run"
+        echo "    initdb"
+        echo "    resetdb"
+        echo "    makemigrations [migration]"
+        echo "    migrate [migration]"
+        echo "    bash"
+        echo "    shell"
+        echo "    dbshell"
+        echo "    restart_celery"
+        echo "    test [path]"
+        echo "    pytest [path]"
+        echo "    collectstatic"
+        echo "    makemessages [locale]"
+        echo "    compilemessages [locale]"
+        echo "    build"
+        echo "    clean"
+        echo "    black"
+        echo "    populate_streams"
+        echo "    generate_preview_images [--all]"
+        echo "    copy_media_to_s3"
+        echo "    set_cors_to_s3 [cors file]"
+        echo "    runweb [command]"
+        echo "    rundb [command]"
         ;;
 esac

requirements.txt

@@ -14,6 +14,8 @@ requests==2.22.0
 responses==0.10.14
 django-rename-app==0.1.2
 pytz>=2021.1
+boto3==1.17.88
+django-storages==1.11.1
 
 # Dev
 black==21.4b0