mirrors/bookwyrm
1
0
Fork 1
mirror of https://github.com/bookwyrm-social/bookwyrm.git synced 2024-06-10 09:19:27 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1163 from joachimesque/storage-s3

Browse source 
Use S3 as static and media storage
This commit is contained in:
Mouse Reeve 2021-08-02 08:29:09 -06:00 committed by GitHub
parent d76106ded5 d70469d2e0
commit b40b8c2d0e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 196 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
.env.dev.example
View file

@ -3,6 +3,7 @@ SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG=true
USE_HTTPS=false
DOMAIN=your.domain.here
#EMAIL=your@email.here
@ -42,6 +43,21 @@ EMAIL_HOST_PASSWORD=emailpassword123
EMAIL_USE_TLS=true
EMAIL_USE_SSL=false
# S3 configuration
USE_S3=false
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
# Commented are example values if you use a non-AWS, S3-compatible service
# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
# AWS_STORAGE_BUCKET_NAME= # "example-bucket-name"
# AWS_S3_CUSTOM_DOMAIN=None # "example-bucket-name.s3.fr-par.scw.cloud"
# AWS_S3_REGION_NAME=None # "fr-par"
# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud"
# Preview image generation can be computing and storage intensive
# ENABLE_PREVIEW_IMAGES=True

16
.env.prod.example
View file

@ -3,6 +3,7 @@ SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG=false
USE_HTTPS=true
DOMAIN=your.domain.here
EMAIL=your@email.here
@ -42,6 +43,21 @@ EMAIL_HOST_PASSWORD=emailpassword123
EMAIL_USE_TLS=true
EMAIL_USE_SSL=false
# S3 configuration
USE_S3=false
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
# Commented are example values if you use a non-AWS, S3-compatible service
# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
# AWS_STORAGE_BUCKET_NAME= # "example-bucket-name"
# AWS_S3_CUSTOM_DOMAIN=None # "example-bucket-name.s3.fr-par.scw.cloud"
# AWS_S3_REGION_NAME=None # "fr-par"
# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud"
# Preview image generation can be computing and storage intensive
# ENABLE_PREVIEW_IMAGES=True

5
bookwyrm/context_processors.py
View file

@ -11,10 +11,7 @@ def site_settings(request): # pylint: disable=unused-argument
return {
"site": models.SiteSettings.objects.get(),
"active_announcements": models.Announcement.active_announcements(),
"static_url": settings.STATIC_URL,
"media_url": settings.MEDIA_URL,
"static_path": settings.STATIC_PATH,
"media_path": settings.MEDIA_PATH,
"media_full_url": settings.MEDIA_FULL_URL,
"preview_images_enabled": settings.ENABLE_PREVIEW_IMAGES,
"request_protocol": request_protocol,
}

9
bookwyrm/preview_images.py
View file

@ -11,6 +11,7 @@ from PIL import Image, ImageDraw, ImageFont, ImageOps, ImageColor
from django.core.files.base import ContentFile
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.core.files.storage import default_storage
from django.db.models import Avg
from bookwyrm import models, settings
@ -319,9 +320,9 @@ def save_and_cleanup(image, instance=None):
try:
try:
old_path = instance.preview_image.path
old_path = instance.preview_image.name
except ValueError:
old_path = ""
old_path = None
# Save
image.save(image_buffer, format="jpeg", quality=75)
@ -342,8 +343,8 @@ def save_and_cleanup(image, instance=None):
instance.save()
# Clean up old file after saving
if os.path.exists(old_path):
os.remove(old_path)
if old_path and default_storage.exists(old_path):
default_storage.delete(old_path)
finally:
image_buffer.close()

56
bookwyrm/settings.py
View file

@ -58,6 +58,7 @@ SECRET_KEY = env("SECRET_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = env.bool("DEBUG", True)
USE_HTTPS = env.bool("USE_HTTPS", False)
ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", ["*"])
@ -74,6 +75,7 @@ INSTALLED_APPS = [
"django_rename_app",
"bookwyrm",
"celery",
"storages",
]
MIDDLEWARE = [
@ -179,19 +181,51 @@ USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.2/howto/static-files/
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
STATIC_URL = "/static/"
STATIC_PATH = "%s/%s" % (DOMAIN, env("STATIC_ROOT", "static"))
STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
MEDIA_URL = "/images/"
MEDIA_PATH = "%s/%s" % (DOMAIN, env("MEDIA_ROOT", "images"))
MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % (
requests.utils.default_user_agent(),
VERSION,
DOMAIN,
)
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.2/howto/static-files/
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
# Storage
PROTOCOL = "http"
if USE_HTTPS:
PROTOCOL = "https"
USE_S3 = env.bool("USE_S3", False)
if USE_S3:
# AWS settings
AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN")
AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME", "")
AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL")
AWS_DEFAULT_ACL = "public-read"
AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"}
# S3 Static settings
STATIC_LOCATION = "static"
STATIC_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, STATIC_LOCATION)
STATICFILES_STORAGE = "bookwyrm.storage_backends.StaticStorage"
# S3 Media settings
MEDIA_LOCATION = "images"
MEDIA_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, MEDIA_LOCATION)
MEDIA_FULL_URL = MEDIA_URL
DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
# I don't know if it's used, but the site crashes without it
STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
else:
STATIC_URL = "/static/"
STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
MEDIA_URL = "/images/"
MEDIA_FULL_URL = "%s://%s%s" % (PROTOCOL, DOMAIN, MEDIA_URL)
MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))

17
bookwyrm/storage_backends.py Normal file
View file

@ -0,0 +1,17 @@
"""Handles backends for storages"""
from storages.backends.s3boto3 import S3Boto3Storage
class StaticStorage(S3Boto3Storage): # pylint: disable=abstract-method
"""Storage class for Static contents"""
location = "static"
default_acl = "public-read"
class ImagesStorage(S3Boto3Storage): # pylint: disable=abstract-method
"""Storage class for Image files"""
location = "images"
default_acl = "public-read"
file_overwrite = False

9
bookwyrm/templates/book/book.html
View file

@ -1,5 +1,10 @@
{% extends 'layout.html' %}
{% load i18n %}{% load bookwyrm_tags %}{% load humanize %}{% load utilities %}{% load layout %}
{% load i18n %}
{% load bookwyrm_tags %}
{% load humanize %}
{% load utilities %}
{% load static %}
{% load layout %}
{% block title %}{{ book|book_title }}{% endblock %}
@ -321,5 +326,5 @@
{% endblock %}
{% block scripts %}
<script src="/static/js/vendor/tabs.js"></script>
<script src="{% static "js/vendor/tabs.js" %}"></script>
{% endblock %}

3
bookwyrm/templates/feed/layout.html
View file

@ -1,5 +1,6 @@
{% extends 'layout.html' %}
{% load i18n %}
{% load static %}
{% block title %}{% trans "Updates" %}{% endblock %}
@ -105,5 +106,5 @@
{% endblock %}
{% block scripts %}
<script src="/static/js/vendor/tabs.js"></script>
<script src="{% static "js/vendor/tabs.js" %}"></script>
{% endblock %}

3
bookwyrm/templates/get_started/layout.html
View file

@ -1,5 +1,6 @@
{% extends 'layout.html' %}
{% load i18n %}
{% load static %}
{% block title %}{% trans "Welcome" %}{% endblock %}
@ -9,7 +10,7 @@
<div class="modal-background"></div>
<div class="modal-card is-fullwidth">
<header class="modal-card-head">
<img class="image logo mr-2" src="{% if site.logo_small %}/images/{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" aria-hidden="true">
<img class="image logo mr-2" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" aria-hidden="true">
<h1 class="modal-card-title" id="get-started-header">
{% blocktrans %}Welcome to {{ site_name }}!{% endblocktrans %}
<span class="subtitle is-block">

3
bookwyrm/templates/import_status.html
View file

@ -1,6 +1,7 @@
{% extends 'layout.html' %}
{% load i18n %}
{% load humanize %}
{% load static %}
{% block title %}{% trans "Import Status" %}{% endblock %}
@ -156,5 +157,5 @@
{% endspaceless %}{% endblock %}
{% block scripts %}
<script src="/static/js/check_all.js"></script>
<script src="{% static "js/check_all.js" %}"></script>
{% endblock %}

18
bookwyrm/templates/layout.html
View file

@ -1,14 +1,16 @@
{% load layout %}{% load i18n %}
{% load layout %}
{% load i18n %}
{% load static %}
<!DOCTYPE html>
<html lang="{% get_lang %}">
<head>
<title>{% block title %}BookWyrm{% endblock %} | {{ site.name }}</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="/static/css/vendor/bulma.min.css">
<link rel="stylesheet" href="/static/css/vendor/icons.css">
<link rel="stylesheet" href="/static/css/bookwyrm.css">
<link rel="stylesheet" href="{% static "css/vendor/bulma.min.css" %}">
<link rel="stylesheet" href="{% static "css/vendor/icons.css" %}">
<link rel="stylesheet" href="{% static "css/bookwyrm.css" %}">
<link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{{ media_url }}{{ site.favicon }}{% else %}/static/images/favicon.ico{% endif %}">
<link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{% get_media_prefix %}{{ site.favicon }}{% else %}{% static "images/favicon.ico" %}{% endif %}">
{% if preview_images_enabled is True %}
<meta name="twitter:card" content="summary_large_image">
@ -30,7 +32,7 @@
<div class="container">
<div class="navbar-brand">
<a class="navbar-item" href="/">
<img class="image logo" src="{% if site.logo_small %}{{ media_url }}{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" alt="Home page">
<img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="Home page">
</a>
<form class="navbar-item column" action="/search/">
<div class="field has-addons">
@ -242,8 +244,8 @@
<script>
var csrf_token = '{{ csrf_token }}';
</script>
<script src="/static/js/bookwyrm.js"></script>
<script src="/static/js/localstorage.js"></script>
<script src="{% static "js/bookwyrm.js" %}"></script>
<script src="{% static "js/localstorage.js" %}"></script>
{% block scripts %}{% endblock %}
</body>
</html>

4
bookwyrm/templates/snippets/about.html
View file

@ -1,7 +1,9 @@
{% load static %}
<div class="columns">
<div class="column is-narrow is-hidden-mobile">
<figure class="block is-w-xl">
<img src="{% if site.logo %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}" alt="BookWyrm logo">
<img src="{% if site.logo %}/images/{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}" alt="BookWyrm logo">
</figure>
</div>
<div class="content">

4
bookwyrm/templates/snippets/avatar.html
View file

@ -1,2 +1,4 @@
<img class="avatar image {% if large %}is-96x96{% elif medium %}is-48x48{% else %}is-32x32{% endif %}" src="{% if user.avatar %}/images/{{ user.avatar }}{% else %}/static/images/default_avi.jpg{% endif %}" {% if ariaHide %}aria-hidden="true"{% endif %} alt="{{ user.alt_text }}">
{% load static %}
<img class="avatar image {% if large %}is-96x96{% elif medium %}is-48x48{% else %}is-32x32{% endif %}" src="{% if user.avatar %}{% get_media_prefix %}{{ user.avatar }}{% else %}{% static "images/default_avi.jpg" %}{% endif %}" {% if ariaHide %}aria-hidden="true"{% endif %} alt="{{ user.alt_text }}">

5
bookwyrm/templates/snippets/book_cover.html
View file

@ -1,6 +1,7 @@
{% spaceless %}
{% load i18n %}
{% load static %}
<figure
class="
@ -20,14 +21,14 @@
class="book-cover"
{% if book.cover %}
src="{% if img_path is None %}/images/{% else %}{{ img_path }}{% endif %}{{ book.cover }}"
src="{% if img_path is None %}{% get_media_prefix %}{% else %}{{ img_path }}{% endif %}{{ book.cover }}"
itemprop="thumbnailUrl"
{% if book.alt_text %}
alt="{{ book.alt_text }}"
{% endif %}
{% else %}
src="/static/images/no_cover.jpg"
src="{% static "images/no_cover.jpg" %}"
alt="{% trans "No cover" %}"
{% endif %}
>

14
bookwyrm/templates/snippets/opengraph_images.html
View file

@ -1,12 +1,14 @@
{% load static %}
{% if preview_images_enabled is True %}
{% if image %}
<meta name="twitter:image" content="{{ request.scheme }}://{{ media_path }}{{ image }}">
<meta name="og:image" content="{{ request.scheme }}://{{ media_path }}{{ image }}">
<meta name="twitter:image" content="{{ media_full_url }}{{ image }}">
<meta name="og:image" content="{{ media_full_url }}{{ image }}">
{% else %}
<meta name="twitter:image" content="{{ request.scheme }}://{{ media_path }}{{ site.preview_image }}">
<meta name="og:image" content="{{ request.scheme }}://{{ media_path }}{{ site.preview_image }}">
<meta name="twitter:image" content="{{ media_full_url }}{{ site.preview_image }}">
<meta name="og:image" content="{{ media_full_url }}{{ site.preview_image }}">
{% endif %}
{% else %}
<meta name="twitter:image" content="{{ request.scheme }}://{% if site.logo %}{{ media_path }}{{ site.logo }}{% else %}{{ static_path }}/images/logo.png{% endif %}">
<meta name="og:image" content="{{ request.scheme }}://{% if site.logo %}{{ media_path }}{{ site.logo }}{% else %}{{ static_path }}/images/logo.png{% endif %}">
<meta name="twitter:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
<meta name="og:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
{% endif %}

5
bookwyrm/templates/snippets/status/content_status.html
View file

@ -1,6 +1,7 @@
{% load bookwyrm_tags %}
{% load markdown %}
{% load i18n %}
{% load static %}
{% with status_type=status.status_type %}
<div
@ -111,12 +112,12 @@
<div class="column is-narrow">
<figure class="image is-128x128">
<a
href="/images/{{ attachment.image }}"
href="{% get_media_prefix %}{{ attachment.image }}"
target="_blank"
aria-label="{% trans 'Open image in new window' %}"
>
<img
src="/images/{{ attachment.image }}"
src="{% get_media_prefix %}{{ attachment.image }}"
{% if attachment.caption %}
alt="{{ attachment.caption }}"

3
bookwyrm/templates/snippets/status/status_header.html
View file

@ -2,6 +2,7 @@
{% load status_display %}
{% load i18n %}
{% load humanize %}
{% load static %}
<div class="media">
<figure class="media-left" aria-hidden="true">
@ -18,7 +19,7 @@
itemtype="https://schema.org/Person"
>
{% if status.user.avatar %}
<meta itemprop="image" content="/images/{{ status.user.avatar }}">
<meta itemprop="image" content="{% get_media_prefix %}{{ status.user.avatar }}">
{% endif %}
<a

50
bw-dev
View file

@ -38,6 +38,17 @@ function makeitblack {
docker-compose run --rm web black celerywyrm bookwyrm
}
function awscommand {
# expose env vars
export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
export AWS_DEFAULT_REGION=${AWS_S3_REGION_NAME}
# first arg is mountpoint, second is the whole aws command
docker run --rm -it -v $1\
-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION\
amazon/aws-cli $2
}
CMD=$1
if [ -n "$CMD" ]; then
shift
@ -115,6 +126,19 @@ case "$CMD" in
generate_preview_images)
runweb python manage.py generate_preview_images $@
;;
copy_media_to_s3)
awscommand "bookwyrm_media_volume:/images"\
"s3 cp /images s3://${AWS_STORAGE_BUCKET_NAME}/images\
--endpoint-url ${AWS_S3_ENDPOINT_URL}\
--recursive --acl public-read"
;;
set_cors_to_s3)
awscommand "$(pwd):/bw"\
"s3api put-bucket-cors\
--bucket ${AWS_STORAGE_BUCKET_NAME}\
--endpoint-url ${AWS_S3_ENDPOINT_URL}\
--cors-configuration file:///bw/$@"
;;
runweb)
runweb "$@"
;;
@ -123,6 +147,30 @@ case "$CMD" in
;;
*)
set +x # No need to echo echo
echo "Unrecognised command. Try: build, clean, up, initdb, resetdb, makemigrations, migrate, bash, shell, dbshell, restart_celery, test, pytest, test_report, black, populate_streams, generate_preview_images"
echo "Unrecognised command. Try:"
echo " up [container]"
echo " run"
echo " initdb"
echo " resetdb"
echo " makemigrations [migration]"
echo " migrate [migration]"
echo " bash"
echo " shell"
echo " dbshell"
echo " restart_celery"
echo " test [path]"
echo " pytest [path]"
echo " collectstatic"
echo " makemessages [locale]"
echo " compilemessages [locale]"
echo " build"
echo " clean"
echo " black"
echo " populate_streams"
echo " generate_preview_images [--all]"
echo " copy_media_to_s3"
echo " set_cors_to_s3 [cors file]"
echo " runweb [command]"
echo " rundb [command]"
;;
esac

2
requirements.txt
View file

@ -14,6 +14,8 @@ requests==2.22.0
responses==0.10.14
django-rename-app==0.1.2
pytz>=2021.1
boto3==1.17.88
django-storages==1.11.1
# Dev
black==21.4b0