From 80193114909a3f6ca1eda9a47b6330ef249a8ee5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 18 Nov 2022 17:24:43 -0700 Subject: [PATCH] Deployment re-jiggling --- Makefile | 3 ++ core/models/config.py | 2 + docker/Dockerfile | 28 ++++--------- docker/start.sh | 6 +-- docs/installation.rst | 46 ++++++++++++++++++++- requirements.txt | 1 + static/css/style.css | 4 ++ stator/runner.py | 6 +-- stator/views.py | 12 ++++-- takahe/settings/base.py | 49 +++------------------- takahe/settings/development.py | 6 +++ takahe/settings/production.py | 75 +++++++++++++++++++++++++++++++--- templates/identity/view.html | 4 +- users/models/password_reset.py | 4 +- 14 files changed, 162 insertions(+), 84 deletions(-) diff --git a/Makefile b/Makefile index b87b2ce..c38f867 100644 --- a/Makefile +++ b/Makefile @@ -2,3 +2,6 @@ image: docker build -t takahe -f docker/Dockerfile . + +docs: + cd docs/ && make html diff --git a/core/models/config.py b/core/models/config.py index 57d9e55..5d2fdfb 100644 --- a/core/models/config.py +++ b/core/models/config.py @@ -168,6 +168,8 @@ class Config(models.Model): identity_max_per_user: int = 5 identity_max_age: int = 24 * 60 * 60 + restricted_usernames: str = "admin\nadmins\nadministrator\nadministrators\nsystem\nroot\nannounce\nannouncement\nannouncements" + class UserOptions(pydantic.BaseModel): pass diff --git a/docker/Dockerfile b/docker/Dockerfile index 14e033b..a35f1ff 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,29 +1,19 @@ -# Build stage +FROM python:3.11.0-slim-buster -FROM python:3.11.0-buster as builder - -RUN mkdir -p /takahe -RUN python -m venv /takahe/.venv -RUN apt-get update && apt-get -y install libpq-dev python3-dev - -WORKDIR /takahe +RUN apt-get update && apt-get -y install libpq-dev python3-dev build-essential COPY requirements.txt requirements.txt -RUN . /takahe/.venv/bin/activate \ - && pip install --upgrade pip \ - && pip install --upgrade -r requirements.txt +RUN pip3 install --upgrade pip \ + && pip3 install --upgrade -r requirements.txt -# Final image stage - -FROM python:3.11.0-slim-buster - -RUN apt-get update && apt-get install -y libpq5 - -COPY --from=builder /takahe /takahe COPY . /takahe WORKDIR /takahe + +# We use development here to skip settings checks +RUN DJANGO_SETTINGS_MODULE=takahe.settings.development python3 manage.py collectstatic + EXPOSE 8000 -CMD ["/takahe/docker/start.sh"] +CMD ["sh", "/takahe/docker/start.sh"] diff --git a/docker/start.sh b/docker/start.sh index 99f1ed0..1c01b6e 100644 --- a/docker/start.sh +++ b/docker/start.sh @@ -1,7 +1,5 @@ #!/bin/sh -. /takahe/.venv/bin/activate +python3 manage.py migrate -python manage.py migrate - -exec gunicorn takahe.asgi:application -k uvicorn.workers.UvicornWorker -b 0.0.0.0:8000 +exec gunicorn takahe.wsgi:application -w 8 -b 0.0.0.0:8000 diff --git a/docs/installation.rst b/docs/installation.rst index 9c39a9d..3e11f9c 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -29,6 +29,9 @@ be provided from the first boot. * ``PGHOST``, ``PGPORT``, ``PGUSER``, ``PGDATABASE``, and ``PGPASSWORD`` are the standard PostgreSQL environment variables for configuring your database. +* ``TAKAHE_SECRET_KEY`` must be a fixed, random value (it's used for internal + cryptography). Don't change this unless you want to invalidate all sessions. + * ``TAKAHE_MEDIA_BACKEND`` must be one of ``local``, ``s3`` or ``gcs``. * If it is set to ``local``, you must also provide ``TAKAHE_MEDIA_ROOT``, @@ -36,7 +39,8 @@ be provided from the first boot. fully-qualified URL prefix that serves that directory. * If it is set to ``gcs``, you must also provide ``TAKAHE_MEDIA_BUCKET``, - the name of the bucket to store files in. + the name of the bucket to store files in. The bucket must be publically + readable and have "uniform access control" enabled. * If it is set to ``s3``, you must also provide ``TAKAHE_MEDIA_BUCKET``, the name of the bucket to store files in. @@ -60,6 +64,36 @@ be provided from the first boot. be automatically promoted to administrator when it signs up. You only need this for initial setup, and can unset it after that if you like. +* ``TAKAHE_STATOR_TOKEN`` should be a random string that you are using to + protect the stator (task runner) endpoint. You'll use this value later. + +* If your installation is behind a HTTPS endpoint that is proxying it, set + ``TAKAHE_SECURE_HEADER`` to the header name used to signify that HTTPS is + being used (usually ``X-Forwarded-Proto``) + +* If you want to receive emails about internal site errors, set + ``TAKAHE_ERROR_EMAILS`` to a comma-separated list of email addresses that + should get them. + + +Setting Up Task Runners +----------------------- + +Takahe is designed to not require a continuously-running background worker; +instead, you can trigger the "Stator Runner" (our internal task system) either +via a periodic admin command or via a periodic hit to a URL (which is useful +if you are on "serverless" hosting that does not allow background tasks). + +To use the URL method, configure something to hit +``/.stator/runner/?token=ABCDEF`` every 60 seconds. You can do this less often +if you don't mind delays in content and profiles being fetched, or more often +if you are under increased load. The value of the token should be the same +as what you set for ``TAKAHE_STATOR_TOKEN``. + +Alternatively, you can set up ``python manage.py runstator`` to run in the +Docker image with the same time interval. We still recommend setting +``TAKAHE_STATOR_TOKEN`` in this case so nobody else can trigger it from a URL. + Making An Admin Account ----------------------- @@ -74,3 +108,13 @@ admin account. If your email settings have a problem and you don't get the email, don't worry; fix them and then follow the "reset my password" flow on the login screen, and you'll get another password reset email that you can use. + + +Adding A Domain +--------------- + +When you login you'll be greeted with the "make an identity" screen, but you +won't be able to as you will have no domains yet. + +You should navigate directly to ``/admin/domains/`` and make one, and then +you will be able to create an identity. diff --git a/requirements.txt b/requirements.txt index 3b0cb1c..b0eafb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ bleach~=5.0.1 pydantic~=1.10.2 django-htmx~=1.13.0 django-storages[google,boto3]~=1.13.1 +whitenoise~=6.2.0 diff --git a/static/css/style.css b/static/css/style.css index 5f35cc2..571e812 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -646,6 +646,10 @@ h1.identity small { margin: -10px 0 0 0; } +.bio { + margin: 0 0 20px 0; +} + .system-note { background: var(--color-bg-menu); color: var(--color-text-dull); diff --git a/stator/runner.py b/stator/runner.py index 187aa47..bb1b009 100644 --- a/stator/runner.py +++ b/stator/runner.py @@ -19,9 +19,9 @@ class StatorRunner: def __init__( self, models: List[Type[StatorModel]], - concurrency: int = 30, - concurrency_per_model: int = 5, - run_period: int = 30, + concurrency: int = 50, + concurrency_per_model: int = 10, + run_period: int = 60, wait_period: int = 30, ): self.models = models diff --git a/stator/views.py b/stator/views.py index ef09b8e..9d2e154 100644 --- a/stator/views.py +++ b/stator/views.py @@ -1,8 +1,9 @@ -from django.http import HttpResponse +from django.conf import settings +from django.http import HttpResponse, HttpResponseForbidden from django.views import View +from stator.models import StatorModel from stator.runner import StatorRunner -from users.models import Follow class RequestRunner(View): @@ -12,6 +13,11 @@ class RequestRunner(View): """ async def get(self, request): - runner = StatorRunner([Follow]) + # Check the token, if supplied + if settings.STATOR_TOKEN: + if request.GET.get("token") != settings.STATOR_TOKEN: + return HttpResponseForbidden() + # Run on all models + runner = StatorRunner(StatorModel.subclasses) handled = await runner.run() return HttpResponse(f"Handled {handled}") diff --git a/takahe/settings/base.py b/takahe/settings/base.py index d2e30c3..719e03b 100644 --- a/takahe/settings/base.py +++ b/takahe/settings/base.py @@ -1,5 +1,4 @@ import os -import sys from pathlib import Path from typing import Optional @@ -23,6 +22,7 @@ INSTALLED_APPS = [ MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", + "whitenoise.middleware.WhiteNoiseMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", @@ -109,49 +109,10 @@ STATICFILES_DIRS = [ BASE_DIR / "static", ] +STATIC_ROOT = BASE_DIR / "static-collected" + ALLOWED_HOSTS = ["*"] -### User-configurable options, pulled from the environment ### +AUTO_ADMIN_EMAIL: Optional[str] = None -MAIN_DOMAIN = os.environ["TAKAHE_MAIN_DOMAIN"] -if "/" in MAIN_DOMAIN: - print("TAKAHE_MAIN_DOMAIN should be just the domain name - no https:// or path") - sys.exit(1) - - -if os.environ.get("TAKAHE_EMAIL_CONSOLE_ONLY"): - EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" - EMAIL_FROM = "test@example.com" -else: - EMAIL_FROM = os.environ["TAKAHE_EMAIL_FROM"] - if "TAKAHE_EMAIL_SENDGRID_KEY" in os.environ: - EMAIL_HOST = "smtp.sendgrid.net" - EMAIL_PORT = 587 - EMAIL_HOST_USER: Optional[str] = "apikey" - EMAIL_HOST_PASSWORD: Optional[str] = os.environ["TAKAHE_EMAIL_SENDGRID_KEY"] - EMAIL_USE_TLS = True - else: - EMAIL_HOST = os.environ["TAKAHE_EMAIL_HOST"] - EMAIL_PORT = int(os.environ["TAKAHE_EMAIL_PORT"]) - EMAIL_HOST_USER = os.environ.get("TAKAHE_EMAIL_USER") - EMAIL_HOST_PASSWORD = os.environ.get("TAKAHE_EMAIL_PASSWORD") - EMAIL_USE_SSL = EMAIL_PORT == 465 - EMAIL_USE_TLS = EMAIL_PORT == 587 - -AUTO_ADMIN_EMAIL = os.environ.get("TAKAHE_AUTO_ADMIN_EMAIL") - -# Set up media storage -MEDIA_BACKEND = os.environ.get("TAKAHE_MEDIA_BACKEND", None) -if MEDIA_BACKEND == "local": - # Note that this MUST be a fully qualified URL in production - MEDIA_URL = os.environ.get("TAKAHE_MEDIA_URL", "/media/") - MEDIA_ROOT = os.environ.get("TAKAHE_MEDIA_ROOT", BASE_DIR / "media") -elif MEDIA_BACKEND == "gcs": - DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" - GS_BUCKET_NAME = os.environ["TAKAHE_MEDIA_BUCKET"] -elif MEDIA_BACKEND == "s3": - DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" - AWS_STORAGE_BUCKET_NAME = os.environ["TAKAHE_MEDIA_BUCKET"] -else: - print("Unknown TAKAHE_MEDIA_BACKEND value") - sys.exit(1) +STATOR_TOKEN: Optional[str] = None diff --git a/takahe/settings/development.py b/takahe/settings/development.py index 30f74a0..d71a406 100644 --- a/takahe/settings/development.py +++ b/takahe/settings/development.py @@ -18,3 +18,9 @@ CSRF_TRUSTED_ORIGINS = [ ] EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" +SERVER_EMAIL = "test@example.com" + +MAIN_DOMAIN = os.environ.get("TAKAHE_MAIN_DOMAIN", "https://example.com") + +MEDIA_URL = os.environ.get("TAKAHE_MEDIA_URL", "/media/") +MEDIA_ROOT = os.environ.get("TAKAHE_MEDIA_ROOT", BASE_DIR / "media") diff --git a/takahe/settings/production.py b/takahe/settings/production.py index f453177..34116af 100644 --- a/takahe/settings/production.py +++ b/takahe/settings/production.py @@ -1,16 +1,79 @@ import os +import sys +from typing import Optional from .base import * # noqa -# Load secret key from environment +# Ensure debug features are off +DEBUG = bool(os.environ.get("TAKAHE__SECURITY_HAZARD__DEBUG", False)) + +# TODO: Allow better setting of allowed_hosts, if we need to +ALLOWED_HOSTS = ["*"] + +### User-configurable options, pulled from the environment ### + +# Secret key try: SECRET_KEY = os.environ["TAKAHE_SECRET_KEY"] except KeyError: print("You must specify the TAKAHE_SECRET_KEY environment variable!") - os._exit(1) + sys.exit(1) -# Ensure debug features are off -DEBUG = False +# SSL proxy header +if "TAKAHE_SECURE_HEADER" in os.environ: + SECURE_PROXY_SSL_HEADER = ( + "HTTP_" + os.environ["TAKAHE_SECURE_HEADER"].replace("-", "_").upper(), + "https", + ) -# TODO: Allow better setting of allowed_hosts, if we need to -ALLOWED_HOSTS = ["*"] +# Fallback domain for links +MAIN_DOMAIN = os.environ["TAKAHE_MAIN_DOMAIN"] +if "/" in MAIN_DOMAIN: + print("TAKAHE_MAIN_DOMAIN should be just the domain name - no https:// or path") + sys.exit(1) + +# Email config +if os.environ.get("TAKAHE_EMAIL_CONSOLE_ONLY"): + EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" + SERVER_EMAIL = "test@example.com" +else: + SERVER_EMAIL = os.environ["TAKAHE_EMAIL_FROM"] + if "TAKAHE_EMAIL_SENDGRID_KEY" in os.environ: + EMAIL_HOST = "smtp.sendgrid.net" + EMAIL_PORT = 587 + EMAIL_HOST_USER: Optional[str] = "apikey" + EMAIL_HOST_PASSWORD: Optional[str] = os.environ["TAKAHE_EMAIL_SENDGRID_KEY"] + EMAIL_USE_TLS = True + else: + EMAIL_HOST = os.environ["TAKAHE_EMAIL_HOST"] + EMAIL_PORT = int(os.environ["TAKAHE_EMAIL_PORT"]) + EMAIL_HOST_USER = os.environ.get("TAKAHE_EMAIL_USER") + EMAIL_HOST_PASSWORD = os.environ.get("TAKAHE_EMAIL_PASSWORD") + EMAIL_USE_SSL = EMAIL_PORT == 465 + EMAIL_USE_TLS = EMAIL_PORT == 587 + +AUTO_ADMIN_EMAIL = os.environ.get("TAKAHE_AUTO_ADMIN_EMAIL") + +# Media storage +MEDIA_BACKEND = os.environ.get("TAKAHE_MEDIA_BACKEND", None) +if MEDIA_BACKEND == "local": + # Note that this MUST be a fully qualified URL in production + MEDIA_URL = os.environ.get("TAKAHE_MEDIA_URL", "/media/") + MEDIA_ROOT = os.environ.get("TAKAHE_MEDIA_ROOT", BASE_DIR / "media") +elif MEDIA_BACKEND == "gcs": + DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" + GS_BUCKET_NAME = os.environ["TAKAHE_MEDIA_BUCKET"] + GS_QUERYSTRING_AUTH = False +elif MEDIA_BACKEND == "s3": + DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" + AWS_STORAGE_BUCKET_NAME = os.environ["TAKAHE_MEDIA_BUCKET"] +else: + print("Unknown TAKAHE_MEDIA_BACKEND value") + sys.exit(1) + +# Stator secret token +STATOR_TOKEN = os.environ.get("TAKAHE_STATOR_TOKEN") + +# Error email recipients +if "TAKAHE_ERROR_EMAILS" in os.environ: + ADMINS = [("Admin", e) for e in os.environ["TAKAHE_ERROR_EMAILS"].split(",")] diff --git a/templates/identity/view.html b/templates/identity/view.html index d584022..bf60c2e 100644 --- a/templates/identity/view.html +++ b/templates/identity/view.html @@ -38,7 +38,7 @@ {% if identity.summary %} -
+
{{ identity.safe_summary }}
{% endif %} @@ -59,6 +59,6 @@ {% for post in posts %} {% include "activities/_post.html" %} {% empty %} - No posts yet. + No posts yet. {% endfor %} {% endblock %} diff --git a/users/models/password_reset.py b/users/models/password_reset.py index 290b08d..c300d23 100644 --- a/users/models/password_reset.py +++ b/users/models/password_reset.py @@ -34,7 +34,7 @@ class PasswordResetStates(StateGraph): "settings": settings, }, ), - from_email=settings.EMAIL_FROM, + from_email=settings.SERVER_EMAIL, recipient_list=[reset.user.email], ) else: @@ -48,7 +48,7 @@ class PasswordResetStates(StateGraph): "settings": settings, }, ), - from_email=settings.EMAIL_FROM, + from_email=settings.SERVER_EMAIL, recipient_list=[reset.user.email], ) return cls.sent