Merge branch 'main' into production

Use run --rm in `update` command

.env.example

@@ -89,3 +89,19 @@ PREVIEW_TEXT_COLOR=#363636
 PREVIEW_IMG_WIDTH=1200
 PREVIEW_IMG_HEIGHT=630
 PREVIEW_DEFAULT_COVER_COLOR=#002549
+
+# Below are example keys if you want to enable automatically
+# sending telemetry to an OTLP-compatible service. Many of
+# the main monitoring apps have OLTP collectors, including
+# NewRelic, DataDog, and Honeycomb.io - consult their
+# documentation for setup instructions, and what exactly to
+# put below!
+#
+# Service name is an arbitrary tag that is attached to any
+# data sent, used to distinguish different sources. Useful
+# for sending prod and dev metrics to the same place and
+# keeping them separate, for instance!
+
+OTEL_EXPORTER_OTLP_ENDPOINT=    # API endpoint for your provider
+OTEL_EXPORTER_OTLP_HEADERS=     # Any headers required, usually authentication info
+OTEL_SERVICE_NAME=              # Service name to identify your app

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,43 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: 'bug'
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Instance**
-On which BookWyrm instance did you encounter this problem.
-
-**Additional context**
-Add any other context about the problem here.
-
----
-
-**Desktop (please complete the following information):**
-    - OS: [e.g. iOS]
-    - Browser [e.g. chrome, safari]
-    - Version [e.g. 22]
-
-**Smartphone (please complete the following information):**
-    - Device: [e.g. iPhone6]
-    - OS: [e.g. iOS8.1]
-    - Browser [e.g. stock browser, safari]
-    - Version [e.g. 22]

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

Dockerfile

@@ -8,4 +8,3 @@ WORKDIR /app
 
 COPY requirements.txt /app/
 RUN pip install -r requirements.txt --no-cache-dir
-RUN apt-get update && apt-get install -y gettext libgettextpo-dev tidy && apt-get clean

bookwyrm/apps.py

@@ -33,6 +33,11 @@ class BookwyrmConfig(AppConfig):
     verbose_name = "BookWyrm"
 
     def ready(self):
+        if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+            from bookwyrm.telemetry import open_telemetry
+
+            open_telemetry.instrumentDjango()
+
         if settings.ENABLE_PREVIEW_IMAGES and settings.FONTS:
             # Download any fonts that we don't have yet
             logger.debug("Downloading fonts..")

bookwyrm/management/commands/erase_streams.py

@@ -7,6 +7,7 @@ from bookwyrm import settings
 r = redis.Redis(
     host=settings.REDIS_ACTIVITY_HOST,
     port=settings.REDIS_ACTIVITY_PORT,
+    password=settings.REDIS_ACTIVITY_PASSWORD,
     db=settings.REDIS_ACTIVITY_DB_INDEX,
 )
 

bookwyrm/settings.py

@@ -333,3 +333,7 @@ else:
     MEDIA_URL = "/images/"
     MEDIA_FULL_URL = f"{PROTOCOL}://{DOMAIN}{MEDIA_URL}"
     STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}"
+
+OTEL_EXPORTER_OTLP_ENDPOINT = env("OTEL_EXPORTER_OTLP_ENDPOINT", None)
+OTEL_EXPORTER_OTLP_HEADERS = env("OTEL_EXPORTER_OTLP_HEADERS", None)
+OTEL_SERVICE_NAME = env("OTEL_SERVICE_NAME", None)

bookwyrm/telemetry/open_telemetry.py

@@ -0,0 +1,22 @@
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+trace.set_tracer_provider(TracerProvider())
+trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
+
+
+def instrumentDjango():
+    from opentelemetry.instrumentation.django import DjangoInstrumentor
+
+    DjangoInstrumentor().instrument()
+
+
+def instrumentCelery():
+    from opentelemetry.instrumentation.celery import CeleryInstrumentor
+    from celery.signals import worker_process_init
+
+    @worker_process_init.connect(weak=False)
+    def init_celery_tracing(*args, **kwargs):
+        CeleryInstrumentor().instrument()

bw-dev

@@ -12,11 +12,8 @@ trap showerr EXIT
 source .env
 trap - EXIT
 
-function clean {
+# show commands as they're executed
-    docker-compose stop
+set -x
-    docker-compose rm -f
-}
-
 function runweb {
     docker-compose run --rm web "$@"
 }
@@ -34,10 +31,6 @@ function initdb {
     runweb python manage.py initdb "$@"
 }
 
-function makeitblack {
-    runweb black celerywyrm bookwyrm
-}
-
 function awscommand {
     # expose env vars
     export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
@@ -61,26 +54,9 @@ case "$CMD" in
     up)
         docker-compose up --build "$@"
         ;;
-    service_ports_web)
-        docker-compose run --rm --service-ports web
-        ;;
     initdb)
         initdb "$@"
         ;;
-    resetdb)
-        clean
-        # Start just the DB so no one else is using it
-        docker-compose up --build -d db
-        execdb dropdb -U ${POSTGRES_USER} ${POSTGRES_DB}
-        execdb createdb -U ${POSTGRES_USER} ${POSTGRES_DB}
-        # Now start up web so we can run the migrations
-        docker-compose up --build -d web
-        initdb
-        clean
-        ;;
-    makemigrations)
-        runweb python manage.py makemigrations "$@"
-        ;;
     migrate)
         runweb python manage.py migrate "$@"
         ;;
@@ -96,35 +72,9 @@ case "$CMD" in
     restart_celery)
         docker-compose restart celery_worker
         ;;
-    pytest)
-        execweb pytest --no-cov-on-fail "$@"
-        ;;
     collectstatic)
         runweb python manage.py collectstatic --no-input
         ;;
-    makemessages)
-        runweb django-admin makemessages --no-wrap --ignore=venv -l en_US $@
-        ;;
-    compilemessages)
-        runweb django-admin compilemessages --ignore venv $@
-        ;;
-    update_locales)
-        git fetch origin l10n_main:l10n_main
-        git checkout l10n_main locale/de_DE
-        git checkout l10n_main locale/es_ES
-        git checkout l10n_main locale/fr_FR
-        git checkout l10n_main locale/gl_ES
-        git checkout l10n_main locale/it_IT
-        git checkout l10n_main locale/lt_LT
-        git checkout l10n_main locale/no_NO
-        git checkout l10n_main locale/pt_PT
-        git checkout l10n_main locale/pt_BR
-        git checkout l10n_main locale/sv_SE
-        git checkout l10n_main locale/zh_Hans
-        git checkout l10n_main locale/zh_Hant
-        runweb django-admin makemessages --no-wrap --ignore=venv -l en_US $@
-        runweb django-admin compilemessages --ignore venv
-        ;;
     build)
         docker-compose build
         ;;
@@ -202,26 +152,16 @@ case "$CMD" in
         set +x # No need to echo echo
         echo "Unrecognised command. Try:"
         echo "    up [container]"
-        echo "    service_ports_web"
         echo "    initdb"
-        echo "    resetdb"
         echo "    makemigrations [migration]"
         echo "    migrate [migration]"
         echo "    bash"
         echo "    shell"
         echo "    dbshell"
         echo "    restart_celery"
-        echo "    pytest [path]"
         echo "    collectstatic"
-        echo "    makemessages"
-        echo "    compilemessages [locale]"
-        echo "    update_locales"
         echo "    build"
         echo "    clean"
-        echo "    black"
-        echo "    prettier"
-        echo "    stylelint"
-        echo "    formatters"
         echo "    populate_streams [--stream=<stream name>]"
         echo "    populate_suggestions"
         echo "    generate_thumbnails"

celerywyrm/apps.py

@@ -0,0 +1,13 @@
+from django.apps import AppConfig
+from celerywyrm import settings
+
+
+class CelerywyrmConfig(AppConfig):
+    name = "celerywyrm"
+    verbose_name = "BookWyrm Celery"
+
+    def ready(self):
+        if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+            from bookwyrm.telemetry import open_telemetry
+
+            open_telemetry.instrumentCelery()

certbot.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-source .env;
-
-if [ "$CERTBOT_INIT" = "true" ]
-then
-    certonly \
-    --webroot \
-    --webroot-path=/var/www/certbot \
-    --email ${EMAIL} \
-    --agree-tos \
-    --no-eff-email \
-    -d ${DOMAIN} \
-    -d www.${DOMAIN}
-else
-    renew \
-    --webroot \
-    --webroot-path \
-    /var/www/certbot
-fi

complete_bwdev.sh

@@ -1,33 +0,0 @@
-#/usr/bin/env bash
-# for zsh, run:
-# autoload bashcompinit
-# bashcompinit
-complete -W "up
-service_ports_web
-initdb
-resetdb
-makemigrations
-migrate
-bash
-shell
-dbshell
-restart_celery
-pytest
-collectstatic
-makemessages
-compilemessages
-update_locales
-build
-clean
-black
-prettier
-stylelint
-formatters
-populate_streams
-populate_lists_streams
-populate_suggestions
-generate_thumbnails
-generate_preview_images
-copy_media_to_s3
-set_cors_to_s3
-runweb" -o bashdefault -o default bw-dev

docker-compose.yml

@@ -4,26 +4,40 @@ services:
   nginx:
     image: nginx:latest
     ports:
-      - 1333:80
+      - 80:80
+      - 443:443
     depends_on:
       - web
     networks:
       - main
     volumes:
       - ./nginx:/etc/nginx/conf.d
+      - ./certbot/conf:/etc/nginx/ssl
+      - ./certbot/data:/var/www/certbot
       - static_volume:/app/static
       - media_volume:/app/images
+  certbot:
+    image: certbot/certbot:latest
+    command: certonly --webroot --webroot-path=/var/www/certbot --email ${EMAIL} --agree-tos --no-eff-email -d ${DOMAIN} -d www.${DOMAIN}
+    #command: renew --webroot --webroot-path /var/www/certbot
+    volumes:
+      - ./certbot/conf:/etc/letsencrypt
+      - ./certbot/logs:/var/log/letsencrypt
+      - ./certbot/data:/var/www/certbot
   db:
-    image: postgres
+    build: postgres-docker
     env_file: .env
+    entrypoint: /bookwyrm-entrypoint.sh
+    command: cron postgres
     volumes:
       - pgdata:/var/lib/postgresql/data
+      - backups:/backups
     networks:
       - main
   web:
     build: .
     env_file: .env
-    command: python manage.py runserver 0.0.0.0:8000
+    command: gunicorn bookwyrm.wsgi:application --bind 0.0.0.0:8000
     volumes:
       - .:/app
       - static_volume:/app/static
@@ -39,25 +53,23 @@ services:
   redis_activity:
     image: redis
     command: redis-server --requirepass ${REDIS_ACTIVITY_PASSWORD} --appendonly yes --port ${REDIS_ACTIVITY_PORT}
-    env_file: .env
-    networks:
-      - main
-    restart: on-failure
     volumes:
       - ./redis.conf:/etc/redis/redis.conf
       - redis_activity_data:/data
-  redis_broker:
-    image: redis
-    command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT}
     env_file: .env
-    ports:
-      - 6379
     networks:
       - main
     restart: on-failure
+  redis_broker:
+    image: redis
+    command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT}
     volumes:
       - ./redis.conf:/etc/redis/redis.conf
       - redis_broker_data:/data
+    env_file: .env
+    networks:
+      - main
+    restart: on-failure
   celery_worker:
     env_file: .env
     build: .
@@ -88,6 +100,7 @@ services:
     restart: on-failure
 volumes:
   pgdata:
+  backups:
   static_volume:
   media_volume:
   redis_broker_data:

nginx/development

@@ -5,20 +5,70 @@ upstream web {
 }
 
 server {
+    listen [::]:80;
     listen 80;
 
-    location / {
+    server_name your-domain.com www.your-domain.com;
-        proxy_pass http://web;
+
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    location ~ /.well-known/acme-challenge {
-        proxy_set_header Host $host;
+        allow all;
-        proxy_redirect off;
+        root /var/www/certbot;
     }
 
-    location /images/ {
+    # redirect http to https
-        alias /app/images/;
+#     return 301 https://your-domain.com$request_uri;
-    }
+# }
-
+#
-    location /static/ {
+# server {
-        alias /app/static/;
+#     listen [::]:443 ssl http2;
-    }
+#     listen 443 ssl http2;
+#
+#     server_name your-domain.com;
+#
+#     # SSL code
+#     ssl_certificate /etc/nginx/ssl/live/your-domain.com/fullchain.pem;
+#     ssl_certificate_key /etc/nginx/ssl/live/your-domain.com/privkey.pem;
+#
+#     location ~ /.well-known/acme-challenge {
+#         allow all;
+#         root /var/www/certbot;
+#     }
+#
+#     location / {
+#         proxy_pass http://web;
+#         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#         proxy_set_header Host $host;
+#         proxy_redirect off;
+#     }
+#
+#     location /images/ {
+#         alias /app/images/;
+#     }
+#
+#     location /static/ {
+#         alias /app/static/;
+#     }
 }
+
+# Reverse-Proxy server
+# server {
+#     listen [::]:8001;
+#     listen 8001;
+
+#     server_name your-domain.com www.your-domain.com;
+
+#     location / {
+#         proxy_pass http://web;
+#         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#         proxy_set_header Host $host;
+#         proxy_redirect off;
+#     }
+
+#     location /images/ {
+#         alias /app/images/;
+#     }
+
+#     location /static/ {
+#         alias /app/static/;
+#     }
+# }

postgres-docker/Dockerfile

@@ -0,0 +1,20 @@
+FROM postgres:13.0
+
+# crontab
+RUN mkdir /backups
+COPY ./backup.sh /backups
+COPY ./weed.sh /backups
+COPY ./cronfile /etc/cron.d/cronfile
+RUN apt-get update && apt-get -y --no-install-recommends install cron
+RUN chmod 0644 /etc/cron.d/cronfile
+RUN crontab /etc/cron.d/cronfile
+RUN touch /var/log/cron.log
+
+# The postgres image's entrypoint expects the docker command to only contain flags to
+# pass postgres. It runs the entrypoint twice, the second times as the postgres user.
+# We need to start the cron service the first time it runs, when it's still being run
+# as the root user. We're going to add a check that looks at the first argument and
+# if it's 'cron', starts the service and then removes that argument.
+RUN awk '$0 ~ /^\t_main "\$@"$/ { print "\tif [[ $1 == cron ]]; then\n\t\tservice cron start\n\t\tshift\n\tfi" }{ print }' docker-entrypoint.sh > bookwyrm-entrypoint.sh
+RUN chown postgres /bookwyrm-entrypoint.sh
+RUN chmod u=rwx,go=r /bookwyrm-entrypoint.sh

postgres-docker/backup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+if [ -z "$POSTGRES_DB" ]; then
+    echo "Database not specified, defaulting to bookwyrm"
+fi
+BACKUP_DB=${POSTGRES_DB:-bookwyrm}
+filename=backup_${BACKUP_DB}_$(date +%F)
+pg_dump -U $BACKUP_DB > /backups/$filename.sql

postgres-docker/cronfile

@@ -0,0 +1,5 @@
+0 0 * * * /backups/backup.sh
+# If uncommented, this script will weed the backups directory. It will keep the 14
+# most-recent backups, then one backup/week for the next four backups, then one
+# backup/month after that.
+# 0 1 * * * /backups/weed.sh -d 14 -w 4 -m -1 /backups

postgres-docker/tests/Dockerfile

@@ -0,0 +1,8 @@
+FROM postgres:latest
+
+RUN apt update && apt install -y shellcheck
+
+COPY ./tests/testing-entrypoint.sh /testing-entrypoint.sh
+RUN chmod u+rx,go=r /testing-entrypoint.sh
+COPY ./weed.sh /weed.sh
+RUN chmod u+rx,go=r /weed.sh

postgres-docker/tests/docker-compose.yaml

@@ -0,0 +1,9 @@
+version: "3"
+
+services:
+  weeding:
+    build:
+      # We need to build from the parent directory so we can access weed.sh
+      context: ..
+      dockerfile: ./tests/Dockerfile
+    entrypoint: /testing-entrypoint.sh

postgres-docker/tests/testing-entrypoint.sh

@@ -0,0 +1,426 @@
+#!/usr/bin/env bash
+# These tests are written to run in their own container, using the same image as the
+# actual postgres service. To run: `docker-compose up --build`
+set -euo pipefail
+
+source /weed.sh
+
+ERROR_COUNT=0
+FAILURE_COUNT=0
+
+# compare two sorted files
+function compare_files {
+    local expected="$1"
+    local actual="$2"
+
+    declare -a missing
+    local missing_index=0
+    declare -a extra
+    local extra_index=0
+
+    old_ifs="$IFS"
+    IFS=$'\n'
+    for line in $(diff --suppress-common-lines "$expected" "$actual"); do
+        if [[ $line =~ ^\< ]]; then
+            missing[missing_index]=${line:1}
+            missing_index=$((missing_index + 1))
+        elif [[ $line =~ ^\> ]]; then
+            extra[extra_index]=${line:1}
+            extra_index=$((extra_index + 1))
+        fi
+    done
+    IFS="$old_ifs"
+
+    if [[ $((missing_index + extra_index)) -gt 0 ]]; then
+        echo 'fail'
+
+        if [[ missing_index -gt 0 ]]; then
+            echo -e "\\t$missing_index missing files:"
+
+            for index in $(seq 0 $((missing_index - 1))); do
+                echo -e "\\t\\t${missing[index]}"
+            done
+        fi
+
+        if [[ extra_index -gt 0 ]]; then
+            echo -e "\\t$extra_index extra files:"
+
+            for index in $(seq 0 $((extra_index - 1))); do
+                echo -e "\\t\\t${extra[index]}"
+            done
+        fi
+
+        FAILURE_COUNT=$((FAILURE_COUNT + 1))
+
+        return 1
+    fi
+}
+
+# This is a wrapper function that handles creating a directory with test files in it,
+# running weed_directory (as the function, as a dry run, then finally actually-deleting
+# files), marking the test as failed/errored as necessary, then cleaning up after
+# itself. the first three arguments passed are the thresholds to pass into
+# weed_directory. The remaining arguments are names of files to create for the test.
+# Bash isn't great at passing arrays so instead of separately passing in a list of
+# expected results, flag the files you expect to be deleted by prepending "DELETE:"
+# to the path.
+function perform_test {
+    echo "${FUNCNAME[1]}" | sed 's/^test_\(.*\)$/\1/' | tr '_\n' ' :'
+    echo -en '\t'
+
+    local daily_threshold="$1"
+    shift
+    local weekly_threshold="$1"
+    shift
+    local monthly_threshold="$1"
+    shift
+
+    # We might as well name the files we're using for running tests in as inflamatory a
+    # way as possible to increase the chances that bad filtering by weed_directory
+    # results in tests failing.
+    local expected="/testing/expected/backup__2020-02-02.sql"
+    local actual="/testing/backup__2020-02-02.sql.actual"
+    local remaining="/testing/remainbackup__2020-02-02.sql"
+    local temp="/testing/backup__2020-TE-MP.sql"
+
+    # create test files
+    mkdir -p /testing/expected
+    if [[ -e "$expected" ]]; then
+        rm "$expected"
+    fi
+    touch "$expected"
+    echo -e "$expected\\n$actual\\n$remaining\\n$temp" > "$remaining"
+    while [[ "$#" -gt 0 ]]; do
+        if [[ "$1" =~ ^DELETE: ]]; then
+            path="/testing/${1:7}"
+            echo "$path" >> "$expected"
+        else
+            path="/testing/$1"
+            echo "$path" >> "$remaining"
+        fi
+
+        directory=$(dirname "$path")
+        mkdir -p "$directory"
+        touch "$path"
+
+        shift
+    done
+    # We don't make any promise about the order files will be listed in by
+    # weed_directory (it is currently reverse-chronological). We should sort the output
+    # and the expected file instead of forcing tests to list files in that order (or
+    # causing tests to fail if weed_directory's order changes)
+    sort "$expected" > "$temp"
+    mv "$temp" "$expected"
+    sort "$remaining" > "$temp"
+    mv "$temp" "$remaining"
+
+    # Part one: call the function directly
+    set +e
+    (
+        weed_directory \
+            "/testing" \
+            "$daily_threshold" \
+            "$weekly_threshold" \
+            "$monthly_threshold" \
+            2> "$temp" \
+        | sort > "$actual"
+    )
+    local result="$?"
+    set -e
+
+    if [[ "$result" -ne 0 ]]; then
+        echo 'error'
+        ERROR_COUNT=$((ERROR_COUNT + 1))
+        if [[ -s "$temp" ]]; then
+            echo 'stderr:'
+            cat "$temp"
+        fi
+    else
+        set +e
+        compare_files "$expected" "$actual"
+        result="$?"
+        set -e
+
+        if [[ "$result" -eq 0 ]]; then
+            # Part two: as a script with the dry-run flag (-l)
+            set +e
+            (
+                "/weed.sh" \
+                "-d" "$daily_threshold" \
+                "-w" "$weekly_threshold" \
+                "-m" "$monthly_threshold" \
+                "-l" \
+                "/testing" \
+                2> "$temp" \
+                | sort > "$actual"
+            )
+            local result="$?"
+            set -e
+
+            if [[ "$result" -ne 0 ]]; then
+                echo 'error'
+                ERROR_COUNT=$((ERROR_COUNT + 1))
+                if [[ -s "$temp" ]]; then
+                    echo 'stderr:'
+                    cat "$temp"
+                fi
+            else
+                set +e
+                compare_files "$expected" "$actual"
+                result="$?"
+                set -e
+
+                if [[ "$result" -eq 0 ]]; then
+                    # Part three: let's try actually deleting files
+                    set +e
+                    (
+                        "/weed.sh" \
+                        "-d" "$daily_threshold" \
+                        "-w" "$weekly_threshold" \
+                        "-m" "$monthly_threshold" \
+                        "/testing" \
+                        2> "$temp"
+                    )
+                    local result="$?"
+                    set -e
+
+                    if [[ "$result" -ne 0 ]]; then
+                        echo 'error'
+                        ERROR_COUNT=$((ERROR_COUNT + 1))
+                        if [[ -s "$temp" ]]; then
+                            echo 'stderr:'
+                            cat "$temp"
+                        fi
+                    else
+                        find /testing -type f | sort > "$actual"
+
+                        set +e
+                        compare_files "$remaining" "$actual"
+                        result="$?"
+                        set -e
+
+                        if [[ "$result" -eq 0 ]]; then
+                            echo 'pass'
+                        elif [[ -s "$temp" ]]; then
+                            echo 'stderr:'
+                            cat "$temp"
+                        fi
+                    fi
+                elif [[ -s "$temp" ]]; then
+                    echo 'stderr:'
+                    cat "$temp"
+                fi
+            fi
+        elif [[ -s "$temp" ]]; then
+            echo 'stderr:'
+            cat "$temp"
+        fi
+    fi
+    rm -rf /testing
+}
+
+# actual tests
+function test_shellcheck {
+    echo -en 'running shellcheck on scripts:\t'
+    shellcheck /weed.sh
+    # Test the tests too! Writing bash is hard
+    shellcheck -x /testing-entrypoint.sh
+    echo 'pass'
+}
+
+function test_empty_directory {
+    perform_test 1 2 3
+}
+
+function test_single_file {
+    perform_test 1 2 3 "backup__2021-02-02.sql"
+}
+
+function test_keep_everything {
+    perform_test -1 0 0 "backup__2021-02-02.sql" "backup__2021-02-01.sql" "backup__2021-01-31.sql"
+}
+
+function test_keep_one {
+    perform_test 1 0 0 "backup__2021-02-02.sql" "DELETE:backup__2021-02-01.sql" "DELETE:backup__2021-01-31.sql"
+}
+
+function test_weekly {
+    # weed.sh follows ISO 8601 and uses %W for day of week, so Monday is the first day
+    # of the week.
+    # backup__2021-03-08.sql: Monday (keep)
+    # backup__2021-03-07.sql: Sunday (keep)
+    # backup__2021-02-28.sql: Sunday (keep)
+    # backup__2021-02-22.sql: Monday (delete)
+    # backup__2021-02-20.sql: Saturday (keep)
+    # backup__2021-02-16.sql: Tuesday (delete)
+    # backup__2021-02-15.sql: Monday (delete)
+    # backup__2021-02-14.sql: Sunday (keep)
+    # backup__2020-02-14.sql: Sunday (same week of year) (keep)
+    perform_test 0 -1 0 \
+        "backup__2021-03-08.sql" \
+        "backup__2021-03-07.sql" \
+        "backup__2021-02-28.sql" \
+        "DELETE:backup__2021-02-22.sql" \
+        "backup__2021-02-20.sql" \
+        "DELETE:backup__2021-02-16.sql" \
+        "DELETE:backup__2021-02-15.sql" \
+        "backup__2021-02-14.sql" \
+        "backup__2020-02-14.sql"
+}
+
+function test_monthly {
+    perform_test 1 0 -1 \
+        "backup__2021-03-08.sql" \
+        "DELETE:backup__2021-03-07.sql" \
+        "backup__2021-02-28.sql" \
+        "DELETE:backup__2021-02-22.sql" \
+        "DELETE:backup__2021-02-20.sql" \
+        "DELETE:backup__2021-02-16.sql" \
+        "DELETE:backup__2021-02-15.sql" \
+        "DELETE:backup__2021-02-14.sql" \
+        "backup__2021-01-14.sql" \
+        "backup__2020-01-13.sql"
+}
+
+function test_annual {
+    perform_test 0 0 0 \
+        "backup__2021-03-08.sql" \
+        "DELETE:backup__2021-03-07.sql" \
+        "DELETE:backup__2021-02-28.sql" \
+        "DELETE:backup__2021-02-22.sql" \
+        "DELETE:backup__2021-02-20.sql" \
+        "DELETE:backup__2021-02-16.sql" \
+        "DELETE:backup__2021-02-15.sql" \
+        "DELETE:backup__2021-02-14.sql" \
+        "DELETE:backup__2021-01-14.sql" \
+        "backup__2020-01-13.sql" \
+        "backup__2019-12-31.sql" \
+        "DELETE:backup__2019-01-13.sql"
+}
+
+# Will not pass while maxdepth is set to 1.
+function skip_test_sort_order {
+    perform_test 0 0 1 \
+        "a/backup__2021-03-08.sql" \
+        "DELETE:b/backup__2021-03-07.sql" \
+        "DELETE:a/backup__2021-02-28.sql" \
+        "DELETE:b/backup__2021-02-22.sql" \
+        "DELETE:a/backup__2021-02-20.sql" \
+        "DELETE:b/backup__2021-02-16.sql" \
+        "DELETE:a/backup__2021-02-15.sql" \
+        "DELETE:b/backup__2021-02-14.sql" \
+        "DELETE:a/backup__2021-01-14.sql" \
+        "b/backup__2020-01-13.sql" \
+        "a/backup__2019-12-31.sql" \
+        "DELETE:b/backup__2019-01-13.sql"
+}
+
+function test_ignore_subdirectories {
+    perform_test 0 0 0 "a/backup__2021-03-08.sql" "backup__2021-03-07.sql"
+}
+
+function test_standard {
+    perform_test 14 4 1 \
+        "backup__2021-03-08.sql" \
+        "backup__2021-03-07.sql" \
+        "backup__2021-03-06.sql" \
+        "backup__2021-03-05.sql" \
+        "backup__2021-03-04.sql" \
+        "backup__2021-03-03.sql" \
+        "backup__2021-03-02.sql" \
+        "backup__2021-03-01.sql" \
+        "backup__2021-02-28.sql" \
+        "backup__2021-02-27.sql" \
+        "backup__2021-02-26.sql" \
+        "backup__2021-02-25.sql" \
+        "backup__2021-02-24.sql" \
+        "backup__2021-02-23.sql" \
+        "DELETE:backup__2021-02-22.sql" \
+        "backup__2021-02-21.sql" \
+        "DELETE:backup__2021-02-20.sql" \
+        "DELETE:backup__2021-02-19.sql" \
+        "DELETE:backup__2021-02-18.sql" \
+        "DELETE:backup__2021-02-17.sql" \
+        "DELETE:backup__2021-02-16.sql" \
+        "DELETE:backup__2021-02-15.sql" \
+        "backup__2021-02-14.sql" \
+        "DELETE:backup__2021-02-13.sql" \
+        "DELETE:backup__2021-02-12.sql" \
+        "DELETE:backup__2021-02-11.sql" \
+        "DELETE:backup__2021-02-10.sql" \
+        "DELETE:backup__2021-02-09.sql" \
+        "DELETE:backup__2021-02-08.sql" \
+        "backup__2021-02-07.sql" \
+        "DELETE:backup__2021-02-06.sql" \
+        "DELETE:backup__2021-02-05.sql" \
+        "DELETE:backup__2021-02-04.sql" \
+        "DELETE:backup__2021-02-03.sql" \
+        "DELETE:backup__2021-02-02.sql" \
+        "DELETE:backup__2021-02-01.sql" \
+        "backup__2021-01-31.sql" \
+        "DELETE:backup__2021-01-30.sql" \
+        "DELETE:backup__2021-01-29.sql" \
+        "DELETE:backup__2021-01-28.sql" \
+        "DELETE:backup__2021-01-27.sql" \
+        "DELETE:backup__2021-01-26.sql" \
+        "DELETE:backup__2021-01-25.sql" \
+        "DELETE:backup__2021-01-24.sql" \
+        "DELETE:backup__2021-01-23.sql" \
+        "DELETE:backup__2021-01-22.sql" \
+        "DELETE:backup__2021-01-21.sql" \
+        "DELETE:backup__2021-01-20.sql" \
+        "DELETE:backup__2021-01-19.sql" \
+        "DELETE:backup__2021-01-18.sql" \
+        "DELETE:backup__2021-01-17.sql" \
+        "DELETE:backup__2021-01-16.sql" \
+        "DELETE:backup__2021-01-15.sql" \
+        "DELETE:backup__2021-01-14.sql" \
+        "DELETE:backup__2021-01-13.sql" \
+        "DELETE:backup__2021-01-12.sql" \
+        "DELETE:backup__2021-01-11.sql" \
+        "DELETE:backup__2021-01-10.sql" \
+        "DELETE:backup__2021-01-09.sql" \
+        "DELETE:backup__2021-01-08.sql" \
+        "DELETE:backup__2021-01-07.sql" \
+        "DELETE:backup__2021-01-06.sql" \
+        "DELETE:backup__2021-01-05.sql" \
+        "DELETE:backup__2021-01-04.sql" \
+        "DELETE:backup__2021-01-03.sql" \
+        "DELETE:backup__2021-01-02.sql" \
+        "DELETE:backup__2021-01-01.sql" \
+        "backup__2020-12-31.sql"
+}
+
+function tests {
+    # Run all functions named test_... in this file in definition order
+    count=0
+    while read -r test; do
+        eval "$test"
+        count=$((count + 1))
+    done < <(awk '$1 == "function" && $2 ~ "^test_" {print $2}' "${BASH_SOURCE[0]}")
+
+    echo "------------------"
+    echo "$((count - ERROR_COUNT - FAILURE_COUNT))/$count tests passed"
+    if [[ $((FAILURE_COUNT + ERROR_COUNT)) -gt 0 ]]; then
+        if [[ "$ERROR_COUNT" -gt 0 ]]; then
+            echo "$ERROR_COUNT tests errored"
+        fi
+
+        if [[ "$FAILURE_COUNT" -gt 0 ]]; then
+            echo "$FAILURE_COUNT tests failed"
+        fi
+        echo 'failure'
+    else
+        echo 'success'
+    fi
+}
+
+if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then
+    trap 'echo -e "\\terror (in ${FUNCNAME[1]} ${BASH_SOURCE[1]}:${BASH_LINENO[1]})\naborting"' EXIT
+    tests
+    trap - EXIT
+
+    if [[ $((FAILURE_COUNT + ERROR_COUNT)) -gt 0 ]]; then
+        exit 1
+    fi
+fi

postgres-docker/weed.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+# Weed old backups. See HELP for details.
+# Tests for this script can be found in:
+# bookwyrm/postgres-docker/tests/testing-entrypoint.sh
+set -euo pipefail
+
+DAILY_THRESHOLD=14
+WEEKLY_THRESHOLD=4
+MONTHLY_THRESHOLD=-1
+
+HELP="\
+NAME
+
+weed -- remove old backups from the backups directory
+
+SYNOPSIS
+
+weed.sh [-d threshold] [-w threshold] [-m threshold] [-l] backup_directory
+
+DESCRIPTION
+
+Reduce the number of backups by only keeping a certain number of daily backups before \
+reducing the frequency to weekly, monthly, and then finaly annually.
+
+For each threshold, setting it to 0 will skip that frequency (e.g., setting weekly to \
+0 will mean backups go directly from daily to monthly), and setting it to -1 will \
+never reduce backups to a lower frequency (e.g., setting weekly to -1 will mean \
+backups never are reduced to monthly backups).
+
+-d threshold: Store this many daily backups before switching to weekly \
+(default $DAILY_THRESHOLD)
+
+-w threshold: Store this many weekly backups before switching to monthly \
+(default $WEEKLY_THRESHOLD)
+
+-m threshold: Store this many monthly backups before switching to annual \
+(default $MONTHLY_THRESHOLD)
+
+-l: Dry run. List the files that would be deleted.
+"
+
+# fail <message>
+# Write a message to stderr then exit
+function fail {
+    echo -e "weed: $1" >&2
+    exit 1
+}
+
+# parse_threshold <hopefully-a-number>
+# Thresholds should be a non-negative number (or -1 for no threshold)
+function parse_threshold {
+    if [[ ! $1 =~ ^-?[0-9]+$ || $1 -lt -1 ]]; then
+        fail "Invalid threshold: $1"
+    fi
+
+    echo "$1"
+}
+
+# weed_directory <directory> <daily_threshold> <weekly_threshold> <monthly_threshold>
+# List files to be deleted
+function weed_directory {
+    local directory=$1
+    local daily_threshold=$2
+    local weekly_threshold=$3
+    local monthly_threshold=$4
+
+    local count=0
+    local thresholds=("$daily_threshold" "$weekly_threshold" "$monthly_threshold" -1)
+    local date_formats=("%Y %m %d" "%Y %W" "%Y %m" "%Y")
+    local index=0
+    local last_date=""
+    local last_format=""
+    local date=""
+
+    # We would like to loop through all the backup files in the backup directory in
+    # reverse-chronological order. Bookwyrm backup files are named such that
+    # chronological and lexical order match. So we should be safe to find all backup
+    # files and reverse sort them. We should be terrified of deleting a backup an
+    # instance maintainer wants to keep, so we will be extra cautious. We're ignoring
+    # any subdirectories in case someone moves an important backup into a meaningfully
+    # named folder. We are also prepending the date to the path before sorting so that
+    # the ordering would be correct even if we were allowed to find backup files in
+    # subdirectories where chronological and lexical order don't match.
+    for date_file in $(
+        find "$directory" \
+            -maxdepth 1 \
+            -name 'backup__[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\.sql' \
+        | sed 's/\(^.*backup__\([0-9-]*\)\.sql$\)/\2\1/' \
+        | sort --reverse
+    ); do
+        date="${date_file:0:10}"
+        file="${date_file:10}"
+        date="${date_file:0:10}"
+        file="${date_file:10}"
+
+        # We can't fall off the end because we set annual backups to unlimited. It seems
+        # unlikely that instance maintainers would have enough concern about the space
+        # one backup/year takes to warrant supporting a cutoff.
+        while [[ ${thresholds[index]} -ne -1 && $count -ge ${thresholds[index]} ]]; do
+            index=$((index + 1))
+            last_format=""
+            count=0
+        done
+
+        if [[ -z "$last_date" ]]; then
+            count=$((count + 1))
+            last_date=$date
+            last_format=""
+        else
+            if [[ -z "$last_format" ]]; then
+                last_format=$(date --date="$last_date" +"${date_formats[index]}")
+            fi
+
+            format=$(date --date="$date" +"${date_formats[index]}")
+
+            if [[ "$format" == "$last_format" ]]; then
+                echo "$file"
+            else
+                count=$((count + 1))
+                last_date="$date"
+                last_format="$format"
+            fi
+        fi
+    done
+}
+
+function main(){
+    local daily_threshold=$DAILY_THRESHOLD
+    local weekly_threshold=$WEEKLY_THRESHOLD
+    local monthly_threshold=$MONTHLY_THRESHOLD
+    local dry_run=""
+
+    while getopts "hd:w:m:l" OPTION; do
+        case "$OPTION" in
+            h)
+                echo "$HELP";
+                exit
+                ;;
+            d)
+                daily_threshold=$(parse_threshold "$OPTARG")
+                ;;
+            w)
+                weekly_threshold=$(parse_threshold "$OPTARG")
+                ;;
+            m)
+                monthly_threshold=$(parse_threshold "$OPTARG")
+                ;;
+            l)
+                dry_run="true"
+                ;;
+            :)
+                fail "Missing argument for '$OPTARG'. To see help run: weed.sh -h"
+                ;;
+            ?)
+                fail "Unknown option '$OPTION'. To see help run: weed.sh -h"
+        esac
+    done
+    shift "$((OPTIND - 1))"
+
+    if [[ $# -ne 1 ]]; then
+        fail "expected a single argument, directory"
+    fi
+
+    local count=0
+    for file in $(weed_directory "$1" "$daily_threshold" "$weekly_threshold" "$monthly_threshold"); do
+        count=$((count + 1))
+        if [[ -n "$dry_run" ]]; then
+            echo "$file"
+        else
+            echo "deleting $file" >&2
+            rm "$file"
+        fi
+    done
+
+    if [[ -n "$dry_run" ]]; then
+        optional_words="would be "
+    else
+        optional_words=""
+    fi
+    echo -e "$count files ${optional_words}deleted" >&2
+}
+
+if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then
+    main "$@"
+fi

requirements.txt

@@ -5,6 +5,7 @@ django-imagekit==4.1.0
 django-model-utils==4.0.0
 environs==9.3.4
 flower==1.0.0
+gunicorn==20.0.4
 Markdown==3.3.3
 Pillow>=9.0.0
 psycopg2==2.8.4
@@ -18,12 +19,8 @@ pytz>=2021.1
 boto3==1.17.88
 django-storages==1.11.1
 django-redis==5.2.0
-
+opentelemetry-api==1.8.0
-# Dev
+opentelemetry-sdk==1.8.0
-black==21.4b2
+opentelemetry-exporter-otlp-proto-grpc==1.8.0
-pytest-django==4.1.0
+opentelemetry-instrumentation-django==0.27b0
-pytest==6.1.2
+opentelemetry-instrumentation-celery==0.27b0
-pytest-cov==2.10.1
-pytest-env==0.6.2
-pytest-xdist==2.3.0
-pytidylib==0.3.2