Use supervisor and fixup env generation

CloudronManifest.json

@@ -17,7 +17,7 @@
     "ldap": {}
   },
   "minBoxVersion": "1.8.1",
-  "manifestVersion": 1,
+  "manifestVersion": 2,
   "website": "https://joinmastodon.org/",
   "contactEmail": "syn+cloudron@syn.im",
   "icon": "logo.png",
@@ -30,5 +30,6 @@
   ],
   "documentationUrl": "https://docs.joinmastodon.org/administration/post-installation/",
   "postInstallMessage": "Follow the post-installation guide to promote users to admins",
-  "optionalSso": true
+  "optionalSso": true,
+  "minBoxVersion": "4.1.4"
 }

Dockerfile

@@ -3,52 +3,53 @@ FROM cloudron/base:1.0.0@sha256:147a648a068a2e746644746bbfb42eb7a50d682437cead3c
 RUN mkdir -p /app/code
 WORKDIR /app/code
 
-RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+RUN apt-key adv --fetch-keys http://dl.yarnpkg.com/debian/pubkey.gpg && \
-RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+    echo "deb http://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list && \
+    apt-get update && \
+    apt-get install -y yarn libprotobuf-dev protobuf-compiler libidn11-dev libicu-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm5 libgdbm-dev libicu-dev libssl-dev libyaml-dev libreadline6-dev libxml2-dev libxslt1-dev && \
+    rm -rf /var/cache/apt /var/lib/apt/lists
 
-# the following does apt-get update
+RUN mkdir -p /usr/local/node-10.15.3 && \
-RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
+    curl -L https://nodejs.org/dist/v10.15.3/node-v10.15.3-linux-x64.tar.gz | tar zxf - --strip-components 1 -C /usr/local/node-10.15.3
 
-RUN apt-get install -y \
+ENV PATH /usr/local/node-10.15.3/bin:$PATH
-    imagemagick ffmpeg libpq-dev libxml2-dev libxslt1-dev file git-core \
-    g++ libprotobuf-dev protobuf-compiler pkg-config nodejs gcc autoconf \
-    bison build-essential libssl-dev libyaml-dev libreadline6-dev \
-    zlib1g-dev libncurses5-dev libffi-dev libgdbm5 libgdbm-dev \
-    nginx redis-server redis-tools postgresql postgresql-contrib \
-    certbot yarn libidn11-dev libicu-dev libjemalloc-dev \
-    ruby2.5
 
-RUN gem update --system
+RUN gem install --no-document bundler -v 1.17.3
-RUN gem install bundler 
 
-RUN rm -r /etc/nginx/sites-enabled/default /var/lib/nginx /var/log/nginx
+ARG VERSION=2.9.2
-RUN mkdir -p /run/nginx && ln -fs /run/nginx /var/lib/nginx && ln -fs /run/nginx/log /var/log/nginx
+ENV RAILS_ENV production
-
+ENV NODE_ENV production
-RUN git init && \
+RUN curl -L https://github.com/tootsuite/mastodon/archive/v${VERSION}.tar.gz | tar -xz --strip-components 1 -f - && \
-    git remote add origin https://github.com/tootsuite/mastodon.git && \
+    bundle install --deployment --without test development && \
-    git fetch --depth=1 origin $(git ls-remote --tags | grep refs/tags | grep -v 'rc[0-9]*$' | cut -f2 | sort -V | tail -n 1 | cut -d '/' -f3-) && \
+    yarn install --pure-lockfile
-    git checkout FETCH_HEAD
 
 COPY patches /app/code/patches
 RUN for patch in /app/code/patches/*; do patch -N -p0 < $patch; done
 
-RUN bundle install -j$(getconf _NPROCESSORS_ONLN) --deployment --without development test && \
-    yarn install --pure-lockfile
-
-ENV GEM_PATH=/app/code/vendor/bundle/ruby/2.5.0/gems/ RAILS_ENV=production NODE_ENV=production
-
 # secret keys are not built into assets, so precompiling is safe to do here
 # (these variables are required by rake though)
 RUN SECRET_KEY_BASE=insecure.secret_key_base OTP_SECRET=insecure.otp_secret \
     bundle exec rake assets:precompile
 
-RUN ln -fs /app/data/.env.production /app/code/.env.production
+# add nginx config
-RUN ln -fs /app/data/bullet.log /app/code/log/bullet.log
+USER root
+RUN rm /etc/nginx/sites-enabled/*
+RUN ln -sf /dev/stdout /var/log/nginx/access.log
+RUN ln -sf /dev/stderr /var/log/nginx/error.log
+ADD nginx_readonlyrootfs.conf /etc/nginx/conf.d/readonlyrootfs.conf
+COPY nginx/mastodon.conf /etc/nginx/sites-available/mastodon
+RUN ln -s /etc/nginx/sites-available/mastodon /etc/nginx/sites-enabled/mastodon
+
+# add supervisor configs
+ADD supervisor/* /etc/supervisor/conf.d/
+RUN ln -sf /run/mastodon/supervisord.log /var/log/supervisor/supervisord.log
+
+RUN ln -fs /app/data/env.production /app/code/.env.production
+RUN ln -fs /run/mastodon/bullet.log /app/code/log/bullet.log
 RUN ln -fs /app/data/system /app/code/public/system
-RUN rm -rf /app/code/tmp && ln -fs /tmp /app/code/tmp
+RUN rm -rf /app/code/tmp && ln -fs /tmp/mastodon /app/code/tmp
 
-CMD /app/code/start.sh
+COPY start.sh env.template /app/code/
+
+CMD [ "/app/code/start.sh" ]
 
-COPY nginx.conf /etc/nginx/sites-enabled/mastodon
-COPY mastodon.env.template /app/code
-COPY start.sh /app/code

docker-compose.yml

@@ -1,33 +0,0 @@
-version: '2.4'
-
-services:
-  mastodon:
-    build: .
-    ports:
-      - 3000
-      - 4000
-      - 80:8000
-    environment:
-      - LOCAL_DOMAIN=localhost
-      - LOCAL_HTTPS=false
-      - POSTGRESQL_HOST=postgres
-      - POSTGRESQL_DATABASE=postgres
-      - POSTGRESQL_USERNAME=postgres
-      - POSTGRESQL_PASSWORD=postgres
-      - REDIS_HOST=redis
-    volumes:
-      - data:/app/data
-    tmpfs:
-      - /run
-      - /tmp
-    read_only: true
-  postgres:
-    image: postgres
-    environment:
-      - POSTGRES_PASSWORD=postgres
-
-  redis:
-    image: redis
-
-volumes:
-  data:

env.template

@@ -0,0 +1,34 @@
+SINGLE_USER_MODE=false
+
+LOCAL_DOMAIN=
+
+DB_HOST=
+DB_PORT=
+DB_NAME=
+DB_USER=
+DB_PASS=
+
+REDIS_HOST=
+REDIS_PORT=
+REDIS_PASSWORD=
+
+SMTP_SERVER=
+SMTP_PORT=
+SMTP_FROM_ADDRESS=
+SMTP_LOGIN=
+SMTP_PASSWORD=
+SMTP_AUTH_METHOD=plain
+SMTP_OPENSSL_VERIFY_MODE=none
+
+# LDAP_ENABLED=$([ -z "${CLOUDRON_LDAP_SERVER}" ] && echo "false" || echo "true")
+# LDAP_HOST=${CLOUDRON_LDAP_SERVER}
+# LDAP_PORT=${CLOUDRON_LDAP_PORT}
+# LDAP_BASE=${CLOUDRON_LDAP_USERS_BASE_DN}
+# LDAP_BIND_DN=${CLOUDRON_LDAP_BIND_DN}
+# LDAP_PASSWORD=${CLOUDRON_LDAP_BIND_PASSWORD}
+# LDAP_UID=username
+# LDAP_SEARCH_FILTER=(|(%{uid}=%{email})(mail=%{email}))
+
+SECRET_KEY_BASE=
+OTP_SECRET=
+

mastodon.env.template

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-cat <<END
-SINGLE_USER_MODE=false
-
-LOCAL_DOMAIN="${APP_DOMAIN:-"localhost"}"
-
-DB_HOST="${POSTGRESQL_HOST:-"localhost"}"
-DB_PORT="${POSTGRESQL_PORT:-"5432"}"
-DB_NAME="${POSTGRESQL_DATABASE:-"mastodon"}"
-DB_USER="${POSTGRESQL_USERNAME:-"mastodon"}"
-DB_PASS="${POSTGRESQL_PASSWORD:-"mastodon"}"
-DATABASE_URL="${POSTGRESQL_URL:-"postgresql://${POSTGRESQL_USERNAME:-"mastodon"}:${POSTGRESQL_PASSWORD:-"mastodon"}@${POSTGRESQL_HOST:-"localhost"}:${POSTGRESQL_PORT:-"5432"}/${POSTGRESQL_DATABASE:-"mastodon"}"}"
-
-REDIS_HOST="${REDIS_HOST:-"localhost"}"
-REDIS_PORT="${REDIS_PORT:-"6379"}"
-REDIS_PASSWORD=${REDIS_PASSWORD}
-
-SMTP_SERVER="${MAIL_SMTP_SERVER:-"localhost"}"
-SMTP_PORT="${MAIL_SMTP_PORT:-"25"}"
-SMTP_FROM_ADDRESS="${MAIL_FROM:-"Mastodon <mastodon@localhost>"}"
-SMTP_AUTH_METHOD=plain
-SMTP_LOGIN=${MAIL_SMTP_USERNAME}
-SMTP_PASSWORD=${MAIL_SMTP_PASSWORD}
-SMTP_OPENSSL_VERIFY_MODE=none
-
-LDAP_ENABLED=$([ -z "$LDAP_SERVER" ] && echo "false" || echo "true")
-LDAP_HOST=${LDAP_SERVER}
-LDAP_PORT=${LDAP_PORT}
-LDAP_BASE=${LDAP_USERS_BASE_DN}
-LDAP_BIND_DN=${LDAP_BIND_DN}
-LDAP_PASSWORD=${LDAP_BIND_PASSWORD}
-LDAP_UID=username
-LDAP_SEARCH_FILTER=(|(%{uid}=%{email})(mail=%{email}))
-
-# the following is generated by start.sh
-END
-
-#SECRET_KEY_BASE=
-#OTP_SECRET=
-#VAPID_PRIVATE_KEY=
-#VAPID_PUBLIC_KEY=

nginx/mastodon.conf

@@ -5,10 +5,6 @@ map $http_upgrade $connection_upgrade {
 
 proxy_cache_path /run/nginx/cache levels=1:2 keys_zone=CACHE:10m inactive=7d max_size=1g;
 
-error_log /dev/stdout info;
-access_log /dev/stdout;
-client_body_temp_path /run/nginx/body 1 2;
-
 server {
   listen 8000;
   listen [::]:8000;

nginx_readonlyrootfs.conf

@@ -0,0 +1,8 @@
+# this is already in a http block
+
+client_body_temp_path /run/nginx;
+proxy_temp_path /run/nginx;
+fastcgi_temp_path /run/nginx;
+uwsgi_temp_path /run/nginx;
+scgi_temp_path /run/nginx;
+

start.sh

@@ -1,32 +1,47 @@
-#!/bin/bash -eu
+#!/bin/bash
-echo "=>Configuring mastodon<="
-bash /app/code/mastodon.env.template > /app/data/.env.production
 
-if ! [ -d /app/data/system ]; then
+set -eu
-    echo "=>First run, generating keys and setting up the DB<="
 
-    export RANDFILE=/tmp/.rnd
+mkdir -p /tmp/mastodon /app/data/system /run/mastodon
-    echo -e "SECRET_KEY_BASE=$(openssl rand -hex 64)\nOTP_SECRET=$(openssl rand -hex 64)" | \
-        tee /app/data/.keys.env >> /app/data/.env.production
 
-    HOME=/app/data bundle exec rake mastodon:webpush:generate_vapid_key | \
+if [[ ! -f /app/data/env.production ]]; then
-        tee -a /app/data/.keys.env >> /app/data/.env.production
+    echo "==> Copying env template on first run"
-
+    cp /app/code/env.template /app/data/env.production
-    SAFETY_ASSURED=1 HOME=/app/data bundle exec rails db:schema:load db:seed
-
-    # the app writes to the following dirs:
-    mkdir -p /app/data/system && chown cloudron:cloudron /app/data/system
-
-else
-    cat /app/data/.keys.env >> /app/data/.env.production
 fi
 
-echo "=>Starting mastodon<="
+echo "==> Configuring mastodon"
+sed -e "s/DB_HOST=.*/DB_HOST=${CLOUDRON_POSTGRESQL_HOST}/g" \
+    -e "s/DB_PORT=.*/DB_PORT=${CLOUDRON_POSTGRESQL_PORT}/g" \
+    -e "s/DB_NAME=.*/DB_NAME=${CLOUDRON_POSTGRESQL_DATABASE}/g" \
+    -e "s/DB_USER=.*/DB_USER=${CLOUDRON_POSTGRESQL_USERNAME}/g" \
+    -e "s/DB_PASS=.*/DB_PASS=${CLOUDRON_POSTGRESQL_PASSWORD}/g" \
+    -e "s/REDIS_HOST=.*/REDIS_HOST=${CLOUDRON_REDIS_HOST}/g" \
+    -e "s/REDIS_PORT=.*/REDIS_PORT=${CLOUDRON_REDIS_PORT}/g" \
+    -e "s/REDIS_PASSWORD=.*/REDIS_PASSWORD=${CLOUDRON_REDIS_PASSWORD}/g" \
+    -e "s/SMTP_SERVER=.*/SMTP_SERVER=${CLOUDRON_MAIL_SMTP_SERVER}/g" \
+    -e "s/SMTP_PORT=.*/SMTP_PORT=${CLOUDRON_MAIL_SMTP_PORT}/g" \
+    -e "s/SMTP_FROM_ADDRESS=.*/SMTP_FROM_ADDRESS=${CLOUDRON_MAIL_FROM}/g" \
+    -e "s/SMTP_LOGIN=.*/SMTP_LOGIN=${CLOUDRON_MAIL_SMTP_USERNAME}/g" \
+    -e "s/SMTP_PASSWORD=.*/SMTP_PASSWORD=${CLOUDRON_MAIL_SMTP_PASSWORD}/g" \
+    -e "s/LOCAL_DOMAIN=.*/LOCAL_DOMAIN=${CLOUDRON_APP_DOMAIN}/g" \
+    -i /app/data/env.production
 
-SUDO='sudo -u cloudron -H -E'
+if grep -q "^SECRET_KEY_BASE=$" /app/data/env.production; then
-PORT=3000 $SUDO bundle exec puma -C config/puma.rb &
+    echo "==> Generating secrets"
-PORT=4000 STREAMING_CLUSTER_NUM=1 $SUDO npm run start &
+    export RANDFILE=/tmp/.rnd
-DB_POOL=25 MALLOC_ARENA_MAX=2 $SUDO bundle exec sidekiq -c 25 &
+    sed -i -e "s/SECRET_KEY_BASE=.*/SECRET_KEY_BASE=$(openssl rand -hex 64)/" \
+        -e "s/OTP_SECRET=.*/OTP_SECRET=$(openssl rand -hex 64)/" \
+        /app/data/env.production
+
+    echo "==> Generating vapid keys"
+    HOME=/app/data bundle exec rake mastodon:webpush:generate_vapid_key >> /app/data/env.production
+
+    echo "==> Init database"
+    HOME=/app/data SAFETY_ASSURED=1 bundle exec rails db:schema:load db:seed
+fi
+
+chown -R cloudron:cloudron /app/data /tmp/mastodon /run/mastodon
+
+echo "==> Starting mastodon"
+exec /usr/bin/supervisord --configuration /etc/supervisor/supervisord.conf --nodaemon -i Mastodon
 
-mkdir -p /run/nginx/log /run/nginx/body /run/nginx/cache
-nginx -g 'daemon off;'

supervisor/nginx.conf

@@ -0,0 +1,12 @@
+[program:nginx]
+priority=100
+directory=/tmp
+command=/usr/sbin/nginx -g "daemon off;"
+user=root
+autostart=true
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+

supervisor/puma.conf

@@ -0,0 +1,13 @@
+[program:puma]
+priority=10
+directory=/app/code
+environment=HOME=/app/code,RAILS_ENV=production,PORT=3000
+command=bundle exec puma -C config/puma.rb
+user=cloudron
+autostart=true
+autorestart=true
+stopsignal=QUIT
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

supervisor/sidekiq.conf

@@ -0,0 +1,12 @@
+[program:sidekiq]
+priority=10
+directory=/app/code
+environment=HOME=/app/code,DB_POOL=25,RAILS_ENV=production,MALLOC_ARENA_MAX=2
+command=bundle exec sidekiq -c 2 -e production
+user=cloudron
+autostart=true
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

supervisor/streaming.conf

@@ -0,0 +1,13 @@
+[program:streaming]
+priority=10
+directory=/app/code
+environment=HOME=/app/code,NODE_ENV=production,PORT=4000,STREAMING_CLUSTER_NUM=1
+command=node /app/code/streaming
+user=cloudron
+autostart=true
+autorestart=true
+stopsignal=QUIT
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0