From b2dfe539d02b23b4750ecb6d2920ba99153f8978 Mon Sep 17 00:00:00 2001 From: Girish Ramakrishnan Date: Wed, 17 Jul 2019 14:14:32 -0700 Subject: [PATCH] Use supervisor and fixup env generation --- CloudronManifest.json | 5 ++- Dockerfile | 67 ++++++++++++++++--------------- docker-compose.yml | 33 --------------- env.template | 34 ++++++++++++++++ mastodon.env.template | 42 ------------------- nginx.conf => nginx/mastodon.conf | 4 -- nginx_readonlyrootfs.conf | 8 ++++ start.sh | 65 ++++++++++++++++++------------ supervisor/nginx.conf | 12 ++++++ supervisor/puma.conf | 13 ++++++ supervisor/sidekiq.conf | 12 ++++++ supervisor/streaming.conf | 13 ++++++ 12 files changed, 169 insertions(+), 139 deletions(-) delete mode 100644 docker-compose.yml create mode 100644 env.template delete mode 100644 mastodon.env.template rename nginx.conf => nginx/mastodon.conf (96%) create mode 100644 nginx_readonlyrootfs.conf create mode 100644 supervisor/nginx.conf create mode 100644 supervisor/puma.conf create mode 100644 supervisor/sidekiq.conf create mode 100644 supervisor/streaming.conf diff --git a/CloudronManifest.json b/CloudronManifest.json index 11dc00b..738df54 100644 --- a/CloudronManifest.json +++ b/CloudronManifest.json @@ -17,7 +17,7 @@ "ldap": {} }, "minBoxVersion": "1.8.1", - "manifestVersion": 1, + "manifestVersion": 2, "website": "https://joinmastodon.org/", "contactEmail": "syn+cloudron@syn.im", "icon": "logo.png", @@ -30,5 +30,6 @@ ], "documentationUrl": "https://docs.joinmastodon.org/administration/post-installation/", "postInstallMessage": "Follow the post-installation guide to promote users to admins", - "optionalSso": true + "optionalSso": true, + "minBoxVersion": "4.1.4" } diff --git a/Dockerfile b/Dockerfile index 0d8210f..89a7829 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,52 +3,53 @@ FROM cloudron/base:1.0.0@sha256:147a648a068a2e746644746bbfb42eb7a50d682437cead3c RUN mkdir -p /app/code WORKDIR /app/code -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +RUN apt-key adv --fetch-keys http://dl.yarnpkg.com/debian/pubkey.gpg && \ + echo "deb http://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list && \ + apt-get update && \ + apt-get install -y yarn libprotobuf-dev protobuf-compiler libidn11-dev libicu-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm5 libgdbm-dev libicu-dev libssl-dev libyaml-dev libreadline6-dev libxml2-dev libxslt1-dev && \ + rm -rf /var/cache/apt /var/lib/apt/lists -# the following does apt-get update -RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - +RUN mkdir -p /usr/local/node-10.15.3 && \ + curl -L https://nodejs.org/dist/v10.15.3/node-v10.15.3-linux-x64.tar.gz | tar zxf - --strip-components 1 -C /usr/local/node-10.15.3 -RUN apt-get install -y \ - imagemagick ffmpeg libpq-dev libxml2-dev libxslt1-dev file git-core \ - g++ libprotobuf-dev protobuf-compiler pkg-config nodejs gcc autoconf \ - bison build-essential libssl-dev libyaml-dev libreadline6-dev \ - zlib1g-dev libncurses5-dev libffi-dev libgdbm5 libgdbm-dev \ - nginx redis-server redis-tools postgresql postgresql-contrib \ - certbot yarn libidn11-dev libicu-dev libjemalloc-dev \ - ruby2.5 +ENV PATH /usr/local/node-10.15.3/bin:$PATH -RUN gem update --system -RUN gem install bundler +RUN gem install --no-document bundler -v 1.17.3 -RUN rm -r /etc/nginx/sites-enabled/default /var/lib/nginx /var/log/nginx -RUN mkdir -p /run/nginx && ln -fs /run/nginx /var/lib/nginx && ln -fs /run/nginx/log /var/log/nginx - -RUN git init && \ - git remote add origin https://github.com/tootsuite/mastodon.git && \ - git fetch --depth=1 origin $(git ls-remote --tags | grep refs/tags | grep -v 'rc[0-9]*$' | cut -f2 | sort -V | tail -n 1 | cut -d '/' -f3-) && \ - git checkout FETCH_HEAD +ARG VERSION=2.9.2 +ENV RAILS_ENV production +ENV NODE_ENV production +RUN curl -L https://github.com/tootsuite/mastodon/archive/v${VERSION}.tar.gz | tar -xz --strip-components 1 -f - && \ + bundle install --deployment --without test development && \ + yarn install --pure-lockfile COPY patches /app/code/patches RUN for patch in /app/code/patches/*; do patch -N -p0 < $patch; done -RUN bundle install -j$(getconf _NPROCESSORS_ONLN) --deployment --without development test && \ - yarn install --pure-lockfile - -ENV GEM_PATH=/app/code/vendor/bundle/ruby/2.5.0/gems/ RAILS_ENV=production NODE_ENV=production - # secret keys are not built into assets, so precompiling is safe to do here # (these variables are required by rake though) RUN SECRET_KEY_BASE=insecure.secret_key_base OTP_SECRET=insecure.otp_secret \ bundle exec rake assets:precompile -RUN ln -fs /app/data/.env.production /app/code/.env.production -RUN ln -fs /app/data/bullet.log /app/code/log/bullet.log +# add nginx config +USER root +RUN rm /etc/nginx/sites-enabled/* +RUN ln -sf /dev/stdout /var/log/nginx/access.log +RUN ln -sf /dev/stderr /var/log/nginx/error.log +ADD nginx_readonlyrootfs.conf /etc/nginx/conf.d/readonlyrootfs.conf +COPY nginx/mastodon.conf /etc/nginx/sites-available/mastodon +RUN ln -s /etc/nginx/sites-available/mastodon /etc/nginx/sites-enabled/mastodon + +# add supervisor configs +ADD supervisor/* /etc/supervisor/conf.d/ +RUN ln -sf /run/mastodon/supervisord.log /var/log/supervisor/supervisord.log + +RUN ln -fs /app/data/env.production /app/code/.env.production +RUN ln -fs /run/mastodon/bullet.log /app/code/log/bullet.log RUN ln -fs /app/data/system /app/code/public/system -RUN rm -rf /app/code/tmp && ln -fs /tmp /app/code/tmp +RUN rm -rf /app/code/tmp && ln -fs /tmp/mastodon /app/code/tmp -CMD /app/code/start.sh +COPY start.sh env.template /app/code/ + +CMD [ "/app/code/start.sh" ] -COPY nginx.conf /etc/nginx/sites-enabled/mastodon -COPY mastodon.env.template /app/code -COPY start.sh /app/code diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 5eb2c71..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: '2.4' - -services: - mastodon: - build: . - ports: - - 3000 - - 4000 - - 80:8000 - environment: - - LOCAL_DOMAIN=localhost - - LOCAL_HTTPS=false - - POSTGRESQL_HOST=postgres - - POSTGRESQL_DATABASE=postgres - - POSTGRESQL_USERNAME=postgres - - POSTGRESQL_PASSWORD=postgres - - REDIS_HOST=redis - volumes: - - data:/app/data - tmpfs: - - /run - - /tmp - read_only: true - postgres: - image: postgres - environment: - - POSTGRES_PASSWORD=postgres - - redis: - image: redis - -volumes: - data: \ No newline at end of file diff --git a/env.template b/env.template new file mode 100644 index 0000000..8d779eb --- /dev/null +++ b/env.template @@ -0,0 +1,34 @@ +SINGLE_USER_MODE=false + +LOCAL_DOMAIN= + +DB_HOST= +DB_PORT= +DB_NAME= +DB_USER= +DB_PASS= + +REDIS_HOST= +REDIS_PORT= +REDIS_PASSWORD= + +SMTP_SERVER= +SMTP_PORT= +SMTP_FROM_ADDRESS= +SMTP_LOGIN= +SMTP_PASSWORD= +SMTP_AUTH_METHOD=plain +SMTP_OPENSSL_VERIFY_MODE=none + +# LDAP_ENABLED=$([ -z "${CLOUDRON_LDAP_SERVER}" ] && echo "false" || echo "true") +# LDAP_HOST=${CLOUDRON_LDAP_SERVER} +# LDAP_PORT=${CLOUDRON_LDAP_PORT} +# LDAP_BASE=${CLOUDRON_LDAP_USERS_BASE_DN} +# LDAP_BIND_DN=${CLOUDRON_LDAP_BIND_DN} +# LDAP_PASSWORD=${CLOUDRON_LDAP_BIND_PASSWORD} +# LDAP_UID=username +# LDAP_SEARCH_FILTER=(|(%{uid}=%{email})(mail=%{email})) + +SECRET_KEY_BASE= +OTP_SECRET= + diff --git a/mastodon.env.template b/mastodon.env.template deleted file mode 100644 index 40151fe..0000000 --- a/mastodon.env.template +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -cat <"}" -SMTP_AUTH_METHOD=plain -SMTP_LOGIN=${MAIL_SMTP_USERNAME} -SMTP_PASSWORD=${MAIL_SMTP_PASSWORD} -SMTP_OPENSSL_VERIFY_MODE=none - -LDAP_ENABLED=$([ -z "$LDAP_SERVER" ] && echo "false" || echo "true") -LDAP_HOST=${LDAP_SERVER} -LDAP_PORT=${LDAP_PORT} -LDAP_BASE=${LDAP_USERS_BASE_DN} -LDAP_BIND_DN=${LDAP_BIND_DN} -LDAP_PASSWORD=${LDAP_BIND_PASSWORD} -LDAP_UID=username -LDAP_SEARCH_FILTER=(|(%{uid}=%{email})(mail=%{email})) - -# the following is generated by start.sh -END - -#SECRET_KEY_BASE= -#OTP_SECRET= -#VAPID_PRIVATE_KEY= -#VAPID_PUBLIC_KEY= \ No newline at end of file diff --git a/nginx.conf b/nginx/mastodon.conf similarity index 96% rename from nginx.conf rename to nginx/mastodon.conf index 7adb57b..357f0dc 100644 --- a/nginx.conf +++ b/nginx/mastodon.conf @@ -5,10 +5,6 @@ map $http_upgrade $connection_upgrade { proxy_cache_path /run/nginx/cache levels=1:2 keys_zone=CACHE:10m inactive=7d max_size=1g; -error_log /dev/stdout info; -access_log /dev/stdout; -client_body_temp_path /run/nginx/body 1 2; - server { listen 8000; listen [::]:8000; diff --git a/nginx_readonlyrootfs.conf b/nginx_readonlyrootfs.conf new file mode 100644 index 0000000..682b427 --- /dev/null +++ b/nginx_readonlyrootfs.conf @@ -0,0 +1,8 @@ +# this is already in a http block + +client_body_temp_path /run/nginx; +proxy_temp_path /run/nginx; +fastcgi_temp_path /run/nginx; +uwsgi_temp_path /run/nginx; +scgi_temp_path /run/nginx; + diff --git a/start.sh b/start.sh index 5909347..d8e46ff 100755 --- a/start.sh +++ b/start.sh @@ -1,32 +1,47 @@ -#!/bin/bash -eu -echo "=>Configuring mastodon<=" -bash /app/code/mastodon.env.template > /app/data/.env.production +#!/bin/bash -if ! [ -d /app/data/system ]; then - echo "=>First run, generating keys and setting up the DB<=" +set -eu - export RANDFILE=/tmp/.rnd - echo -e "SECRET_KEY_BASE=$(openssl rand -hex 64)\nOTP_SECRET=$(openssl rand -hex 64)" | \ - tee /app/data/.keys.env >> /app/data/.env.production +mkdir -p /tmp/mastodon /app/data/system /run/mastodon - HOME=/app/data bundle exec rake mastodon:webpush:generate_vapid_key | \ - tee -a /app/data/.keys.env >> /app/data/.env.production - - SAFETY_ASSURED=1 HOME=/app/data bundle exec rails db:schema:load db:seed - - # the app writes to the following dirs: - mkdir -p /app/data/system && chown cloudron:cloudron /app/data/system - -else - cat /app/data/.keys.env >> /app/data/.env.production +if [[ ! -f /app/data/env.production ]]; then + echo "==> Copying env template on first run" + cp /app/code/env.template /app/data/env.production fi -echo "=>Starting mastodon<=" +echo "==> Configuring mastodon" +sed -e "s/DB_HOST=.*/DB_HOST=${CLOUDRON_POSTGRESQL_HOST}/g" \ + -e "s/DB_PORT=.*/DB_PORT=${CLOUDRON_POSTGRESQL_PORT}/g" \ + -e "s/DB_NAME=.*/DB_NAME=${CLOUDRON_POSTGRESQL_DATABASE}/g" \ + -e "s/DB_USER=.*/DB_USER=${CLOUDRON_POSTGRESQL_USERNAME}/g" \ + -e "s/DB_PASS=.*/DB_PASS=${CLOUDRON_POSTGRESQL_PASSWORD}/g" \ + -e "s/REDIS_HOST=.*/REDIS_HOST=${CLOUDRON_REDIS_HOST}/g" \ + -e "s/REDIS_PORT=.*/REDIS_PORT=${CLOUDRON_REDIS_PORT}/g" \ + -e "s/REDIS_PASSWORD=.*/REDIS_PASSWORD=${CLOUDRON_REDIS_PASSWORD}/g" \ + -e "s/SMTP_SERVER=.*/SMTP_SERVER=${CLOUDRON_MAIL_SMTP_SERVER}/g" \ + -e "s/SMTP_PORT=.*/SMTP_PORT=${CLOUDRON_MAIL_SMTP_PORT}/g" \ + -e "s/SMTP_FROM_ADDRESS=.*/SMTP_FROM_ADDRESS=${CLOUDRON_MAIL_FROM}/g" \ + -e "s/SMTP_LOGIN=.*/SMTP_LOGIN=${CLOUDRON_MAIL_SMTP_USERNAME}/g" \ + -e "s/SMTP_PASSWORD=.*/SMTP_PASSWORD=${CLOUDRON_MAIL_SMTP_PASSWORD}/g" \ + -e "s/LOCAL_DOMAIN=.*/LOCAL_DOMAIN=${CLOUDRON_APP_DOMAIN}/g" \ + -i /app/data/env.production -SUDO='sudo -u cloudron -H -E' -PORT=3000 $SUDO bundle exec puma -C config/puma.rb & -PORT=4000 STREAMING_CLUSTER_NUM=1 $SUDO npm run start & -DB_POOL=25 MALLOC_ARENA_MAX=2 $SUDO bundle exec sidekiq -c 25 & +if grep -q "^SECRET_KEY_BASE=$" /app/data/env.production; then + echo "==> Generating secrets" + export RANDFILE=/tmp/.rnd + sed -i -e "s/SECRET_KEY_BASE=.*/SECRET_KEY_BASE=$(openssl rand -hex 64)/" \ + -e "s/OTP_SECRET=.*/OTP_SECRET=$(openssl rand -hex 64)/" \ + /app/data/env.production + + echo "==> Generating vapid keys" + HOME=/app/data bundle exec rake mastodon:webpush:generate_vapid_key >> /app/data/env.production + + echo "==> Init database" + HOME=/app/data SAFETY_ASSURED=1 bundle exec rails db:schema:load db:seed +fi + +chown -R cloudron:cloudron /app/data /tmp/mastodon /run/mastodon + +echo "==> Starting mastodon" +exec /usr/bin/supervisord --configuration /etc/supervisor/supervisord.conf --nodaemon -i Mastodon -mkdir -p /run/nginx/log /run/nginx/body /run/nginx/cache -nginx -g 'daemon off;' diff --git a/supervisor/nginx.conf b/supervisor/nginx.conf new file mode 100644 index 0000000..05cb8b9 --- /dev/null +++ b/supervisor/nginx.conf @@ -0,0 +1,12 @@ +[program:nginx] +priority=100 +directory=/tmp +command=/usr/sbin/nginx -g "daemon off;" +user=root +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + diff --git a/supervisor/puma.conf b/supervisor/puma.conf new file mode 100644 index 0000000..9a8f3bf --- /dev/null +++ b/supervisor/puma.conf @@ -0,0 +1,13 @@ +[program:puma] +priority=10 +directory=/app/code +environment=HOME=/app/code,RAILS_ENV=production,PORT=3000 +command=bundle exec puma -C config/puma.rb +user=cloudron +autostart=true +autorestart=true +stopsignal=QUIT +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/supervisor/sidekiq.conf b/supervisor/sidekiq.conf new file mode 100644 index 0000000..2b60959 --- /dev/null +++ b/supervisor/sidekiq.conf @@ -0,0 +1,12 @@ +[program:sidekiq] +priority=10 +directory=/app/code +environment=HOME=/app/code,DB_POOL=25,RAILS_ENV=production,MALLOC_ARENA_MAX=2 +command=bundle exec sidekiq -c 2 -e production +user=cloudron +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/supervisor/streaming.conf b/supervisor/streaming.conf new file mode 100644 index 0000000..2045ecb --- /dev/null +++ b/supervisor/streaming.conf @@ -0,0 +1,13 @@ +[program:streaming] +priority=10 +directory=/app/code +environment=HOME=/app/code,NODE_ENV=production,PORT=4000,STREAMING_CLUSTER_NUM=1 +command=node /app/code/streaming +user=cloudron +autostart=true +autorestart=true +stopsignal=QUIT +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0