From 6aad1c9250475c7b2d74e029a1f2db57268aad9d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 22 Mar 2020 11:21:19 -0700 Subject: [PATCH 1/7] Dockerize app --- .env.example | 19 ++++++++++++------- Dockerfile | 7 +++++++ docker-compose.yml | 19 +++++++++++++++++++ fedireads/settings.py | 26 ++++++++------------------ 4 files changed, 46 insertions(+), 25 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/.env.example b/.env.example index 9f3dd381..141dc376 100644 --- a/.env.example +++ b/.env.example @@ -1,18 +1,23 @@ # SECURITY WARNING: keep the secret key used in production secret! -export SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr" +SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr" # SECURITY WARNING: don't run with debug turned on in production! -export DEBUG=true +DEBUG=true -export DOMAIN=your.domain.here +DOMAIN=your.domain.here ## Leave unset to allow all hosts -# export ALLOWED_HOSTS="localhost,127.0.0.1,[::1]" +# ALLOWED_HOSTS="localhost,127.0.0.1,[::1]" -export OL_URL="https://openlibrary.org" +OL_URL="https://openlibrary.org" ## Database backend to use. ## Default is postgres, sqlite is for dev quickstart only (NOT production!!!) -export FEDIREADS_DATABASE_BACKEND=postgres +FEDIREADS_DATABASE_BACKEND=postgres + +MEDIA_ROOT="images/" + +POSTGRES_PASSWORD="fedireads" +POSTGRES_USER="postgres" +POSTGRES_DB="postgres" -export MEDIA_ROOT="images/" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..cd56e6ee --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3 +ENV PYTHONUNBUFFERED 1 +RUN mkdir /code +WORKDIR /code +COPY requirements.txt /code/ +RUN pip install -r requirements.txt +COPY ./fedireads /code diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..9545ef94 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3' + +services: + db: + image: postgres + env_file: .env + volumes: + - pgdata:/var/lib/posgresql/data + web: + build: . + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + ports: + - "8000:8000" + depends_on: + - db +volumes: + pgdata: diff --git a/fedireads/settings.py b/fedireads/settings.py index c5cdfd9b..65d86d50 100644 --- a/fedireads/settings.py +++ b/fedireads/settings.py @@ -70,25 +70,15 @@ WSGI_APPLICATION = 'fedireads.wsgi.application' # Database # https://docs.djangoproject.com/en/2.0/ref/settings/#databases -FEDIREADS_DATABASE_BACKEND = env('FEDIREADS_DATABASE_BACKEND', 'postgres') - -FEDIREADS_DBS = { - 'postgres': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'fedireads', - 'USER': 'fedireads', - 'PASSWORD': 'fedireads', - 'HOST': '', - 'PORT': 5432 - }, - 'sqlite': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'fedireads.db') - } -} - DATABASES = { - 'default': FEDIREADS_DBS[FEDIREADS_DATABASE_BACKEND] + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'postgres', + 'USER': 'postgres', + 'PASSWORD': '"%s"' % env('POSTGRES_PASSWORD', 'fedireads'), + 'HOST': 'db', + 'PORT': 5432 + } } LOGIN_URL = '/login/' From 1a6ae00a8093f620787f384b5b8c778e3027c5aa Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 22 Mar 2020 11:40:50 -0700 Subject: [PATCH 2/7] Updated readme for docker --- README.md | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index e63912df..b4a33ba2 100644 --- a/README.md +++ b/README.md @@ -45,27 +45,14 @@ This project is still in its very early stages, but these are the higher-level f But this isn't a set in stone, unchangeable list, so if you have ideas about how this could be tweaked, changed, or improved, please open an issue and start a conversation about it. ## Setting up the developer environment -You will need postgres installed and running on your computer. +Install docker and run: ``` bash -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -createdb fedireads +docker-compose build +docker-compose up +docker-compose exec web python manage.py migrate ``` -Create the psql user in `psql fedireads`: -``` psql -CREATE ROLE fedireads WITH LOGIN PASSWORD 'fedireads'; -GRANT ALL PRIVILEGES ON DATABASE fedireads TO fedireads; -``` - -Initialize the database (or, more specifically, delete the existing database, run migrations, and start fresh): -``` bash -./rebuilddb.sh -``` -This creates two users, `mouse` with password `password123` and `rat` with password `ratword`. - And go to the app at `localhost:8000` For most testing, you'll want to use ngrok. Remember to set the DOMAIN in `.env` to your ngrok domain. From dd18433faa4caeddff6bf480ed3939338b5e1757 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 22 Mar 2020 14:33:26 -0700 Subject: [PATCH 3/7] Adds celery and rabbitmq --- .env.example | 3 +++ Dockerfile | 8 ++++---- docker-compose.yml | 29 ++++++++++++++++++++++++++++- fedireads/__init__.py | 8 ++++++++ fedireads/celery.py | 24 ++++++++++++++++++++++++ fedireads/settings.py | 9 +++++++-- requirements.txt | 1 + 7 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 fedireads/celery.py diff --git a/.env.example b/.env.example index 141dc376..83290d9f 100644 --- a/.env.example +++ b/.env.example @@ -21,3 +21,6 @@ POSTGRES_PASSWORD="fedireads" POSTGRES_USER="postgres" POSTGRES_DB="postgres" +RABBITMQ_DEFAULT_USER=rabbit +RABBITMQ_DEFAULT_PASS=changeme +CELERY_BROKER=amqp://rabbit:changeme@rabbitmq:5672 diff --git a/Dockerfile b/Dockerfile index cd56e6ee..c845035f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3 ENV PYTHONUNBUFFERED 1 -RUN mkdir /code -WORKDIR /code -COPY requirements.txt /code/ +RUN mkdir /app +WORKDIR /app +COPY requirements.txt /app/ RUN pip install -r requirements.txt -COPY ./fedireads /code +COPY ./fedireads /app diff --git a/docker-compose.yml b/docker-compose.yml index 9545ef94..6be45726 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,14 +6,41 @@ services: env_file: .env volumes: - pgdata:/var/lib/posgresql/data + networks: + - main web: build: . command: python manage.py runserver 0.0.0.0:8000 volumes: - - .:/code + - .:/app ports: - "8000:8000" depends_on: - db + - celery_worker + networks: + - main + rabbitmq: + env_file: .env + image: rabbitmq:latest + networks: + - main + ports: + - "5672:5672" + restart: on-failure + celery_worker: + env_file: .env + build: . + networks: + - main + command: celery -A fedireads worker -l info + volumes: + - .:/app + depends_on: + - db + - rabbitmq + restart: on-failure volumes: pgdata: +networks: + main: diff --git a/fedireads/__init__.py b/fedireads/__init__.py index e69de29b..b23e378a 100644 --- a/fedireads/__init__.py +++ b/fedireads/__init__.py @@ -0,0 +1,8 @@ +from __future__ import absolute_import, unicode_literals + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ('celery_app',) + diff --git a/fedireads/celery.py b/fedireads/celery.py new file mode 100644 index 00000000..7c26dc06 --- /dev/null +++ b/fedireads/celery.py @@ -0,0 +1,24 @@ +from __future__ import absolute_import, unicode_literals + +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fedireads.settings') + +app = Celery('fedireads') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() + + +@app.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) diff --git a/fedireads/settings.py b/fedireads/settings.py index 65d86d50..7612b716 100644 --- a/fedireads/settings.py +++ b/fedireads/settings.py @@ -17,11 +17,15 @@ SECRET_KEY = env('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! DEBUG = env.bool('DEBUG', True) -# TODO: annoying that I keep changing and re-commiting this DOMAIN = env('DOMAIN') ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', ['*']) OL_URL = env('OL_URL') +# celery/rebbitmq +CELERY_BROKER_URL = env('CELERY_BROKER') +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' + # Application definition INSTALLED_APPS = [ @@ -33,6 +37,7 @@ INSTALLED_APPS = [ 'django.contrib.staticfiles', 'django.contrib.humanize', 'fedireads', + 'celery', ] MIDDLEWARE = [ @@ -75,7 +80,7 @@ DATABASES = { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'postgres', 'USER': 'postgres', - 'PASSWORD': '"%s"' % env('POSTGRES_PASSWORD', 'fedireads'), + 'PASSWORD': '%s' % env('POSTGRES_PASSWORD', 'fedireads'), 'HOST': 'db', 'PORT': 5432 } diff --git a/requirements.txt b/requirements.txt index 1320435c..0375a27d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +celery==4.4.2 Django==3.0.3 django-model-utils==4.0.0 environs==7.2.0 From b97006d43c7f248a353dc81dc518fe001b1cf703 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 22 Mar 2020 17:13:52 -0700 Subject: [PATCH 4/7] Adds celery broker to settings --- fedireads/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fedireads/settings.py b/fedireads/settings.py index 7612b716..23b2c1fd 100644 --- a/fedireads/settings.py +++ b/fedireads/settings.py @@ -25,6 +25,7 @@ OL_URL = env('OL_URL') CELERY_BROKER_URL = env('CELERY_BROKER') CELERY_ACCEPT_CONTENT = ['json'] CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_BACKEND = 'amqp' # Application definition From 0c766f9867b6faf4134d3f5a6d61cddbcf00d3e4 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 27 Mar 2020 13:04:06 -0700 Subject: [PATCH 5/7] Keep more development environment options --- .env.example | 4 ++-- README.md | 31 +++++++++++++++++++++++++++---- docker-compose.yml | 2 +- fedireads/settings.py | 21 ++++++++++++++++----- 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index 83290d9f..4d9815fe 100644 --- a/.env.example +++ b/.env.example @@ -18,8 +18,8 @@ FEDIREADS_DATABASE_BACKEND=postgres MEDIA_ROOT="images/" POSTGRES_PASSWORD="fedireads" -POSTGRES_USER="postgres" -POSTGRES_DB="postgres" +POSTGRES_USER="fedireads" +POSTGRES_DB="fedireads" RABBITMQ_DEFAULT_USER=rabbit RABBITMQ_DEFAULT_PASS=changeme diff --git a/README.md b/README.md index b4a33ba2..68156d06 100644 --- a/README.md +++ b/README.md @@ -45,19 +45,42 @@ This project is still in its very early stages, but these are the higher-level f But this isn't a set in stone, unchangeable list, so if you have ideas about how this could be tweaked, changed, or improved, please open an issue and start a conversation about it. ## Setting up the developer environment +You will need postgres installed and running on your computer. -Install docker and run: ``` bash -docker-compose build -docker-compose up -docker-compose exec web python manage.py migrate +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +createdb fedireads ``` +Create the psql user in `psql fedireads`: +``` psql +CREATE ROLE fedireads WITH LOGIN PASSWORD 'fedireads'; +GRANT ALL PRIVILEGES ON DATABASE fedireads TO fedireads; +``` + +Initialize the database (or, more specifically, delete the existing database, run migrations, and start fresh): +``` bash +./rebuilddb.sh +``` +This creates two users, `mouse` with password `password123` and `rat` with password `ratword`. + And go to the app at `localhost:8000` For most testing, you'll want to use ngrok. Remember to set the DOMAIN in `.env` to your ngrok domain. +### Docker +You can also run the application in a docker container. You'll have to install the Docker and docker-compose: + +```bash +docker-compose build +docker-compose up +docker-compose exec web python manage.py migrate +``` + + ## Project structure All the url routing is in `fedireads/urls.py`. This includes the application views (your home page, user page, book page, etc), application endpoints (things that happen when you click buttons), and federation api endpoints (inboxes, outboxes, webfinger, etc). diff --git a/docker-compose.yml b/docker-compose.yml index 6be45726..a119f25d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: image: postgres env_file: .env volumes: - - pgdata:/var/lib/posgresql/data + - pgdata:/var/lib/postgresql/data networks: - main web: diff --git a/fedireads/settings.py b/fedireads/settings.py index 23b2c1fd..c4ce0b2c 100644 --- a/fedireads/settings.py +++ b/fedireads/settings.py @@ -76,17 +76,28 @@ WSGI_APPLICATION = 'fedireads.wsgi.application' # Database # https://docs.djangoproject.com/en/2.0/ref/settings/#databases -DATABASES = { - 'default': { +FEDIREADS_DATABASE_BACKEND = env('FEDIREADS_DATABASE_BACKEND', 'postgres') + +FEDIREADS_DBS = { + 'postgres': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'postgres', - 'USER': 'postgres', - 'PASSWORD': '%s' % env('POSTGRES_PASSWORD', 'fedireads'), + 'NAME': env('POSTGRES_DB', 'fedireads'), + 'USER': env('POSTGRES_USER', 'fedireads'), + 'PASSWORD': env('POSTGRES_PASSWORD', 'fedireads'), 'HOST': 'db', 'PORT': 5432 + }, + 'sqlite': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'fedireads.db') } } +DATABASES = { + 'default': FEDIREADS_DBS[FEDIREADS_DATABASE_BACKEND] +} + + LOGIN_URL = '/login/' AUTH_USER_MODEL = 'fedireads.User' From b71a9880076a967e88a1a4e01914775518beb939 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 27 Mar 2020 13:27:32 -0700 Subject: [PATCH 6/7] consistent formatting in .env.example --- .env.example | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 4d9815fe..509331ec 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr" +SECRET_KEY=7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr # SECURITY WARNING: don't run with debug turned on in production! DEBUG=true @@ -9,17 +9,17 @@ DOMAIN=your.domain.here ## Leave unset to allow all hosts # ALLOWED_HOSTS="localhost,127.0.0.1,[::1]" -OL_URL="https://openlibrary.org" +OL_URL=https://openlibrary.org ## Database backend to use. ## Default is postgres, sqlite is for dev quickstart only (NOT production!!!) FEDIREADS_DATABASE_BACKEND=postgres -MEDIA_ROOT="images/" +MEDIA_ROOT=images/ -POSTGRES_PASSWORD="fedireads" -POSTGRES_USER="fedireads" -POSTGRES_DB="fedireads" +POSTGRES_PASSWORD=fedireads +POSTGRES_USER=fedireads +POSTGRES_DB=fedireads RABBITMQ_DEFAULT_USER=rabbit RABBITMQ_DEFAULT_PASS=changeme From fdefd06e041fe8e00bc576fd9030441b8249c3d2 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 28 Mar 2020 11:34:41 -0700 Subject: [PATCH 7/7] Adds init.dbdocker instructions to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 68156d06..1fa8cd3e 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ You can also run the application in a docker container. You'll have to install t docker-compose build docker-compose up docker-compose exec web python manage.py migrate +docker-compose exec web python manage.py shell -c 'import init_db' ```