From 8c6c28c7d822d9bf24abc6a2292388fdb5fca0fc Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 10:29:33 -0800 Subject: [PATCH 01/12] Move initdb into a management command --- bookwyrm/management/commands/initdb.py | 100 +++++++++++++++++++++++++ fr-dev | 13 +++- init_db.py | 91 ---------------------- rebuilddb.sh | 2 +- 4 files changed, 111 insertions(+), 95 deletions(-) create mode 100644 bookwyrm/management/commands/initdb.py delete mode 100644 init_db.py diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py new file mode 100644 index 000000000..f29ed1027 --- /dev/null +++ b/bookwyrm/management/commands/initdb.py @@ -0,0 +1,100 @@ +from django.core.management.base import BaseCommand, CommandError +from django.contrib.auth.models import Group, Permission +from django.contrib.contenttypes.models import ContentType + +from bookwyrm.models import Connector, User +from bookwyrm.settings import DOMAIN + +def init_groups(): + groups = ['admin', 'moderator', 'editor'] + for group in groups: + Group.objects.create(name=group) + +def init_permissions(): + permissions = [{ + 'codename': 'edit_instance_settings', + 'name': 'change the instance info', + 'groups': ['admin',] + }, { + 'codename': 'set_user_group', + 'name': 'change what group a user is in', + 'groups': ['admin', 'moderator'] + }, { + 'codename': 'control_federation', + 'name': 'control who to federate with', + 'groups': ['admin', 'moderator'] + }, { + 'codename': 'create_invites', + 'name': 'issue invitations to join', + 'groups': ['admin', 'moderator'] + }, { + 'codename': 'moderate_user', + 'name': 'deactivate or silence a user', + 'groups': ['admin', 'moderator'] + }, { + 'codename': 'moderate_post', + 'name': 'delete other users\' posts', + 'groups': ['admin', 'moderator'] + }, { + 'codename': 'edit_book', + 'name': 'edit book info', + 'groups': ['admin', 'moderator', 'editor'] + }] + + content_type = ContentType.objects.get_for_model(User) + for permission in permissions: + permission_obj = Permission.objects.create( + codename=permission['codename'], + name=permission['name'], + content_type=content_type, + ) + # add the permission to the appropriate groups + for group_name in permission['groups']: + Group.objects.get(name=group_name).permissions.add(permission_obj) + + # while the groups and permissions shouldn't be changed because the code + # depends on them, what permissions go with what groups should be editable + + +def init_connectors(): + Connector.objects.create( + identifier=DOMAIN, + name='Local', + local=True, + connector_file='self_connector', + base_url='https://%s' % DOMAIN, + books_url='https://%s/book' % DOMAIN, + covers_url='https://%s/images/covers' % DOMAIN, + search_url='https://%s/search?q=' % DOMAIN, + priority=1, + ) + + Connector.objects.create( + identifier='bookwyrm.social', + name='BookWyrm dot Social', + connector_file='bookwyrm_connector', + base_url='https://bookwyrm.social' , + books_url='https://bookwyrm.social/book', + covers_url='https://bookwyrm.social/images/covers', + search_url='https://bookwyrm.social/search?q=', + priority=2, + ) + + Connector.objects.create( + identifier='openlibrary.org', + name='OpenLibrary', + connector_file='openlibrary', + base_url='https://openlibrary.org', + books_url='https://openlibrary.org', + covers_url='https://covers.openlibrary.org', + search_url='https://openlibrary.org/search?q=', + priority=3, + ) + +class Command(BaseCommand): + help = 'Initializes the database with starter data' + + def handle(self, *args, **options): + init_groups() + init_permissions() + init_connectors() \ No newline at end of file diff --git a/fr-dev b/fr-dev index 56494473b..41c393507 100755 --- a/fr-dev +++ b/fr-dev @@ -12,7 +12,7 @@ case "$1" in ;; initdb) docker-compose exec web python manage.py migrate - docker-compose exec web python manage.py shell -c 'import init_db' + docker-compose exec web python manage.py initdb ;; resetdb) docker-compose stop web @@ -20,7 +20,7 @@ case "$1" in docker-compose exec db createdb -U fedireads fedireads docker-compose start web docker-compose exec web python manage.py migrate - docker-compose exec web python manage.py shell -c 'import init_db' + docker-compose exec web python manage.py initdb ;; makemigrations) docker-compose exec web python manage.py makemigrations @@ -47,7 +47,14 @@ case "$1" in collectstatic) docker-compose exec web python manage.py collectstatic --no-input ;; + build) + docker-compose build + ;; + clean) + docker-compose stop + docker-compose rm -f + ;; *) - echo "Unrecognised command. Try: up, initdb, resetdb, makemigrations, migrate, shell, dbshell, restart_celery, test, test_report" + echo "Unrecognised command. Try: build, clean, up, initdb, resetdb, makemigrations, migrate, shell, dbshell, restart_celery, test, test_report" ;; esac diff --git a/init_db.py b/init_db.py deleted file mode 100644 index e2ade67bd..000000000 --- a/init_db.py +++ /dev/null @@ -1,91 +0,0 @@ -''' starter data ''' -from django.contrib.auth.models import Group, Permission -from django.contrib.contenttypes.models import ContentType - -from bookwyrm.models import Connector, User -from bookwyrm.settings import DOMAIN - - -groups = ['admin', 'moderator', 'editor'] -for group in groups: - Group.objects.create(name=group) - -permissions = [{ - 'codename': 'edit_instance_settings', - 'name': 'change the instance info', - 'groups': ['admin',] - }, { - 'codename': 'set_user_group', - 'name': 'change what group a user is in', - 'groups': ['admin', 'moderator'] - }, { - 'codename': 'control_federation', - 'name': 'control who to federate with', - 'groups': ['admin', 'moderator'] - }, { - 'codename': 'create_invites', - 'name': 'issue invitations to join', - 'groups': ['admin', 'moderator'] - }, { - 'codename': 'moderate_user', - 'name': 'deactivate or silence a user', - 'groups': ['admin', 'moderator'] - }, { - 'codename': 'moderate_post', - 'name': 'delete other users\' posts', - 'groups': ['admin', 'moderator'] - }, { - 'codename': 'edit_book', - 'name': 'edit book info', - 'groups': ['admin', 'moderator', 'editor'] - }] - -content_type = ContentType.objects.get_for_model(User) -for permission in permissions: - permission_obj = Permission.objects.create( - codename=permission['codename'], - name=permission['name'], - content_type=content_type, - ) - # add the permission to the appropriate groups - for group_name in permission['groups']: - Group.objects.get(name=group_name).permissions.add(permission_obj) - -# while the groups and permissions shouldn't be changed because the code -# depends on them, what permissions go with what groups should be editable - - - -Connector.objects.create( - identifier=DOMAIN, - name='Local', - local=True, - connector_file='self_connector', - base_url='https://%s' % DOMAIN, - books_url='https://%s/book' % DOMAIN, - covers_url='https://%s/images/covers' % DOMAIN, - search_url='https://%s/search?q=' % DOMAIN, - priority=1, -) - -Connector.objects.create( - identifier='bookwyrm.social', - name='BookWyrm dot Social', - connector_file='bookwyrm_connector', - base_url='https://bookwyrm.social' , - books_url='https://bookwyrm.social/book', - covers_url='https://bookwyrm.social/images/covers', - search_url='https://bookwyrm.social/search?q=', - priority=2, -) - -Connector.objects.create( - identifier='openlibrary.org', - name='OpenLibrary', - connector_file='openlibrary', - base_url='https://openlibrary.org', - books_url='https://openlibrary.org', - covers_url='https://covers.openlibrary.org', - search_url='https://openlibrary.org/search?q=', - priority=3, -) diff --git a/rebuilddb.sh b/rebuilddb.sh index 99be0268e..d32e5dab4 100755 --- a/rebuilddb.sh +++ b/rebuilddb.sh @@ -21,5 +21,5 @@ fi python manage.py makemigrations fedireads python manage.py migrate -python manage.py shell < init_db.py +python manage.py initdb python manage.py runserver From 04d987a9a39a97a7d72c6fbe68102e5ebd1ba40e Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 11:42:41 -0800 Subject: [PATCH 02/12] Ignore coverage report --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 7d53fd2f5..1384056f2 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ # BookWyrm .env /images/ + +# Testing +.coverage \ No newline at end of file From 7eea1b27dadabc3f56250506f12aaf7ecca70ae1 Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 11:43:04 -0800 Subject: [PATCH 03/12] Use docker-compose run instead of exec so you don't have to be running containers to execute commands --- fr-dev | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/fr-dev b/fr-dev index 41c393507..bf1ea500b 100755 --- a/fr-dev +++ b/fr-dev @@ -3,6 +3,19 @@ set -e set -x +function clean { + docker-compose stop + docker-compose rm -f +} + +function rundb { + docker-compose run --rm db $@ +} + +function runweb { + docker-compose run --rm web $@ +} + case "$1" in up) docker-compose up --build @@ -11,41 +24,40 @@ case "$1" in docker-compose run --service-ports web ;; initdb) - docker-compose exec web python manage.py migrate - docker-compose exec web python manage.py initdb + runweb python manage.py migrate + runweb python manage.py initdb ;; resetdb) - docker-compose stop web - docker-compose exec db dropdb -U fedireads fedireads - docker-compose exec db createdb -U fedireads fedireads - docker-compose start web - docker-compose exec web python manage.py migrate - docker-compose exec web python manage.py initdb + clean + rundb dropdb -U fedireads fedireads + rundb createdb -U fedireads fedireads + runweb python manage.py migrate + runweb python manage.py initdb ;; makemigrations) - docker-compose exec web python manage.py makemigrations + runweb python manage.py makemigrations ;; migrate) - docker-compose exec web python manage.py migrate + runweb python manage.py migrate ;; shell) - docker-compose exec web python manage.py shell + runweb python manage.py shell ;; dbshell) - docker-compose exec db psql -U fedireads fedireads + rundb psql -U fedireads fedireads ;; restart_celery) docker-compose restart celery_worker ;; test) shift 1 - docker-compose exec web coverage run --source='.' --omit="*/test*,celerywyrm*,bookwyrm/migrations/*" manage.py test "$@" + runweb coverage run --source='.' --omit="*/test*,celerywyrm*,bookwyrm/migrations/*" manage.py test "$@" ;; test_report) - docker-compose exec web coverage report + runweb coverage report ;; collectstatic) - docker-compose exec web python manage.py collectstatic --no-input + runweb python manage.py collectstatic --no-input ;; build) docker-compose build From 391361c5e3edcfeced10b5e0e55705d6b569702d Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 11:44:52 -0800 Subject: [PATCH 04/12] Update readme to use run instead of exec --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4d67bd85e..5e16597c1 100644 --- a/README.md +++ b/README.md @@ -65,9 +65,8 @@ You'll have to install the Docker and docker-compose. When you're ready, run: ```bash docker-compose build -docker-compose up -docker-compose exec web python manage.py migrate -docker-compose exec web python manage.py shell -c 'import init_db' +docker-compose run --rm web python manage.py migrate +docker-compose run --rm web python manage.py initdb ``` ### Without Docker From f1816d2ef55882b0c42346cc52225c3613b734fb Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 11:45:21 -0800 Subject: [PATCH 05/12] Have clean command use clean function --- fr-dev | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fr-dev b/fr-dev index bf1ea500b..163cb4c6f 100755 --- a/fr-dev +++ b/fr-dev @@ -63,8 +63,7 @@ case "$1" in docker-compose build ;; clean) - docker-compose stop - docker-compose rm -f + clean ;; *) echo "Unrecognised command. Try: build, clean, up, initdb, resetdb, makemigrations, migrate, shell, dbshell, restart_celery, test, test_report" From 0176f5bbf883bda2f71539600685bfead5f8de12 Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 11:50:20 -0800 Subject: [PATCH 06/12] Pull initdb into a function; have db commands exec instead of run to ensure container is up --- fr-dev | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/fr-dev b/fr-dev index 163cb4c6f..9766ca88a 100755 --- a/fr-dev +++ b/fr-dev @@ -9,30 +9,35 @@ function clean { } function rundb { - docker-compose run --rm db $@ + docker-compose exec db $@ } function runweb { docker-compose run --rm web $@ } +function initdb { + runweb python manage.py migrate + runweb python manage.py initdb +} + case "$1" in up) docker-compose up --build ;; run) - docker-compose run --service-ports web + docker-compose run --rm --service-ports web ;; initdb) - runweb python manage.py migrate - runweb python manage.py initdb + initdb ;; resetdb) clean + docker-compose up --build -d rundb dropdb -U fedireads fedireads rundb createdb -U fedireads fedireads - runweb python manage.py migrate - runweb python manage.py initdb + initdb + clean ;; makemigrations) runweb python manage.py makemigrations From 58237b6984043dd3aeba232e8e9f7cdf15f0fb5d Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 11:52:08 -0800 Subject: [PATCH 07/12] Have runweb commands clean up after themselves --- fr-dev | 1 + 1 file changed, 1 insertion(+) diff --git a/fr-dev b/fr-dev index 9766ca88a..b9502b663 100755 --- a/fr-dev +++ b/fr-dev @@ -14,6 +14,7 @@ function rundb { function runweb { docker-compose run --rm web $@ + clean } function initdb { From 0b8a9cf1a65dae88a55a58c9b9b1a878d90daaaa Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 12:14:57 -0800 Subject: [PATCH 08/12] Add support for pytest --- .coveragerc | 2 ++ docker-compose.yml | 1 + fr-dev | 9 ++++++++- pytest.ini | 4 ++++ requirements.txt | 3 +++ 5 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 .coveragerc create mode 100644 pytest.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..35bf78f5f --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit = */test*,celerywyrm*,bookwyrm/migrations/* \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 18b58032c..08567ce33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,7 @@ services: - main web: build: . + env_file: .env command: python manage.py runserver 0.0.0.0:8000 volumes: - .:/app diff --git a/fr-dev b/fr-dev index b9502b663..39c8b1839 100755 --- a/fr-dev +++ b/fr-dev @@ -46,6 +46,9 @@ case "$1" in migrate) runweb python manage.py migrate ;; + bash) + runweb bash + ;; shell) runweb python manage.py shell ;; @@ -59,6 +62,10 @@ case "$1" in shift 1 runweb coverage run --source='.' --omit="*/test*,celerywyrm*,bookwyrm/migrations/*" manage.py test "$@" ;; + pytest) + shift 1 + runweb pytest "$@" + ;; test_report) runweb coverage report ;; @@ -72,6 +79,6 @@ case "$1" in clean ;; *) - echo "Unrecognised command. Try: build, clean, up, initdb, resetdb, makemigrations, migrate, shell, dbshell, restart_celery, test, test_report" + echo "Unrecognised command. Try: build, clean, up, initdb, resetdb, makemigrations, migrate, bash, shell, dbshell, restart_celery, test, pytest, test_report" ;; esac diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..c6a261d6c --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +DJANGO_SETTINGS_MODULE = bookwyrm.settings +python_files = tests.py test_*.py *_tests.py +addopts = --cov=bookwyrm --cov-config=.coveragerc \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e041ec25d..ff5848d56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,9 @@ Pillow>=7.1.0 psycopg2==2.8.4 pycryptodome==3.9.4 python-dateutil==2.8.1 +pytest_django==4.1.0 +pytest==6.1.2 +pytest-cov==2.10.1 redis==3.4.1 requests==2.22.0 responses==0.10.14 From e1e5dd8b0d610f05a227219dfa35e230b0a65412 Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 12:17:10 -0800 Subject: [PATCH 09/12] Add a dockerignore file and don't map some python files locally --- .dockerignore | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..3bf9f2c5b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +.git +.github +.pytest* \ No newline at end of file From 1f9891dcf5c7c0c149bbe7c5f0154c6bfbead26b Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 13:07:37 -0800 Subject: [PATCH 10/12] Create example marker to avoid tests that require external domain --- bookwyrm/tests/test_signing.py | 5 +++++ fr-dev | 2 +- pytest.ini | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py index 8653823d3..5170b4167 100644 --- a/bookwyrm/tests/test_signing.py +++ b/bookwyrm/tests/test_signing.py @@ -6,6 +6,8 @@ import pathlib import json import responses +import pytest + from django.test import TestCase, Client from django.utils.http import http_date @@ -167,6 +169,7 @@ class Signature(TestCase): response = self.send_test_request(sender=self.fake_remote) self.assertEqual(response.status_code, 401) + @pytest.mark.integration def test_changed_data(self): '''Message data must match the digest header.''' response = self.send_test_request( @@ -174,12 +177,14 @@ class Signature(TestCase): send_data=get_follow_data(self.mouse, self.cat)) self.assertEqual(response.status_code, 401) + @pytest.mark.integration def test_invalid_digest(self): response = self.send_test_request( self.mouse, digest='SHA-256=AAAAAAAAAAAAAAAAAA') self.assertEqual(response.status_code, 401) + @pytest.mark.integration def test_old_message(self): '''Old messages should be rejected to prevent replay attacks.''' response = self.send_test_request( diff --git a/fr-dev b/fr-dev index 39c8b1839..32dff5ff0 100755 --- a/fr-dev +++ b/fr-dev @@ -13,7 +13,7 @@ function rundb { } function runweb { - docker-compose run --rm web $@ + docker-compose run --rm web "$@" clean } diff --git a/pytest.ini b/pytest.ini index c6a261d6c..fa9dbc59f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,6 @@ [pytest] DJANGO_SETTINGS_MODULE = bookwyrm.settings python_files = tests.py test_*.py *_tests.py -addopts = --cov=bookwyrm --cov-config=.coveragerc \ No newline at end of file +addopts = --cov=bookwyrm --cov-config=.coveragerc +markers = + integration: marks tests as requiring external resources (deselect with '-m "not integration"') From 21a0e4a8f2870e65bfb4546c19405f8bb9886a08 Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 13:20:31 -0800 Subject: [PATCH 11/12] Alphabetize requirements and use a dash instead of hyphen in the pytest-django package requirement --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ff5848d56..12a7b6e3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,10 +7,10 @@ flower==0.9.4 Pillow>=7.1.0 psycopg2==2.8.4 pycryptodome==3.9.4 -python-dateutil==2.8.1 -pytest_django==4.1.0 +pytest-django==4.1.0 pytest==6.1.2 pytest-cov==2.10.1 +python-dateutil==2.8.1 redis==3.4.1 requests==2.22.0 responses==0.10.14 From 061193cdd5ee2d8731abdc9fde194c737ec41228 Mon Sep 17 00:00:00 2001 From: Jim Fingal Date: Sun, 8 Nov 2020 17:30:47 -0800 Subject: [PATCH 12/12] Go back to exec for everything --- fr-dev | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/fr-dev b/fr-dev index 32dff5ff0..c63a10a14 100755 --- a/fr-dev +++ b/fr-dev @@ -8,18 +8,22 @@ function clean { docker-compose rm -f } -function rundb { - docker-compose exec db $@ -} - function runweb { docker-compose run --rm web "$@" clean } +function execdb { + docker-compose exec db $@ +} + +function execweb { + docker-compose exec web "$@" +} + function initdb { - runweb python manage.py migrate - runweb python manage.py initdb + execweb python manage.py migrate + execweb python manage.py initdb } case "$1" in @@ -35,42 +39,42 @@ case "$1" in resetdb) clean docker-compose up --build -d - rundb dropdb -U fedireads fedireads - rundb createdb -U fedireads fedireads + execdb dropdb -U fedireads fedireads + execdb createdb -U fedireads fedireads initdb clean ;; makemigrations) - runweb python manage.py makemigrations + execweb python manage.py makemigrations ;; migrate) - runweb python manage.py migrate + execweb python manage.py migrate ;; bash) - runweb bash + execweb bash ;; shell) - runweb python manage.py shell + execweb python manage.py shell ;; dbshell) - rundb psql -U fedireads fedireads + execdb psql -U fedireads fedireads ;; restart_celery) docker-compose restart celery_worker ;; test) shift 1 - runweb coverage run --source='.' --omit="*/test*,celerywyrm*,bookwyrm/migrations/*" manage.py test "$@" + execweb coverage run --source='.' --omit="*/test*,celerywyrm*,bookwyrm/migrations/*" manage.py test "$@" ;; pytest) shift 1 - runweb pytest "$@" + execweb pytest "$@" ;; test_report) - runweb coverage report + execweb coverage report ;; collectstatic) - runweb python manage.py collectstatic --no-input + execweb python manage.py collectstatic --no-input ;; build) docker-compose build