From bb04a400441b097088d0e4c84a344fd3a31fafc3 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 31 Mar 2020 16:31:33 -0700 Subject: [PATCH] Move to redis and fix a bunch of things --- .env.example | 7 +- Dockerfile | 2 + docker-compose.yml | 18 ++-- fedireads/__init__.py | 9 -- fedireads/incoming.py | 26 ++--- fedireads/settings.py | 13 +-- fedireads/tasks.py | 42 +++++++++ fr_celery/__init__.py | 10 ++ fr_celery/asgi.py | 16 ++++ {fedireads => fr_celery}/celery.py | 8 +- fr_celery/settings.py | 146 +++++++++++++++++++++++++++++ fr_celery/urls.py | 21 +++++ fr_celery/wsgi.py | 16 ++++ requirements.txt | 2 + 14 files changed, 286 insertions(+), 50 deletions(-) create mode 100644 fedireads/tasks.py create mode 100644 fr_celery/__init__.py create mode 100644 fr_celery/asgi.py rename {fedireads => fr_celery}/celery.py (74%) create mode 100644 fr_celery/settings.py create mode 100644 fr_celery/urls.py create mode 100644 fr_celery/wsgi.py diff --git a/.env.example b/.env.example index a7a37c81e..dbb5e0bf7 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,7 @@ POSTGRES_USER=fedireads POSTGRES_DB=fedireads POSTGRES_HOST=db -RABBITMQ_DEFAULT_USER=rabbit -RABBITMQ_DEFAULT_PASS=changeme -CELERY_BROKER=amqp://rabbit:changeme@rabbitmq:5672 +CELERY_BROKER=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 + +FLOWER_PORT=5555 diff --git a/Dockerfile b/Dockerfile index c845035f9..f6dbb7402 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,3 +5,5 @@ WORKDIR /app COPY requirements.txt /app/ RUN pip install -r requirements.txt COPY ./fedireads /app +COPY ./fr_celery /app +EXPOSE 5555 diff --git a/docker-compose.yml b/docker-compose.yml index a119f25d0..cb20794e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,25 +20,31 @@ services: - celery_worker networks: - main - rabbitmq: + redis: + image: redis env_file: .env - image: rabbitmq:latest + ports: + - "6379:6379" networks: - main - ports: - - "5672:5672" restart: on-failure celery_worker: env_file: .env build: . networks: - main - command: celery -A fedireads worker -l info + command: celery -A fr_celery worker -l info volumes: - .:/app depends_on: - db - - rabbitmq + - redis + restart: on-failure + flower: + image: mher/flower + env_file: .env + ports: + - "5555:5555" restart: on-failure volumes: pgdata: diff --git a/fedireads/__init__.py b/fedireads/__init__.py index 18666a189..e69de29bb 100644 --- a/fedireads/__init__.py +++ b/fedireads/__init__.py @@ -1,9 +0,0 @@ -''' we need this file to initialize celery ''' -from __future__ import absolute_import, unicode_literals - -# This will make sure the app is always imported when -# Django starts so that shared_task will use this app. -from .celery import app as celery_app - -__all__ = ('celery_app',) - diff --git a/fedireads/incoming.py b/fedireads/incoming.py index a766cb688..252a1ecfe 100644 --- a/fedireads/incoming.py +++ b/fedireads/incoming.py @@ -13,6 +13,7 @@ import requests from fedireads import models, outgoing from fedireads import status as status_builder from fedireads.remote_user import get_or_create_remote_user +from fedireads import tasks @csrf_exempt @@ -256,31 +257,16 @@ def handle_create(activity): def handle_favorite(activity): ''' approval of your good good post ''' - try: - status_id = activity['object'].split('/')[-1] - status = models.Status.objects.get(id=status_id) - liker = get_or_create_remote_user(activity['actor']) - except (models.Status.DoesNotExist, models.User.DoesNotExist): - return HttpResponseNotFound() - - if not liker.local: - status_builder.create_favorite_from_activity(liker, activity) - - status_builder.create_notification( - status.user, - 'FAVORITE', - related_user=liker, - related_status=status, - ) + print('hiii!') + tasks.handle_incoming_favorite.delay(activity) return HttpResponse() def handle_unfavorite(activity): ''' approval of your good good post ''' - try: - favorite_id = activity['object']['id'] - fav = status_builder.get_favorite(favorite_id) - except models.Favorite.DoesNotExist: + favorite_id = activity['object']['id'] + fav = status_builder.get_favorite(favorite_id) + if not fav: return HttpResponseNotFound() fav.delete() diff --git a/fedireads/settings.py b/fedireads/settings.py index 3c951496d..ff93ad149 100644 --- a/fedireads/settings.py +++ b/fedireads/settings.py @@ -5,6 +5,13 @@ from environs import Env env = Env() +# celery +CELERY_BROKER = env('CELERY_BROKER') +CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND') +CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -21,12 +28,6 @@ DOMAIN = env('DOMAIN') ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', ['*']) OL_URL = env('OL_URL') -# celery/rebbitmq -CELERY_BROKER_URL = env('CELERY_BROKER') -CELERY_ACCEPT_CONTENT = ['json'] -CELERY_TASK_SERIALIZER = 'json' -CELERY_RESULT_BACKEND = 'amqp' - # Application definition INSTALLED_APPS = [ diff --git a/fedireads/tasks.py b/fedireads/tasks.py new file mode 100644 index 000000000..760dffef8 --- /dev/null +++ b/fedireads/tasks.py @@ -0,0 +1,42 @@ +''' background tasks ''' +from celery import Celery +import os + +from fedireads import models +from fedireads import status as status_builder +from fedireads.outgoing import get_or_create_remote_user +from fedireads import settings + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings') +app = Celery( + 'tasks', + broker=settings.CELERY_BROKER, +) + + +@app.task +def handle_incoming_favorite(activity): + ''' ugh ''' + print('here we go') + try: + status_id = activity['object'].split('/')[-1] + print(status_id) + status = models.Status.objects.get(id=status_id) + liker = get_or_create_remote_user(activity['actor']) + except (models.Status.DoesNotExist, models.User.DoesNotExist): + print('gonna return') + return + + print('got the status okay') + if not liker.local: + status_builder.create_favorite_from_activity(liker, activity) + + status_builder.create_notification( + status.user, + 'FAVORITE', + related_user=liker, + related_status=status, + ) + print('done') + diff --git a/fr_celery/__init__.py b/fr_celery/__init__.py new file mode 100644 index 000000000..3e6ab9e52 --- /dev/null +++ b/fr_celery/__init__.py @@ -0,0 +1,10 @@ +''' we need this file to initialize celery ''' +from __future__ import absolute_import, unicode_literals + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ('celery_app',) + + diff --git a/fr_celery/asgi.py b/fr_celery/asgi.py new file mode 100644 index 000000000..f66a43b91 --- /dev/null +++ b/fr_celery/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for fr_celery project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings') + +application = get_asgi_application() diff --git a/fedireads/celery.py b/fr_celery/celery.py similarity index 74% rename from fedireads/celery.py rename to fr_celery/celery.py index 7c26dc069..3368a8fb2 100644 --- a/fedireads/celery.py +++ b/fr_celery/celery.py @@ -5,9 +5,9 @@ import os from celery import Celery # set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fedireads.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings') -app = Celery('fedireads') +app = Celery('fr_celery') # Using a string here means the worker doesn't have to serialize # the configuration object to child processes. @@ -18,7 +18,3 @@ app.config_from_object('django.conf:settings', namespace='CELERY') # Load task modules from all registered Django app configs. app.autodiscover_tasks() - -@app.task(bind=True) -def debug_task(self): - print('Request: {0!r}'.format(self.request)) diff --git a/fr_celery/settings.py b/fr_celery/settings.py new file mode 100644 index 000000000..48aa1643b --- /dev/null +++ b/fr_celery/settings.py @@ -0,0 +1,146 @@ +""" +Django settings for fr_celery project. + +Generated by 'django-admin startproject' using Django 3.0.3. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os +from environs import Env + +env = Env() + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# celery/rebbitmq +CELERY_BROKER_URL = env('CELERY_BROKER') +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_BACKEND = 'redis' + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '0a^0gpwjc1ap+lb$dinin=efc@e&_0%102$o3(>9e7lndiaw' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'fr_celery', + 'fedireads', + 'celery', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'fr_celery.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'fr_celery.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases + +FEDIREADS_DATABASE_BACKEND = env('FEDIREADS_DATABASE_BACKEND', 'postgres') + +FEDIREADS_DBS = { + 'postgres': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': env('POSTGRES_DB', 'fedireads'), + 'USER': env('POSTGRES_USER', 'fedireads'), + 'PASSWORD': env('POSTGRES_PASSWORD', 'fedireads'), + 'HOST': env('POSTGRES_HOST', ''), + 'PORT': 5432 + }, + 'sqlite': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'fedireads.db') + } +} + +DATABASES = { + 'default': FEDIREADS_DBS[FEDIREADS_DATABASE_BACKEND] +} + + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/fr_celery/urls.py b/fr_celery/urls.py new file mode 100644 index 000000000..c8cc543b4 --- /dev/null +++ b/fr_celery/urls.py @@ -0,0 +1,21 @@ +"""fr_celery URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path('admin/', admin.site.urls), +] diff --git a/fr_celery/wsgi.py b/fr_celery/wsgi.py new file mode 100644 index 000000000..381265564 --- /dev/null +++ b/fr_celery/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for fr_celery project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings') + +application = get_wsgi_application() diff --git a/requirements.txt b/requirements.txt index b8836226b..13baa2257 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,10 @@ celery==4.4.2 Django==3.0.3 django-model-utils==4.0.0 environs==7.2.0 +flower==0.9.4 Pillow==7.0.0 psycopg2==2.8.4 pycryptodome==3.9.4 python-dateutil==2.8.1 +redis==3.4.1 requests==2.22.0