Move to redis and fix a bunch of things

This commit is contained in:
Mouse Reeve 2020-03-31 16:31:33 -07:00
parent ca26a712c3
commit bb04a40044
14 changed files with 286 additions and 50 deletions

View file

@ -22,6 +22,7 @@ POSTGRES_USER=fedireads
POSTGRES_DB=fedireads POSTGRES_DB=fedireads
POSTGRES_HOST=db POSTGRES_HOST=db
RABBITMQ_DEFAULT_USER=rabbit CELERY_BROKER=redis://redis:6379/0
RABBITMQ_DEFAULT_PASS=changeme CELERY_RESULT_BACKEND=redis://redis:6379/0
CELERY_BROKER=amqp://rabbit:changeme@rabbitmq:5672
FLOWER_PORT=5555

View file

@ -5,3 +5,5 @@ WORKDIR /app
COPY requirements.txt /app/ COPY requirements.txt /app/
RUN pip install -r requirements.txt RUN pip install -r requirements.txt
COPY ./fedireads /app COPY ./fedireads /app
COPY ./fr_celery /app
EXPOSE 5555

View file

@ -20,25 +20,31 @@ services:
- celery_worker - celery_worker
networks: networks:
- main - main
rabbitmq: redis:
image: redis
env_file: .env env_file: .env
image: rabbitmq:latest ports:
- "6379:6379"
networks: networks:
- main - main
ports:
- "5672:5672"
restart: on-failure restart: on-failure
celery_worker: celery_worker:
env_file: .env env_file: .env
build: . build: .
networks: networks:
- main - main
command: celery -A fedireads worker -l info command: celery -A fr_celery worker -l info
volumes: volumes:
- .:/app - .:/app
depends_on: depends_on:
- db - db
- rabbitmq - redis
restart: on-failure
flower:
image: mher/flower
env_file: .env
ports:
- "5555:5555"
restart: on-failure restart: on-failure
volumes: volumes:
pgdata: pgdata:

View file

@ -1,9 +0,0 @@
''' we need this file to initialize celery '''
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ('celery_app',)

View file

@ -13,6 +13,7 @@ import requests
from fedireads import models, outgoing from fedireads import models, outgoing
from fedireads import status as status_builder from fedireads import status as status_builder
from fedireads.remote_user import get_or_create_remote_user from fedireads.remote_user import get_or_create_remote_user
from fedireads import tasks
@csrf_exempt @csrf_exempt
@ -256,31 +257,16 @@ def handle_create(activity):
def handle_favorite(activity): def handle_favorite(activity):
''' approval of your good good post ''' ''' approval of your good good post '''
try: print('hiii!')
status_id = activity['object'].split('/')[-1] tasks.handle_incoming_favorite.delay(activity)
status = models.Status.objects.get(id=status_id)
liker = get_or_create_remote_user(activity['actor'])
except (models.Status.DoesNotExist, models.User.DoesNotExist):
return HttpResponseNotFound()
if not liker.local:
status_builder.create_favorite_from_activity(liker, activity)
status_builder.create_notification(
status.user,
'FAVORITE',
related_user=liker,
related_status=status,
)
return HttpResponse() return HttpResponse()
def handle_unfavorite(activity): def handle_unfavorite(activity):
''' approval of your good good post ''' ''' approval of your good good post '''
try: favorite_id = activity['object']['id']
favorite_id = activity['object']['id'] fav = status_builder.get_favorite(favorite_id)
fav = status_builder.get_favorite(favorite_id) if not fav:
except models.Favorite.DoesNotExist:
return HttpResponseNotFound() return HttpResponseNotFound()
fav.delete() fav.delete()

View file

@ -5,6 +5,13 @@ from environs import Env
env = Env() env = Env()
# celery
CELERY_BROKER = env('CELERY_BROKER')
CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND')
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
# Build paths inside the project like this: os.path.join(BASE_DIR, ...) # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@ -21,12 +28,6 @@ DOMAIN = env('DOMAIN')
ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', ['*']) ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', ['*'])
OL_URL = env('OL_URL') OL_URL = env('OL_URL')
# celery/rebbitmq
CELERY_BROKER_URL = env('CELERY_BROKER')
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_BACKEND = 'amqp'
# Application definition # Application definition
INSTALLED_APPS = [ INSTALLED_APPS = [

42
fedireads/tasks.py Normal file
View file

@ -0,0 +1,42 @@
''' background tasks '''
from celery import Celery
import os
from fedireads import models
from fedireads import status as status_builder
from fedireads.outgoing import get_or_create_remote_user
from fedireads import settings
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
app = Celery(
'tasks',
broker=settings.CELERY_BROKER,
)
@app.task
def handle_incoming_favorite(activity):
''' ugh '''
print('here we go')
try:
status_id = activity['object'].split('/')[-1]
print(status_id)
status = models.Status.objects.get(id=status_id)
liker = get_or_create_remote_user(activity['actor'])
except (models.Status.DoesNotExist, models.User.DoesNotExist):
print('gonna return')
return
print('got the status okay')
if not liker.local:
status_builder.create_favorite_from_activity(liker, activity)
status_builder.create_notification(
status.user,
'FAVORITE',
related_user=liker,
related_status=status,
)
print('done')

10
fr_celery/__init__.py Normal file
View file

@ -0,0 +1,10 @@
''' we need this file to initialize celery '''
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ('celery_app',)

16
fr_celery/asgi.py Normal file
View file

@ -0,0 +1,16 @@
"""
ASGI config for fr_celery project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
application = get_asgi_application()

View file

@ -5,9 +5,9 @@ import os
from celery import Celery from celery import Celery
# set the default Django settings module for the 'celery' program. # set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fedireads.settings') os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
app = Celery('fedireads') app = Celery('fr_celery')
# Using a string here means the worker doesn't have to serialize # Using a string here means the worker doesn't have to serialize
# the configuration object to child processes. # the configuration object to child processes.
@ -18,7 +18,3 @@ app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs. # Load task modules from all registered Django app configs.
app.autodiscover_tasks() app.autodiscover_tasks()
@app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))

146
fr_celery/settings.py Normal file
View file

@ -0,0 +1,146 @@
"""
Django settings for fr_celery project.
Generated by 'django-admin startproject' using Django 3.0.3.
For more information on this file, see
https://docs.djangoproject.com/en/3.0/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.0/ref/settings/
"""
import os
from environs import Env
env = Env()
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# celery/rebbitmq
CELERY_BROKER_URL = env('CELERY_BROKER')
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_BACKEND = 'redis'
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '0a^0gpwjc1ap+lb$dinin=efc@e&_0%102$o3(&gt9e7lndiaw'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'fr_celery',
'fedireads',
'celery',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'fr_celery.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'fr_celery.wsgi.application'
# Database
# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
FEDIREADS_DATABASE_BACKEND = env('FEDIREADS_DATABASE_BACKEND', 'postgres')
FEDIREADS_DBS = {
'postgres': {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'NAME': env('POSTGRES_DB', 'fedireads'),
'USER': env('POSTGRES_USER', 'fedireads'),
'PASSWORD': env('POSTGRES_PASSWORD', 'fedireads'),
'HOST': env('POSTGRES_HOST', ''),
'PORT': 5432
},
'sqlite': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'fedireads.db')
}
}
DATABASES = {
'default': FEDIREADS_DBS[FEDIREADS_DATABASE_BACKEND]
}
# Password validation
# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/3.0/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.0/howto/static-files/
STATIC_URL = '/static/'

21
fr_celery/urls.py Normal file
View file

@ -0,0 +1,21 @@
"""fr_celery URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
urlpatterns = [
path('admin/', admin.site.urls),
]

16
fr_celery/wsgi.py Normal file
View file

@ -0,0 +1,16 @@
"""
WSGI config for fr_celery project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
application = get_wsgi_application()

View file

@ -2,8 +2,10 @@ celery==4.4.2
Django==3.0.3 Django==3.0.3
django-model-utils==4.0.0 django-model-utils==4.0.0
environs==7.2.0 environs==7.2.0
flower==0.9.4
Pillow==7.0.0 Pillow==7.0.0
psycopg2==2.8.4 psycopg2==2.8.4
pycryptodome==3.9.4 pycryptodome==3.9.4
python-dateutil==2.8.1 python-dateutil==2.8.1
redis==3.4.1
requests==2.22.0 requests==2.22.0