mirror of
https://github.com/bookwyrm-social/bookwyrm.git
synced 2024-11-25 11:01:12 +00:00
Move to redis and fix a bunch of things
This commit is contained in:
parent
ca26a712c3
commit
bb04a40044
14 changed files with 286 additions and 50 deletions
|
@ -22,6 +22,7 @@ POSTGRES_USER=fedireads
|
|||
POSTGRES_DB=fedireads
|
||||
POSTGRES_HOST=db
|
||||
|
||||
RABBITMQ_DEFAULT_USER=rabbit
|
||||
RABBITMQ_DEFAULT_PASS=changeme
|
||||
CELERY_BROKER=amqp://rabbit:changeme@rabbitmq:5672
|
||||
CELERY_BROKER=redis://redis:6379/0
|
||||
CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
|
||||
FLOWER_PORT=5555
|
||||
|
|
|
@ -5,3 +5,5 @@ WORKDIR /app
|
|||
COPY requirements.txt /app/
|
||||
RUN pip install -r requirements.txt
|
||||
COPY ./fedireads /app
|
||||
COPY ./fr_celery /app
|
||||
EXPOSE 5555
|
||||
|
|
|
@ -20,25 +20,31 @@ services:
|
|||
- celery_worker
|
||||
networks:
|
||||
- main
|
||||
rabbitmq:
|
||||
redis:
|
||||
image: redis
|
||||
env_file: .env
|
||||
image: rabbitmq:latest
|
||||
ports:
|
||||
- "6379:6379"
|
||||
networks:
|
||||
- main
|
||||
ports:
|
||||
- "5672:5672"
|
||||
restart: on-failure
|
||||
celery_worker:
|
||||
env_file: .env
|
||||
build: .
|
||||
networks:
|
||||
- main
|
||||
command: celery -A fedireads worker -l info
|
||||
command: celery -A fr_celery worker -l info
|
||||
volumes:
|
||||
- .:/app
|
||||
depends_on:
|
||||
- db
|
||||
- rabbitmq
|
||||
- redis
|
||||
restart: on-failure
|
||||
flower:
|
||||
image: mher/flower
|
||||
env_file: .env
|
||||
ports:
|
||||
- "5555:5555"
|
||||
restart: on-failure
|
||||
volumes:
|
||||
pgdata:
|
||||
|
|
|
@ -1,9 +0,0 @@
|
|||
''' we need this file to initialize celery '''
|
||||
from __future__ import absolute_import, unicode_literals
|
||||
|
||||
# This will make sure the app is always imported when
|
||||
# Django starts so that shared_task will use this app.
|
||||
from .celery import app as celery_app
|
||||
|
||||
__all__ = ('celery_app',)
|
||||
|
|
@ -13,6 +13,7 @@ import requests
|
|||
from fedireads import models, outgoing
|
||||
from fedireads import status as status_builder
|
||||
from fedireads.remote_user import get_or_create_remote_user
|
||||
from fedireads import tasks
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
|
@ -256,31 +257,16 @@ def handle_create(activity):
|
|||
|
||||
def handle_favorite(activity):
|
||||
''' approval of your good good post '''
|
||||
try:
|
||||
status_id = activity['object'].split('/')[-1]
|
||||
status = models.Status.objects.get(id=status_id)
|
||||
liker = get_or_create_remote_user(activity['actor'])
|
||||
except (models.Status.DoesNotExist, models.User.DoesNotExist):
|
||||
return HttpResponseNotFound()
|
||||
|
||||
if not liker.local:
|
||||
status_builder.create_favorite_from_activity(liker, activity)
|
||||
|
||||
status_builder.create_notification(
|
||||
status.user,
|
||||
'FAVORITE',
|
||||
related_user=liker,
|
||||
related_status=status,
|
||||
)
|
||||
print('hiii!')
|
||||
tasks.handle_incoming_favorite.delay(activity)
|
||||
return HttpResponse()
|
||||
|
||||
|
||||
def handle_unfavorite(activity):
|
||||
''' approval of your good good post '''
|
||||
try:
|
||||
favorite_id = activity['object']['id']
|
||||
fav = status_builder.get_favorite(favorite_id)
|
||||
except models.Favorite.DoesNotExist:
|
||||
favorite_id = activity['object']['id']
|
||||
fav = status_builder.get_favorite(favorite_id)
|
||||
if not fav:
|
||||
return HttpResponseNotFound()
|
||||
|
||||
fav.delete()
|
||||
|
|
|
@ -5,6 +5,13 @@ from environs import Env
|
|||
|
||||
env = Env()
|
||||
|
||||
# celery
|
||||
CELERY_BROKER = env('CELERY_BROKER')
|
||||
CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND')
|
||||
CELERY_ACCEPT_CONTENT = ['application/json']
|
||||
CELERY_TASK_SERIALIZER = 'json'
|
||||
CELERY_RESULT_SERIALIZER = 'json'
|
||||
|
||||
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
@ -21,12 +28,6 @@ DOMAIN = env('DOMAIN')
|
|||
ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', ['*'])
|
||||
OL_URL = env('OL_URL')
|
||||
|
||||
# celery/rebbitmq
|
||||
CELERY_BROKER_URL = env('CELERY_BROKER')
|
||||
CELERY_ACCEPT_CONTENT = ['json']
|
||||
CELERY_TASK_SERIALIZER = 'json'
|
||||
CELERY_RESULT_BACKEND = 'amqp'
|
||||
|
||||
# Application definition
|
||||
|
||||
INSTALLED_APPS = [
|
||||
|
|
42
fedireads/tasks.py
Normal file
42
fedireads/tasks.py
Normal file
|
@ -0,0 +1,42 @@
|
|||
''' background tasks '''
|
||||
from celery import Celery
|
||||
import os
|
||||
|
||||
from fedireads import models
|
||||
from fedireads import status as status_builder
|
||||
from fedireads.outgoing import get_or_create_remote_user
|
||||
from fedireads import settings
|
||||
|
||||
# set the default Django settings module for the 'celery' program.
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
|
||||
app = Celery(
|
||||
'tasks',
|
||||
broker=settings.CELERY_BROKER,
|
||||
)
|
||||
|
||||
|
||||
@app.task
|
||||
def handle_incoming_favorite(activity):
|
||||
''' ugh '''
|
||||
print('here we go')
|
||||
try:
|
||||
status_id = activity['object'].split('/')[-1]
|
||||
print(status_id)
|
||||
status = models.Status.objects.get(id=status_id)
|
||||
liker = get_or_create_remote_user(activity['actor'])
|
||||
except (models.Status.DoesNotExist, models.User.DoesNotExist):
|
||||
print('gonna return')
|
||||
return
|
||||
|
||||
print('got the status okay')
|
||||
if not liker.local:
|
||||
status_builder.create_favorite_from_activity(liker, activity)
|
||||
|
||||
status_builder.create_notification(
|
||||
status.user,
|
||||
'FAVORITE',
|
||||
related_user=liker,
|
||||
related_status=status,
|
||||
)
|
||||
print('done')
|
||||
|
10
fr_celery/__init__.py
Normal file
10
fr_celery/__init__.py
Normal file
|
@ -0,0 +1,10 @@
|
|||
''' we need this file to initialize celery '''
|
||||
from __future__ import absolute_import, unicode_literals
|
||||
|
||||
# This will make sure the app is always imported when
|
||||
# Django starts so that shared_task will use this app.
|
||||
from .celery import app as celery_app
|
||||
|
||||
__all__ = ('celery_app',)
|
||||
|
||||
|
16
fr_celery/asgi.py
Normal file
16
fr_celery/asgi.py
Normal file
|
@ -0,0 +1,16 @@
|
|||
"""
|
||||
ASGI config for fr_celery project.
|
||||
|
||||
It exposes the ASGI callable as a module-level variable named ``application``.
|
||||
|
||||
For more information on this file, see
|
||||
https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from django.core.asgi import get_asgi_application
|
||||
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
|
||||
|
||||
application = get_asgi_application()
|
|
@ -5,9 +5,9 @@ import os
|
|||
from celery import Celery
|
||||
|
||||
# set the default Django settings module for the 'celery' program.
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fedireads.settings')
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
|
||||
|
||||
app = Celery('fedireads')
|
||||
app = Celery('fr_celery')
|
||||
|
||||
# Using a string here means the worker doesn't have to serialize
|
||||
# the configuration object to child processes.
|
||||
|
@ -18,7 +18,3 @@ app.config_from_object('django.conf:settings', namespace='CELERY')
|
|||
# Load task modules from all registered Django app configs.
|
||||
app.autodiscover_tasks()
|
||||
|
||||
|
||||
@app.task(bind=True)
|
||||
def debug_task(self):
|
||||
print('Request: {0!r}'.format(self.request))
|
146
fr_celery/settings.py
Normal file
146
fr_celery/settings.py
Normal file
|
@ -0,0 +1,146 @@
|
|||
"""
|
||||
Django settings for fr_celery project.
|
||||
|
||||
Generated by 'django-admin startproject' using Django 3.0.3.
|
||||
|
||||
For more information on this file, see
|
||||
https://docs.djangoproject.com/en/3.0/topics/settings/
|
||||
|
||||
For the full list of settings and their values, see
|
||||
https://docs.djangoproject.com/en/3.0/ref/settings/
|
||||
"""
|
||||
|
||||
import os
|
||||
from environs import Env
|
||||
|
||||
env = Env()
|
||||
|
||||
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
# celery/rebbitmq
|
||||
CELERY_BROKER_URL = env('CELERY_BROKER')
|
||||
CELERY_ACCEPT_CONTENT = ['json']
|
||||
CELERY_TASK_SERIALIZER = 'json'
|
||||
CELERY_RESULT_BACKEND = 'redis'
|
||||
|
||||
|
||||
# Quick-start development settings - unsuitable for production
|
||||
# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
|
||||
|
||||
# SECURITY WARNING: keep the secret key used in production secret!
|
||||
SECRET_KEY = '0a^0gpwjc1ap+lb$dinin=efc@e&_0%102$o3(>9e7lndiaw'
|
||||
|
||||
# SECURITY WARNING: don't run with debug turned on in production!
|
||||
DEBUG = True
|
||||
|
||||
ALLOWED_HOSTS = []
|
||||
|
||||
|
||||
# Application definition
|
||||
|
||||
INSTALLED_APPS = [
|
||||
'django.contrib.admin',
|
||||
'django.contrib.auth',
|
||||
'django.contrib.contenttypes',
|
||||
'django.contrib.sessions',
|
||||
'django.contrib.messages',
|
||||
'django.contrib.staticfiles',
|
||||
'fr_celery',
|
||||
'fedireads',
|
||||
'celery',
|
||||
]
|
||||
|
||||
MIDDLEWARE = [
|
||||
'django.middleware.security.SecurityMiddleware',
|
||||
'django.contrib.sessions.middleware.SessionMiddleware',
|
||||
'django.middleware.common.CommonMiddleware',
|
||||
'django.middleware.csrf.CsrfViewMiddleware',
|
||||
'django.contrib.auth.middleware.AuthenticationMiddleware',
|
||||
'django.contrib.messages.middleware.MessageMiddleware',
|
||||
'django.middleware.clickjacking.XFrameOptionsMiddleware',
|
||||
]
|
||||
|
||||
ROOT_URLCONF = 'fr_celery.urls'
|
||||
|
||||
TEMPLATES = [
|
||||
{
|
||||
'BACKEND': 'django.template.backends.django.DjangoTemplates',
|
||||
'DIRS': [],
|
||||
'APP_DIRS': True,
|
||||
'OPTIONS': {
|
||||
'context_processors': [
|
||||
'django.template.context_processors.debug',
|
||||
'django.template.context_processors.request',
|
||||
'django.contrib.auth.context_processors.auth',
|
||||
'django.contrib.messages.context_processors.messages',
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
WSGI_APPLICATION = 'fr_celery.wsgi.application'
|
||||
|
||||
|
||||
# Database
|
||||
# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
|
||||
|
||||
FEDIREADS_DATABASE_BACKEND = env('FEDIREADS_DATABASE_BACKEND', 'postgres')
|
||||
|
||||
FEDIREADS_DBS = {
|
||||
'postgres': {
|
||||
'ENGINE': 'django.db.backends.postgresql_psycopg2',
|
||||
'NAME': env('POSTGRES_DB', 'fedireads'),
|
||||
'USER': env('POSTGRES_USER', 'fedireads'),
|
||||
'PASSWORD': env('POSTGRES_PASSWORD', 'fedireads'),
|
||||
'HOST': env('POSTGRES_HOST', ''),
|
||||
'PORT': 5432
|
||||
},
|
||||
'sqlite': {
|
||||
'ENGINE': 'django.db.backends.sqlite3',
|
||||
'NAME': os.path.join(BASE_DIR, 'fedireads.db')
|
||||
}
|
||||
}
|
||||
|
||||
DATABASES = {
|
||||
'default': FEDIREADS_DBS[FEDIREADS_DATABASE_BACKEND]
|
||||
}
|
||||
|
||||
|
||||
# Password validation
|
||||
# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
|
||||
|
||||
AUTH_PASSWORD_VALIDATORS = [
|
||||
{
|
||||
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
|
||||
},
|
||||
{
|
||||
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
|
||||
},
|
||||
{
|
||||
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
|
||||
},
|
||||
{
|
||||
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# Internationalization
|
||||
# https://docs.djangoproject.com/en/3.0/topics/i18n/
|
||||
|
||||
LANGUAGE_CODE = 'en-us'
|
||||
|
||||
TIME_ZONE = 'UTC'
|
||||
|
||||
USE_I18N = True
|
||||
|
||||
USE_L10N = True
|
||||
|
||||
USE_TZ = True
|
||||
|
||||
|
||||
# Static files (CSS, JavaScript, Images)
|
||||
# https://docs.djangoproject.com/en/3.0/howto/static-files/
|
||||
|
||||
STATIC_URL = '/static/'
|
21
fr_celery/urls.py
Normal file
21
fr_celery/urls.py
Normal file
|
@ -0,0 +1,21 @@
|
|||
"""fr_celery URL Configuration
|
||||
|
||||
The `urlpatterns` list routes URLs to views. For more information please see:
|
||||
https://docs.djangoproject.com/en/3.0/topics/http/urls/
|
||||
Examples:
|
||||
Function views
|
||||
1. Add an import: from my_app import views
|
||||
2. Add a URL to urlpatterns: path('', views.home, name='home')
|
||||
Class-based views
|
||||
1. Add an import: from other_app.views import Home
|
||||
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
|
||||
Including another URLconf
|
||||
1. Import the include() function: from django.urls import include, path
|
||||
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
|
||||
"""
|
||||
from django.contrib import admin
|
||||
from django.urls import path
|
||||
|
||||
urlpatterns = [
|
||||
path('admin/', admin.site.urls),
|
||||
]
|
16
fr_celery/wsgi.py
Normal file
16
fr_celery/wsgi.py
Normal file
|
@ -0,0 +1,16 @@
|
|||
"""
|
||||
WSGI config for fr_celery project.
|
||||
|
||||
It exposes the WSGI callable as a module-level variable named ``application``.
|
||||
|
||||
For more information on this file, see
|
||||
https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from django.core.wsgi import get_wsgi_application
|
||||
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
|
||||
|
||||
application = get_wsgi_application()
|
|
@ -2,8 +2,10 @@ celery==4.4.2
|
|||
Django==3.0.3
|
||||
django-model-utils==4.0.0
|
||||
environs==7.2.0
|
||||
flower==0.9.4
|
||||
Pillow==7.0.0
|
||||
psycopg2==2.8.4
|
||||
pycryptodome==3.9.4
|
||||
python-dateutil==2.8.1
|
||||
redis==3.4.1
|
||||
requests==2.22.0
|
||||
|
|
Loading…
Reference in a new issue