Move to redis and fix a bunch of things

.env.example

@@ -22,6 +22,7 @@ POSTGRES_USER=fedireads
 POSTGRES_DB=fedireads
 POSTGRES_HOST=db
 
-RABBITMQ_DEFAULT_USER=rabbit
+CELERY_BROKER=redis://redis:6379/0
-RABBITMQ_DEFAULT_PASS=changeme
+CELERY_RESULT_BACKEND=redis://redis:6379/0
-CELERY_BROKER=amqp://rabbit:changeme@rabbitmq:5672
+
+FLOWER_PORT=5555

Dockerfile

@@ -5,3 +5,5 @@ WORKDIR /app
 COPY requirements.txt /app/
 RUN pip install -r requirements.txt
 COPY ./fedireads /app
+COPY ./fr_celery /app
+EXPOSE 5555

docker-compose.yml

@@ -20,25 +20,31 @@ services:
             - celery_worker
         networks:
             - main
-    rabbitmq:
+    redis:
+        image: redis
         env_file: .env
-        image: rabbitmq:latest
+        ports:
+          - "6379:6379"
         networks:
             - main
-        ports:
-            - "5672:5672"
         restart: on-failure
     celery_worker:
         env_file: .env
         build: .
         networks:
             - main
-        command: celery -A fedireads worker -l info
+        command: celery -A fr_celery worker -l info
         volumes:
             - .:/app
         depends_on:
             - db
-            - rabbitmq
+            - redis
+        restart: on-failure
+    flower:
+        image: mher/flower
+        env_file: .env
+        ports:
+            - "5555:5555"
         restart: on-failure
 volumes:
     pgdata:

fedireads/__init__.py

@@ -1,9 +0,0 @@
-''' we need this file to initialize celery '''
-from __future__ import absolute_import, unicode_literals
-
-# This will make sure the app is always imported when
-# Django starts so that shared_task will use this app.
-from .celery import app as celery_app
-
-__all__ = ('celery_app',)
-

fedireads/incoming.py

@@ -13,6 +13,7 @@ import requests
 from fedireads import models, outgoing
 from fedireads import status as status_builder
 from fedireads.remote_user import get_or_create_remote_user
+from fedireads import tasks
 
 
 @csrf_exempt
@@ -256,31 +257,16 @@ def handle_create(activity):
 
 def handle_favorite(activity):
     ''' approval of your good good post '''
-    try:
+    print('hiii!')
-        status_id = activity['object'].split('/')[-1]
+    tasks.handle_incoming_favorite.delay(activity)
-        status = models.Status.objects.get(id=status_id)
-        liker = get_or_create_remote_user(activity['actor'])
-    except (models.Status.DoesNotExist, models.User.DoesNotExist):
-        return HttpResponseNotFound()
-
-    if not liker.local:
-        status_builder.create_favorite_from_activity(liker, activity)
-
-    status_builder.create_notification(
-        status.user,
-        'FAVORITE',
-        related_user=liker,
-        related_status=status,
-    )
     return HttpResponse()
 
 
 def handle_unfavorite(activity):
     ''' approval of your good good post '''
-    try:
+    favorite_id = activity['object']['id']
-        favorite_id = activity['object']['id']
+    fav = status_builder.get_favorite(favorite_id)
-        fav = status_builder.get_favorite(favorite_id)
+    if not fav:
-    except models.Favorite.DoesNotExist:
         return HttpResponseNotFound()
 
     fav.delete()

fedireads/settings.py

@@ -5,6 +5,13 @@ from environs import Env
 
 env = Env()
 
+# celery
+CELERY_BROKER = env('CELERY_BROKER')
+CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND')
+CELERY_ACCEPT_CONTENT = ['application/json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -21,12 +28,6 @@ DOMAIN = env('DOMAIN')
 ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', ['*'])
 OL_URL = env('OL_URL')
 
-# celery/rebbitmq
-CELERY_BROKER_URL = env('CELERY_BROKER')
-CELERY_ACCEPT_CONTENT = ['json']
-CELERY_TASK_SERIALIZER = 'json'
-CELERY_RESULT_BACKEND = 'amqp'
-
 # Application definition
 
 INSTALLED_APPS = [

fedireads/tasks.py

@@ -0,0 +1,42 @@
+''' background tasks '''
+from celery import Celery
+import os
+
+from fedireads import models
+from fedireads import status as status_builder
+from fedireads.outgoing import get_or_create_remote_user
+from fedireads import settings
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
+app = Celery(
+    'tasks',
+    broker=settings.CELERY_BROKER,
+)
+
+
+@app.task
+def handle_incoming_favorite(activity):
+    ''' ugh '''
+    print('here we go')
+    try:
+        status_id = activity['object'].split('/')[-1]
+        print(status_id)
+        status = models.Status.objects.get(id=status_id)
+        liker = get_or_create_remote_user(activity['actor'])
+    except (models.Status.DoesNotExist, models.User.DoesNotExist):
+        print('gonna return')
+        return
+
+    print('got the status okay')
+    if not liker.local:
+        status_builder.create_favorite_from_activity(liker, activity)
+
+    status_builder.create_notification(
+        status.user,
+        'FAVORITE',
+        related_user=liker,
+        related_status=status,
+    )
+    print('done')
+

fr_celery/__init__.py

@@ -0,0 +1,10 @@
+''' we need this file to initialize celery '''
+from __future__ import absolute_import, unicode_literals
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)
+
+

fr_celery/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for fr_celery project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
+
+application = get_asgi_application()

fr_celery/celery.py

@@ -5,9 +5,9 @@ import os
 from celery import Celery
 
 # set the default Django settings module for the 'celery' program.
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fedireads.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
 
-app = Celery('fedireads')
+app = Celery('fr_celery')
 
 # Using a string here means the worker doesn't have to serialize
 # the configuration object to child processes.
@@ -18,7 +18,3 @@ app.config_from_object('django.conf:settings', namespace='CELERY')
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks()
 
-
-@app.task(bind=True)
-def debug_task(self):
-    print('Request: {0!r}'.format(self.request))

fr_celery/settings.py

@@ -0,0 +1,146 @@
+"""
+Django settings for fr_celery project.
+
+Generated by 'django-admin startproject' using Django 3.0.3.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.0/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/3.0/ref/settings/
+"""
+
+import os
+from environs import Env
+
+env = Env()
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# celery/rebbitmq
+CELERY_BROKER_URL = env('CELERY_BROKER')
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_BACKEND = 'redis'
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = '0a^0gpwjc1ap+lb$dinin=efc@e&_0%102$o3(&gt9e7lndiaw'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'fr_celery',
+    'fedireads',
+    'celery',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'fr_celery.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'fr_celery.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
+
+FEDIREADS_DATABASE_BACKEND = env('FEDIREADS_DATABASE_BACKEND', 'postgres')
+
+FEDIREADS_DBS = {
+    'postgres': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': env('POSTGRES_DB', 'fedireads'),
+        'USER': env('POSTGRES_USER', 'fedireads'),
+        'PASSWORD': env('POSTGRES_PASSWORD', 'fedireads'),
+        'HOST': env('POSTGRES_HOST', ''),
+        'PORT': 5432
+    },
+    'sqlite': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'fedireads.db')
+    }
+}
+
+DATABASES = {
+    'default': FEDIREADS_DBS[FEDIREADS_DATABASE_BACKEND]
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/3.0/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.0/howto/static-files/
+
+STATIC_URL = '/static/'

fr_celery/urls.py

@@ -0,0 +1,21 @@
+"""fr_celery URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/3.0/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path
+
+urlpatterns = [
+    path('admin/', admin.site.urls),
+]

fr_celery/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for fr_celery project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings')
+
+application = get_wsgi_application()

requirements.txt

@@ -2,8 +2,10 @@ celery==4.4.2
 Django==3.0.3
 django-model-utils==4.0.0
 environs==7.2.0
+flower==0.9.4
 Pillow==7.0.0
 psycopg2==2.8.4
 pycryptodome==3.9.4
 python-dateutil==2.8.1
+redis==3.4.1
 requests==2.22.0