Adds priority queues to celery

celerywyrm/celery.py

@@ -3,7 +3,7 @@ from __future__ import absolute_import, unicode_literals
 import os
 
 from celery import Celery
-from . import settings
+from . import settings # pylint: disable=unused-import
 
 
 # set the default Django settings module for the 'celery' program.

celerywyrm/settings.py

@@ -10,6 +10,22 @@ CELERY_RESULT_BACKEND = "redis://:{}@redis_broker:{}/0".format(
     requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT")
 )
 
+CELERY_TASK_ROUTES = ([
+    # high - should really happen ASAP
+    ("bookwyrm.emailing.*", {"queue": "high_priority"}),
+    # medium - should really happen
+    ("bookwyrm.activitypub.base_activity.*", {"queue": "medium_priority"}),
+    ("bookwyrm.views.inbox.*", {"queue": "medium_priority"}),
+    ("bookwyrm.broadcast.*", {"queue": "medium_priority"}),
+    ("bookwyrm.activitystreams.*", {"queue": "medium_priority"}),
+    # low - no rush
+    ("bookwyrm.connectors.abstract_connector.*", {"queue": "low_priority"}),
+    ("bookwyrm.goodreads_import.*", {"queue": "low_priority"}),
+    ("bookwyrm.models.user.*", {"queue": "low_priority"}),
+    ("bookwyrm.suggested_users.*", {"queue": "low_priority"}),
+    ("bookwyrm.preview_images.*", {"queue": "low_priority"}),
+])
+
 CELERY_ACCEPT_CONTENT = ["json"]
 CELERY_TASK_SERIALIZER = "json"
 CELERY_RESULT_SERIALIZER = "json"

docker-compose.yml

@@ -63,7 +63,7 @@ services:
     build: .
     networks:
       - main
-    command: celery -A celerywyrm worker -l info
+    command: celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority
     volumes:
       - .:/app
       - static_volume:/app/static