Adds priority queues to celery

This commit is contained in:
Mouse Reeve 2021-09-07 16:06:54 -07:00
parent b41293e2ce
commit c0b662cbd7
3 changed files with 18 additions and 2 deletions

View file

@ -3,7 +3,7 @@ from __future__ import absolute_import, unicode_literals
import os import os
from celery import Celery from celery import Celery
from . import settings from . import settings # pylint: disable=unused-import
# set the default Django settings module for the 'celery' program. # set the default Django settings module for the 'celery' program.

View file

@ -10,6 +10,22 @@ CELERY_RESULT_BACKEND = "redis://:{}@redis_broker:{}/0".format(
requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT") requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT")
) )
CELERY_TASK_ROUTES = ([
# high - should really happen ASAP
("bookwyrm.emailing.*", {"queue": "high_priority"}),
# medium - should really happen
("bookwyrm.activitypub.base_activity.*", {"queue": "medium_priority"}),
("bookwyrm.views.inbox.*", {"queue": "medium_priority"}),
("bookwyrm.broadcast.*", {"queue": "medium_priority"}),
("bookwyrm.activitystreams.*", {"queue": "medium_priority"}),
# low - no rush
("bookwyrm.connectors.abstract_connector.*", {"queue": "low_priority"}),
("bookwyrm.goodreads_import.*", {"queue": "low_priority"}),
("bookwyrm.models.user.*", {"queue": "low_priority"}),
("bookwyrm.suggested_users.*", {"queue": "low_priority"}),
("bookwyrm.preview_images.*", {"queue": "low_priority"}),
])
CELERY_ACCEPT_CONTENT = ["json"] CELERY_ACCEPT_CONTENT = ["json"]
CELERY_TASK_SERIALIZER = "json" CELERY_TASK_SERIALIZER = "json"
CELERY_RESULT_SERIALIZER = "json" CELERY_RESULT_SERIALIZER = "json"

View file

@ -63,7 +63,7 @@ services:
build: . build: .
networks: networks:
- main - main
command: celery -A celerywyrm worker -l info command: celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority
volumes: volumes:
- .:/app - .:/app
- static_volume:/app/static - static_volume:/app/static