From 207abed654dff732586f1a5e113119a71e8eb008 Mon Sep 17 00:00:00 2001 From: Robert George Date: Sat, 14 Jan 2023 14:52:54 -0800 Subject: [PATCH 1/3] Add support for specifying redis url to support unix sockets --- .env.example | 4 ++++ bookwyrm/management/commands/erase_streams.py | 7 +------ bookwyrm/redis_store.py | 7 +------ bookwyrm/settings.py | 4 ++-- bookwyrm/views/admin/celery_status.py | 7 +------ celerywyrm/settings.py | 5 +++-- 6 files changed, 12 insertions(+), 22 deletions(-) diff --git a/.env.example b/.env.example index dd7bec4f2..4c1c2eefe 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,8 @@ REDIS_ACTIVITY_PORT=6379 REDIS_ACTIVITY_PASSWORD=redispassword345 # Optional, use a different redis database (defaults to 0) # REDIS_ACTIVITY_DB_INDEX=0 +# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket +# REDIS_ACTIVITY_URL= # Redis as celery broker REDIS_BROKER_HOST=redis_broker @@ -39,6 +41,8 @@ REDIS_BROKER_PORT=6379 REDIS_BROKER_PASSWORD=redispassword123 # Optional, use a different redis database (defaults to 0) # REDIS_BROKER_DB_INDEX=0 +# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket +# REDIS_BROKER_URL= # Monitoring for celery FLOWER_PORT=8888 diff --git a/bookwyrm/management/commands/erase_streams.py b/bookwyrm/management/commands/erase_streams.py index 9d971d699..ecd36006c 100644 --- a/bookwyrm/management/commands/erase_streams.py +++ b/bookwyrm/management/commands/erase_streams.py @@ -4,12 +4,7 @@ import redis from bookwyrm import settings -r = redis.Redis( - host=settings.REDIS_ACTIVITY_HOST, - port=settings.REDIS_ACTIVITY_PORT, - password=settings.REDIS_ACTIVITY_PASSWORD, - db=settings.REDIS_ACTIVITY_DB_INDEX, -) +r = redis.from_url(settings.REDIS_ACTIVITY_URL) def erase_streams(): diff --git a/bookwyrm/redis_store.py b/bookwyrm/redis_store.py index ae50db2ee..f25829f5c 100644 --- a/bookwyrm/redis_store.py +++ b/bookwyrm/redis_store.py @@ -4,12 +4,7 @@ import redis from bookwyrm import settings -r = redis.Redis( - host=settings.REDIS_ACTIVITY_HOST, - port=settings.REDIS_ACTIVITY_PORT, - password=settings.REDIS_ACTIVITY_PASSWORD, - db=settings.REDIS_ACTIVITY_DB_INDEX, -) +r = redis.from_url(settings.REDIS_ACTIVITY_URL) class RedisStore(ABC): diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 74ef7d313..3090d27f5 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -207,7 +207,7 @@ REDIS_ACTIVITY_HOST = env("REDIS_ACTIVITY_HOST", "localhost") REDIS_ACTIVITY_PORT = env("REDIS_ACTIVITY_PORT", 6379) REDIS_ACTIVITY_PASSWORD = env("REDIS_ACTIVITY_PASSWORD", None) REDIS_ACTIVITY_DB_INDEX = env("REDIS_ACTIVITY_DB_INDEX", 0) - +REDIS_ACTIVITY_URL = env("REDIS_ACTIVITY_URL", f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/{REDIS_ACTIVITY_DB_INDEX}") MAX_STREAM_LENGTH = int(env("MAX_STREAM_LENGTH", 200)) STREAMS = [ @@ -232,7 +232,7 @@ else: CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/{REDIS_ACTIVITY_DB_INDEX}", + "LOCATION": REDIS_ACTIVITY_URL, "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", }, diff --git a/bookwyrm/views/admin/celery_status.py b/bookwyrm/views/admin/celery_status.py index 7da148a06..5221fd619 100644 --- a/bookwyrm/views/admin/celery_status.py +++ b/bookwyrm/views/admin/celery_status.py @@ -8,12 +8,7 @@ import redis from celerywyrm import settings from bookwyrm.tasks import app as celery -r = redis.Redis( - host=settings.REDIS_BROKER_HOST, - port=settings.REDIS_BROKER_PORT, - password=settings.REDIS_BROKER_PASSWORD, - db=settings.REDIS_BROKER_DB_INDEX, -) +r = redis.from_url(settings.REDIS_BROKER_URL) # pylint: disable= no-self-use @method_decorator(login_required, name="dispatch") diff --git a/celerywyrm/settings.py b/celerywyrm/settings.py index 94e530221..3b6dc998a 100644 --- a/celerywyrm/settings.py +++ b/celerywyrm/settings.py @@ -8,9 +8,10 @@ REDIS_BROKER_PASSWORD = requests.utils.quote(env("REDIS_BROKER_PASSWORD", None)) REDIS_BROKER_HOST = env("REDIS_BROKER_HOST", "redis_broker") REDIS_BROKER_PORT = env("REDIS_BROKER_PORT", 6379) REDIS_BROKER_DB_INDEX = env("REDIS_BROKER_DB_INDEX", 0) +REDIS_BROKER_URL = env("REDIS_BROKER_URL",f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/{REDIS_BROKER_DB_INDEX}") -CELERY_BROKER_URL = f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/{REDIS_BROKER_DB_INDEX}" -CELERY_RESULT_BACKEND = f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/{REDIS_BROKER_DB_INDEX}" +CELERY_BROKER_URL = REDIS_BROKER_URL.replace("unix:","redis+socket:") +CELERY_RESULT_BACKEND = REDIS_BROKER_URL.replace("unix:","redis+socket:") CELERY_DEFAULT_QUEUE = "low_priority" CELERY_CREATE_MISSING_QUEUES = True From f362343dfaa404d9fab08f13da6bccaeb05f8ea0 Mon Sep 17 00:00:00 2001 From: Robert George Date: Sat, 14 Jan 2023 15:44:10 -0800 Subject: [PATCH 2/3] style fixes --- bookwyrm/settings.py | 5 ++++- celerywyrm/settings.py | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 3090d27f5..5967b010b 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -207,7 +207,10 @@ REDIS_ACTIVITY_HOST = env("REDIS_ACTIVITY_HOST", "localhost") REDIS_ACTIVITY_PORT = env("REDIS_ACTIVITY_PORT", 6379) REDIS_ACTIVITY_PASSWORD = env("REDIS_ACTIVITY_PASSWORD", None) REDIS_ACTIVITY_DB_INDEX = env("REDIS_ACTIVITY_DB_INDEX", 0) -REDIS_ACTIVITY_URL = env("REDIS_ACTIVITY_URL", f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/{REDIS_ACTIVITY_DB_INDEX}") +REDIS_ACTIVITY_URL = env( + "REDIS_ACTIVITY_URL", + f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/{REDIS_ACTIVITY_DB_INDEX}" +) MAX_STREAM_LENGTH = int(env("MAX_STREAM_LENGTH", 200)) STREAMS = [ diff --git a/celerywyrm/settings.py b/celerywyrm/settings.py index 3b6dc998a..146699f53 100644 --- a/celerywyrm/settings.py +++ b/celerywyrm/settings.py @@ -8,10 +8,13 @@ REDIS_BROKER_PASSWORD = requests.utils.quote(env("REDIS_BROKER_PASSWORD", None)) REDIS_BROKER_HOST = env("REDIS_BROKER_HOST", "redis_broker") REDIS_BROKER_PORT = env("REDIS_BROKER_PORT", 6379) REDIS_BROKER_DB_INDEX = env("REDIS_BROKER_DB_INDEX", 0) -REDIS_BROKER_URL = env("REDIS_BROKER_URL",f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/{REDIS_BROKER_DB_INDEX}") +REDIS_BROKER_URL = env( + "REDIS_BROKER_URL", + f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/{REDIS_BROKER_DB_INDEX}" +) -CELERY_BROKER_URL = REDIS_BROKER_URL.replace("unix:","redis+socket:") -CELERY_RESULT_BACKEND = REDIS_BROKER_URL.replace("unix:","redis+socket:") +CELERY_BROKER_URL = REDIS_BROKER_URL.replace("unix:", "redis+socket:") +CELERY_RESULT_BACKEND = REDIS_BROKER_URL.replace("unix:", "redis+socket:") CELERY_DEFAULT_QUEUE = "low_priority" CELERY_CREATE_MISSING_QUEUES = True From 95b66480c9d087881d279e87b0a6a900f96726fd Mon Sep 17 00:00:00 2001 From: Robert George Date: Sat, 14 Jan 2023 18:20:37 -0800 Subject: [PATCH 3/3] lint fixes --- bookwyrm/settings.py | 2 +- celerywyrm/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 5967b010b..1a45c4c6d 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -209,7 +209,7 @@ REDIS_ACTIVITY_PASSWORD = env("REDIS_ACTIVITY_PASSWORD", None) REDIS_ACTIVITY_DB_INDEX = env("REDIS_ACTIVITY_DB_INDEX", 0) REDIS_ACTIVITY_URL = env( "REDIS_ACTIVITY_URL", - f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/{REDIS_ACTIVITY_DB_INDEX}" + f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/{REDIS_ACTIVITY_DB_INDEX}", ) MAX_STREAM_LENGTH = int(env("MAX_STREAM_LENGTH", 200)) diff --git a/celerywyrm/settings.py b/celerywyrm/settings.py index 146699f53..7519ea6e3 100644 --- a/celerywyrm/settings.py +++ b/celerywyrm/settings.py @@ -10,7 +10,7 @@ REDIS_BROKER_PORT = env("REDIS_BROKER_PORT", 6379) REDIS_BROKER_DB_INDEX = env("REDIS_BROKER_DB_INDEX", 0) REDIS_BROKER_URL = env( "REDIS_BROKER_URL", - f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/{REDIS_BROKER_DB_INDEX}" + f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/{REDIS_BROKER_DB_INDEX}", ) CELERY_BROKER_URL = REDIS_BROKER_URL.replace("unix:", "redis+socket:")