From 7811a9920e27ac2be5d3d8e6e9676603f875ffaf Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Feb 2022 18:34:17 -0800 Subject: [PATCH 1/3] Copy config from prod branch to main --- .env.example | 16 ++++++++++++++ bookwyrm/apps.py | 5 +++++ bookwyrm/management/commands/erase_streams.py | 1 + bookwyrm/settings.py | 4 ++++ bookwyrm/telemetry/open_telemetry.py | 22 +++++++++++++++++++ celerywyrm/apps.py | 13 +++++++++++ certbot.sh | 19 ---------------- docker-compose.yml | 10 ++++----- requirements.txt | 5 +++++ 9 files changed, 70 insertions(+), 25 deletions(-) create mode 100644 bookwyrm/telemetry/open_telemetry.py create mode 100644 celerywyrm/apps.py delete mode 100644 certbot.sh diff --git a/.env.example b/.env.example index ca6f65bb7..7769a67b1 100644 --- a/.env.example +++ b/.env.example @@ -89,3 +89,19 @@ PREVIEW_TEXT_COLOR=#363636 PREVIEW_IMG_WIDTH=1200 PREVIEW_IMG_HEIGHT=630 PREVIEW_DEFAULT_COVER_COLOR=#002549 + +# Below are example keys if you want to enable automatically +# sending telemetry to an OTLP-compatible service. Many of +# the main monitoring apps have OLTP collectors, including +# NewRelic, DataDog, and Honeycomb.io - consult their +# documentation for setup instructions, and what exactly to +# put below! +# +# Service name is an arbitrary tag that is attached to any +# data sent, used to distinguish different sources. Useful +# for sending prod and dev metrics to the same place and +# keeping them separate, for instance! + +OTEL_EXPORTER_OTLP_ENDPOINT= # API endpoint for your provider +OTEL_EXPORTER_OTLP_HEADERS= # Any headers required, usually authentication info +OTEL_SERVICE_NAME= # Service name to identify your app diff --git a/bookwyrm/apps.py b/bookwyrm/apps.py index 8940edccc..af3b064e9 100644 --- a/bookwyrm/apps.py +++ b/bookwyrm/apps.py @@ -33,6 +33,11 @@ class BookwyrmConfig(AppConfig): verbose_name = "BookWyrm" def ready(self): + if settings.OTEL_EXPORTER_OTLP_ENDPOINT: + from bookwyrm.telemetry import open_telemetry + + open_telemetry.instrumentDjango() + if settings.ENABLE_PREVIEW_IMAGES and settings.FONTS: # Download any fonts that we don't have yet logger.debug("Downloading fonts..") diff --git a/bookwyrm/management/commands/erase_streams.py b/bookwyrm/management/commands/erase_streams.py index 7b8074029..9d971d699 100644 --- a/bookwyrm/management/commands/erase_streams.py +++ b/bookwyrm/management/commands/erase_streams.py @@ -7,6 +7,7 @@ from bookwyrm import settings r = redis.Redis( host=settings.REDIS_ACTIVITY_HOST, port=settings.REDIS_ACTIVITY_PORT, + password=settings.REDIS_ACTIVITY_PASSWORD, db=settings.REDIS_ACTIVITY_DB_INDEX, ) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index e8cbe344b..e9f3d0585 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -333,3 +333,7 @@ else: MEDIA_URL = "/images/" MEDIA_FULL_URL = f"{PROTOCOL}://{DOMAIN}{MEDIA_URL}" STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}" + +OTEL_EXPORTER_OTLP_ENDPOINT = env("OTEL_EXPORTER_OTLP_ENDPOINT", None) +OTEL_EXPORTER_OTLP_HEADERS = env("OTEL_EXPORTER_OTLP_HEADERS", None) +OTEL_SERVICE_NAME = env("OTEL_SERVICE_NAME", None) diff --git a/bookwyrm/telemetry/open_telemetry.py b/bookwyrm/telemetry/open_telemetry.py new file mode 100644 index 000000000..0b38a04b1 --- /dev/null +++ b/bookwyrm/telemetry/open_telemetry.py @@ -0,0 +1,22 @@ +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +trace.set_tracer_provider(TracerProvider()) +trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + + +def instrumentDjango(): + from opentelemetry.instrumentation.django import DjangoInstrumentor + + DjangoInstrumentor().instrument() + + +def instrumentCelery(): + from opentelemetry.instrumentation.celery import CeleryInstrumentor + from celery.signals import worker_process_init + + @worker_process_init.connect(weak=False) + def init_celery_tracing(*args, **kwargs): + CeleryInstrumentor().instrument() diff --git a/celerywyrm/apps.py b/celerywyrm/apps.py new file mode 100644 index 000000000..6aae849cd --- /dev/null +++ b/celerywyrm/apps.py @@ -0,0 +1,13 @@ +from django.apps import AppConfig +from celerywyrm import settings + + +class CelerywyrmConfig(AppConfig): + name = "celerywyrm" + verbose_name = "BookWyrm Celery" + + def ready(self): + if settings.OTEL_EXPORTER_OTLP_ENDPOINT: + from bookwyrm.telemetry import open_telemetry + + open_telemetry.instrumentCelery() diff --git a/certbot.sh b/certbot.sh deleted file mode 100644 index 6d2c3cd90..000000000 --- a/certbot.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -source .env; - -if [ "$CERTBOT_INIT" = "true" ] -then - certonly \ - --webroot \ - --webroot-path=/var/www/certbot \ - --email ${EMAIL} \ - --agree-tos \ - --no-eff-email \ - -d ${DOMAIN} \ - -d www.${DOMAIN} -else - renew \ - --webroot \ - --webroot-path \ - /var/www/certbot -fi diff --git a/docker-compose.yml b/docker-compose.yml index fa28dbee6..d3b47467c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,15 +49,13 @@ services: redis_broker: image: redis command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT} - env_file: .env - ports: - - 6379 - networks: - - main - restart: on-failure volumes: - ./redis.conf:/etc/redis/redis.conf - redis_broker_data:/data + env_file: .env + networks: + - main + restart: on-failure celery_worker: env_file: .env build: . diff --git a/requirements.txt b/requirements.txt index 4bda6bd5a..638ed91b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,11 @@ pytz>=2021.1 boto3==1.17.88 django-storages==1.11.1 django-redis==5.2.0 +opentelemetry-api==1.8.0 +opentelemetry-sdk==1.8.0 +opentelemetry-exporter-otlp-proto-grpc==1.8.0 +opentelemetry-instrumentation-django==0.27b0 +opentelemetry-instrumentation-celery==0.27b0 # Dev black==21.4b2 From bc7aa91b97d7bf89feb3787c731ab76b1bd65e9d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 4 Feb 2022 18:50:57 -0800 Subject: [PATCH 2/3] Fixes pylint complaints --- bookwyrm/apps.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bookwyrm/apps.py b/bookwyrm/apps.py index af3b064e9..d494877d6 100644 --- a/bookwyrm/apps.py +++ b/bookwyrm/apps.py @@ -32,8 +32,11 @@ class BookwyrmConfig(AppConfig): name = "bookwyrm" verbose_name = "BookWyrm" + # pylint: disable=no-self-use def ready(self): + """set up OTLP and preview image files, if desired""" if settings.OTEL_EXPORTER_OTLP_ENDPOINT: + # pylint: disable=import-outside-toplevel from bookwyrm.telemetry import open_telemetry open_telemetry.instrumentDjango() From 74fd13fb22e60913f376c9c2bdc34db69a086d53 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 6 Feb 2022 06:37:03 -0800 Subject: [PATCH 3/3] Consistent ordering of items in docker-compose file --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d3b47467c..672ea4c1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,13 +39,13 @@ services: redis_activity: image: redis command: redis-server --requirepass ${REDIS_ACTIVITY_PASSWORD} --appendonly yes --port ${REDIS_ACTIVITY_PORT} + volumes: + - ./redis.conf:/etc/redis/redis.conf + - redis_activity_data:/data env_file: .env networks: - main restart: on-failure - volumes: - - ./redis.conf:/etc/redis/redis.conf - - redis_activity_data:/data redis_broker: image: redis command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT}