Merge pull request #1920 from bookwyrm-social/branch-convergence

Moves towards single branch setup

.env.example

@@ -89,3 +89,19 @@ PREVIEW_TEXT_COLOR=#363636
 PREVIEW_IMG_WIDTH=1200
 PREVIEW_IMG_HEIGHT=630
 PREVIEW_DEFAULT_COVER_COLOR=#002549
+
+# Below are example keys if you want to enable automatically
+# sending telemetry to an OTLP-compatible service. Many of
+# the main monitoring apps have OLTP collectors, including
+# NewRelic, DataDog, and Honeycomb.io - consult their
+# documentation for setup instructions, and what exactly to
+# put below!
+#
+# Service name is an arbitrary tag that is attached to any
+# data sent, used to distinguish different sources. Useful
+# for sending prod and dev metrics to the same place and
+# keeping them separate, for instance!
+
+OTEL_EXPORTER_OTLP_ENDPOINT=    # API endpoint for your provider
+OTEL_EXPORTER_OTLP_HEADERS=     # Any headers required, usually authentication info
+OTEL_SERVICE_NAME=              # Service name to identify your app

bookwyrm/apps.py

@@ -32,7 +32,15 @@ class BookwyrmConfig(AppConfig):
     name = "bookwyrm"
     verbose_name = "BookWyrm"
 
+    # pylint: disable=no-self-use
     def ready(self):
+        """set up OTLP and preview image files, if desired"""
+        if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+            # pylint: disable=import-outside-toplevel
+            from bookwyrm.telemetry import open_telemetry
+
+            open_telemetry.instrumentDjango()
+
         if settings.ENABLE_PREVIEW_IMAGES and settings.FONTS:
             # Download any fonts that we don't have yet
             logger.debug("Downloading fonts..")

bookwyrm/management/commands/erase_streams.py

@@ -7,6 +7,7 @@ from bookwyrm import settings
 r = redis.Redis(
     host=settings.REDIS_ACTIVITY_HOST,
     port=settings.REDIS_ACTIVITY_PORT,
+    password=settings.REDIS_ACTIVITY_PASSWORD,
     db=settings.REDIS_ACTIVITY_DB_INDEX,
 )
 

bookwyrm/settings.py

@@ -333,3 +333,7 @@ else:
     MEDIA_URL = "/images/"
     MEDIA_FULL_URL = f"{PROTOCOL}://{DOMAIN}{MEDIA_URL}"
     STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}"
+
+OTEL_EXPORTER_OTLP_ENDPOINT = env("OTEL_EXPORTER_OTLP_ENDPOINT", None)
+OTEL_EXPORTER_OTLP_HEADERS = env("OTEL_EXPORTER_OTLP_HEADERS", None)
+OTEL_SERVICE_NAME = env("OTEL_SERVICE_NAME", None)

bookwyrm/telemetry/open_telemetry.py

@@ -0,0 +1,22 @@
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+trace.set_tracer_provider(TracerProvider())
+trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
+
+
+def instrumentDjango():
+    from opentelemetry.instrumentation.django import DjangoInstrumentor
+
+    DjangoInstrumentor().instrument()
+
+
+def instrumentCelery():
+    from opentelemetry.instrumentation.celery import CeleryInstrumentor
+    from celery.signals import worker_process_init
+
+    @worker_process_init.connect(weak=False)
+    def init_celery_tracing(*args, **kwargs):
+        CeleryInstrumentor().instrument()

celerywyrm/apps.py

@@ -0,0 +1,13 @@
+from django.apps import AppConfig
+from celerywyrm import settings
+
+
+class CelerywyrmConfig(AppConfig):
+    name = "celerywyrm"
+    verbose_name = "BookWyrm Celery"
+
+    def ready(self):
+        if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+            from bookwyrm.telemetry import open_telemetry
+
+            open_telemetry.instrumentCelery()

certbot.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-source .env;
-
-if [ "$CERTBOT_INIT" = "true" ]
-then
-    certonly \
-    --webroot \
-    --webroot-path=/var/www/certbot \
-    --email ${EMAIL} \
-    --agree-tos \
-    --no-eff-email \
-    -d ${DOMAIN} \
-    -d www.${DOMAIN}
-else
-    renew \
-    --webroot \
-    --webroot-path \
-    /var/www/certbot
-fi

docker-compose.yml

@@ -39,25 +39,23 @@ services:
   redis_activity:
     image: redis
     command: redis-server --requirepass ${REDIS_ACTIVITY_PASSWORD} --appendonly yes --port ${REDIS_ACTIVITY_PORT}
-    env_file: .env
-    networks:
-      - main
-    restart: on-failure
     volumes:
       - ./redis.conf:/etc/redis/redis.conf
       - redis_activity_data:/data
-  redis_broker:
-    image: redis
-    command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT}
     env_file: .env
-    ports:
-      - 6379
     networks:
       - main
     restart: on-failure
+  redis_broker:
+    image: redis
+    command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT}
     volumes:
       - ./redis.conf:/etc/redis/redis.conf
       - redis_broker_data:/data
+    env_file: .env
+    networks:
+      - main
+    restart: on-failure
   celery_worker:
     env_file: .env
     build: .

requirements.txt

@@ -18,6 +18,11 @@ pytz>=2021.1
 boto3==1.17.88
 django-storages==1.11.1
 django-redis==5.2.0
+opentelemetry-api==1.8.0
+opentelemetry-sdk==1.8.0
+opentelemetry-exporter-otlp-proto-grpc==1.8.0
+opentelemetry-instrumentation-django==0.27b0
+opentelemetry-instrumentation-celery==0.27b0
 
 # Dev
 black==21.4b2