Merge pull request #1751 from cincodenada/open-telemetry

Adds OpenTelemetry exporter for use with various monitoring tools

.env.example

@@ -83,3 +83,19 @@ PREVIEW_TEXT_COLOR=#363636
 PREVIEW_IMG_WIDTH=1200
 PREVIEW_IMG_HEIGHT=630
 PREVIEW_DEFAULT_COVER_COLOR=#002549
+
+# Below are example keys if you want to enable automatically
+# sending telemetry to an OTLP-compatible service. Many of
+# the main monitoring apps have OLTP collectors, including
+# NewRelic, DataDog, and Honeycomb.io - consult their
+# documentation for setup instructions, and what exactly to
+# put below!
+#
+# Service name is an arbitrary tag that is attached to any
+# data sent, used to distinguish different sources. Useful
+# for sending prod and dev metrics to the same place and
+# keeping them separate, for instance!
+
+OTEL_EXPORTER_OTLP_ENDPOINT=    # API endpoint for your provider
+OTEL_EXPORTER_OTLP_HEADERS=     # Any headers required, usually authentication info
+OTEL_SERVICE_NAME=              # Service name to identify your app

bookwyrm/apps.py

@@ -0,0 +1,13 @@
+from django.apps import AppConfig
+from bookwyrm import settings
+
+
+class BookwyrmConfig(AppConfig):
+    name = "bookwyrm"
+    verbose_name = "BookWyrm"
+
+    def ready(self):
+        if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+            from bookwyrm.telemetry import open_telemetry
+
+            open_telemetry.instrumentDjango()

bookwyrm/settings.py

@@ -267,3 +267,7 @@ else:
     MEDIA_FULL_URL = f"{PROTOCOL}://{DOMAIN}{MEDIA_URL}"
     STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}"
     MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
+
+OTEL_EXPORTER_OTLP_ENDPOINT = env("OTEL_EXPORTER_OTLP_ENDPOINT", None)
+OTEL_EXPORTER_OTLP_HEADERS = env("OTEL_EXPORTER_OTLP_HEADERS", None)
+OTEL_SERVICE_NAME = env("OTEL_SERVICE_NAME", None)

bookwyrm/telemetry/open_telemetry.py

@@ -0,0 +1,22 @@
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+trace.set_tracer_provider(TracerProvider())
+trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
+
+
+def instrumentDjango():
+    from opentelemetry.instrumentation.django import DjangoInstrumentor
+
+    DjangoInstrumentor().instrument()
+
+
+def instrumentCelery():
+    from opentelemetry.instrumentation.celery import CeleryInstrumentor
+    from celery.signals import worker_process_init
+
+    @worker_process_init.connect(weak=False)
+    def init_celery_tracing(*args, **kwargs):
+        CeleryInstrumentor().instrument()

bookwyrm/urls.py

@@ -34,9 +34,11 @@ urlpatterns = [
         TemplateView.as_view(template_name="robots.txt", content_type="text/plain"),
     ),
     # federation endpoints
-    re_path(r"^inbox/?$", views.Inbox.as_view()),
-    re_path(rf"{LOCAL_USER_PATH}/inbox/?$", views.Inbox.as_view()),
-    re_path(rf"{LOCAL_USER_PATH}/outbox/?$", views.Outbox.as_view()),
+    re_path(r"^inbox/?$", views.Inbox.as_view(), name="inbox"),
+    re_path(rf"{LOCAL_USER_PATH}/inbox/?$", views.Inbox.as_view(), name="user_inbox"),
+    re_path(
+        rf"{LOCAL_USER_PATH}/outbox/?$", views.Outbox.as_view(), name="user_outbox"
+    ),
     re_path(r"^\.well-known/webfinger/?$", views.webfinger),
     re_path(r"^\.well-known/nodeinfo/?$", views.nodeinfo_pointer),
     re_path(r"^\.well-known/host-meta/?$", views.host_meta),
@@ -46,8 +48,16 @@ urlpatterns = [
     re_path(r"^opensearch.xml$", views.opensearch, name="opensearch"),
     re_path(r"^ostatus_subscribe/?$", views.ostatus_follow_request),
     # polling updates
-    re_path("^api/updates/notifications/?$", views.get_notification_count),
-    re_path("^api/updates/stream/(?P<stream>[a-z]+)/?$", views.get_unread_status_count),
+    re_path(
+        "^api/updates/notifications/?$",
+        views.get_notification_count,
+        name="notification-updates",
+    ),
+    re_path(
+        "^api/updates/stream/(?P<stream>[a-z]+)/?$",
+        views.get_unread_status_count,
+        name="stream-updates",
+    ),
     # authentication
     re_path(r"^login/?$", views.Login.as_view(), name="login"),
     re_path(r"^login/(?P<confirmed>confirmed)/?$", views.Login.as_view(), name="login"),
@@ -147,7 +157,9 @@ urlpatterns = [
     re_path(
         r"^invite-request/?$", views.InviteRequest.as_view(), name="invite-request"
     ),
-    re_path(r"^invite/(?P<code>[A-Za-z0-9]+)/?$", views.Invite.as_view()),
+    re_path(
+        r"^invite/(?P<code>[A-Za-z0-9]+)/?$", views.Invite.as_view(), name="invite"
+    ),
     re_path(
         r"^settings/email-blocklist/?$",
         views.EmailBlocklist.as_view(),

celerywyrm/apps.py

@@ -0,0 +1,13 @@
+from django.apps import AppConfig
+from celerywyrm import settings
+
+
+class CelerywyrmConfig(AppConfig):
+    name = "celerywyrm"
+    verbose_name = "BookWyrm Celery"
+
+    def ready(self):
+        if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+            from bookwyrm.telemetry import open_telemetry
+
+            open_telemetry.instrumentCelery()

requirements.txt

@@ -19,3 +19,8 @@ pytz>=2021.1
 boto3==1.17.88
 django-storages==1.11.1
 django-redis==5.2.0
+opentelemetry-api==1.8.0
+opentelemetry-sdk==1.8.0
+opentelemetry-exporter-otlp-proto-grpc==1.8.0
+opentelemetry-instrumentation-django==0.27b0
+opentelemetry-instrumentation-celery==0.27b0