refactor(telemetry): simplify setup (#51)

overhead/perf metrics are replaced by traces to be more accurate

depends on
CS-SI/opentelemetry-instrumentation-eodag#18

---------

Signed-off-by: Aubin Lambaré <aubin.lambare@cs-soprasteria.com>

Author: alambare

.dockerignore

@@ -46,6 +46,7 @@ tests/logging.ini
 .Python
 env/
 venv/
+*venv*
 build/
 develop-eggs/
 dist/

Dockerfile

@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION=3.12
 
-FROM python:${PYTHON_VERSION}-slim as base
+FROM python:${PYTHON_VERSION}-slim AS base
 
 # Any python libraries that require system libraries to be installed will likely
 # need the following packages in order to build
@@ -12,7 +12,7 @@ RUN apt-get update && \
 
 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 
-FROM base as builder
+FROM base AS builder
 
 WORKDIR /app
 

config/otelcol_config.yml

@@ -31,14 +31,19 @@ exporters:
     endpoint: 0.0.0.0:8000
     namespace: eodag-otelcol-exporter
 
+  otlphttp:
+    endpoint: "http://jaeger:4318"
+    tls:
+      insecure: true
+
 service:
 
   pipelines:
 
     traces:
       receivers: [otlp]
       processors: [batch]
-      exporters: [debug]
+      exporters: [debug,otlphttp]
 
     metrics:
 #      receivers: [otlp, prometheus]

docker-compose-otel.yaml

@@ -0,0 +1,14 @@
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: ["--config=/etc/otelcol-config.yml"]
+    volumes:
+      - ./config/otelcol_config.yml:/etc/otelcol-config.yml
+    ports:
+      - "4318:4318"   # OTLP HTTP
+      - "8000:8000"   # Prometheus metrics
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    ports:
+      - "16686:16686" # Jaeger UI
+      - "14318:4318"   # OTLP HTTP

stac_fastapi/eodag/app.py

@@ -164,8 +164,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
     if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", ""):
         from stac_fastapi.eodag.telemetry import instrument_eodag
 
-        instrument_eodag(app.state.dag)
-    # init_cache(app)
+        instrument_eodag(app)
+
     app.state.stac_metadata_model = stac_metadata_model
     yield
 

stac_fastapi/eodag/telemetry.py

@@ -20,134 +20,76 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Union
+from typing import Union
 
+from fastapi import FastAPI
 from opentelemetry import metrics, trace
 from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 from opentelemetry.instrumentation.eodag import EODAGInstrumentor
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.metrics._internal.aggregation import (
-    ExplicitBucketHistogramAggregation,
-)
 from opentelemetry.sdk.metrics._internal.export import PeriodicExportingMetricReader
-from opentelemetry.sdk.metrics._internal.view import View
-
-# See https://github.com/open-telemetry/opentelemetry-python/issues/4615 for the type ignore
-from opentelemetry.sdk.resources import Resource  # type: ignore[attr-defined]
+from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.semconv.resource import ResourceAttributes
 
-if TYPE_CHECKING:
-    from fastapi import FastAPI
+logger = logging.getLogger(__name__)
 
-    from eodag import EODataAccessGateway
 
+def get_resource(app: FastAPI) -> Resource:
+    """create opentelemetry resource"""
+    if not getattr(app.state, "resource", None):
+        app.state.otel_resource = Resource.create().merge(Resource.create({"service.name": "stac-fastapi-eodag"}))
 
-logger = logging.getLogger(__name__)
+    return app.state.otel_resource
 
 
-def create_tracer_provider(resource: Resource) -> Union[TracerProvider, trace.TracerProvider]:
+def get_tracer_provider(resource: Resource) -> Union[TracerProvider, trace.TracerProvider]:
     """create opentelemetry tracer provider"""
     tracer_provider = trace.get_tracer_provider()
     if tracer_provider and not isinstance(tracer_provider, trace.ProxyTracerProvider):
-        logger.debug("Tracer provider already set, skipping creation.")
         return tracer_provider
 
     tracer_provider = TracerProvider(resource=resource)
     processor = BatchSpanProcessor(OTLPSpanExporter())
     tracer_provider.add_span_processor(processor)
     trace.set_tracer_provider(tracer_provider)
+
     return tracer_provider
 
 
-def create_meter_provider(resource: Resource) -> Union[MeterProvider, metrics.MeterProvider]:
+def get_meter_provider(resource: Resource) -> Union[MeterProvider, metrics.MeterProvider]:
     """create opentelemetry meter provider"""
     meter_provider = metrics.get_meter_provider()
     if meter_provider and not isinstance(meter_provider, metrics._internal._ProxyMeterProvider):
-        logger.debug("Meter provider already set, skipping creation.")
         return meter_provider
 
     reader = PeriodicExportingMetricReader(OTLPMetricExporter())
-    view_histograms: View = View(
-        instrument_type=metrics.Histogram,
-        aggregation=ExplicitBucketHistogramAggregation(
-            boundaries=(
-                0.25,
-                0.50,
-                0.75,
-                1.0,
-                1.5,
-                2.0,
-                3.0,
-                4.0,
-                5.0,
-                6.0,
-                7.0,
-                8.0,
-                9.0,
-                10.0,
-            )
-        ),
-    )
-    view_overhead_histograms: View = View(
-        instrument_type=metrics.Histogram,
-        instrument_name="*overhead*",
-        aggregation=ExplicitBucketHistogramAggregation(
-            boundaries=(
-                0.030,
-                0.040,
-                0.050,
-                0.060,
-                0.070,
-                0.080,
-                0.090,
-                0.100,
-                0.125,
-                0.150,
-                0.175,
-                0.200,
-                0.250,
-                0.500,
-            )
-        ),
-    )
-    meter_provider = MeterProvider(
-        resource=resource,
-        metric_readers=[reader],
-        views=(
-            view_histograms,
-            view_overhead_histograms,
-        ),
-    )
+    meter_provider = MeterProvider(resource=resource, metric_readers=[reader])
     metrics.set_meter_provider(meter_provider)
+
     return meter_provider
 
 
-def instrument_fastapi(
-    fastapi_app: FastAPI,
-) -> None:
+def instrument_fastapi(app: FastAPI):
     """Instrument FastAPI app."""
     logger.info("Instrument FastAPI app")
-    resource = Resource(attributes={ResourceAttributes.SERVICE_NAME: "stac-fastapi-eodag"})
-    tracer_provider = create_tracer_provider(resource)
-    meter_provider = create_meter_provider(resource)
+    resource = get_resource(app)
+
     FastAPIInstrumentor.instrument_app(
-        app=fastapi_app,
-        tracer_provider=tracer_provider,
-        meter_provider=meter_provider,
+        app=app,
+        tracer_provider=get_tracer_provider(resource),
+        meter_provider=get_meter_provider(resource),
     )
 
 
-def instrument_eodag(eodag_api: EODataAccessGateway):
+def instrument_eodag(app: FastAPI):
     """Instrument EODAG app"""
     logger.info("Instrument EODAG app")
-    resource = Resource(attributes={ResourceAttributes.SERVICE_NAME: "stac-fastapi-eodag"})
-    tracer_provider = create_tracer_provider(resource)
-    meter_provider = create_meter_provider(resource)
-    EODAGInstrumentor(eodag_api).instrument(
-        tracer_provider=tracer_provider,
-        meter_provider=meter_provider,
+    resource = get_resource(app)
+
+    EODAGInstrumentor(app.state.dag).instrument(
+        tracer_provider=get_tracer_provider(resource),
+        meter_provider=get_meter_provider(resource),
     )

tests/test_telemetry.py

@@ -31,8 +31,8 @@
 from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
 
 from stac_fastapi.eodag.telemetry import (
-    create_meter_provider,
-    create_tracer_provider,
+    get_meter_provider,
+    get_tracer_provider,
     instrument_eodag,
     instrument_fastapi,
 )
@@ -47,14 +47,16 @@ def resource() -> Resource:
     return Resource.create({"service.name": "test-service"})
 
 
-def test_create_tracer_provider(resource: Resource) -> None:
-    """Test that ``create_tracer_provider`` sets up a tracer provider and records spans.
+def test_get_tracer_provider(resource: Resource, monkeypatch: pytest.MonkeyPatch) -> None:
+    """Test that ``get_tracer_provider`` sets up a tracer provider and records spans.
 
     :param resource: fixture providing a test Resource
     """
     # Patch exporter to in-memory
+    monkeypatch.setattr("stac_fastapi.eodag.telemetry.OTLPSpanExporter", lambda *a, **kw: InMemorySpanExporter())
+
     exporter = InMemorySpanExporter()
-    tracer_provider = create_tracer_provider(resource)
+    tracer_provider = get_tracer_provider(resource)
     tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
 
     tracer = trace.get_tracer("test")
@@ -66,8 +68,8 @@ def test_create_tracer_provider(resource: Resource) -> None:
     assert spans[0].name == "test-span"
 
 
-def test_create_meter_provider(resource: Resource, monkeypatch: pytest.MonkeyPatch) -> None:
-    """Test that ``create_meter_provider`` sets up a meter provider and records metrics.
+def test_get_meter_provider(resource: Resource, monkeypatch: pytest.MonkeyPatch) -> None:
+    """Test that ``get_meter_provider`` sets up a meter provider and records metrics.
 
     :param resource: fixture providing a test Resource
     :param monkeypatch: pytest fixture to patch dependencies
@@ -81,7 +83,7 @@ def test_create_meter_provider(resource: Resource, monkeypatch: pytest.MonkeyPat
     )
 
     # Call the function (will use patched reader)
-    meter_provider = create_meter_provider(resource)
+    meter_provider = get_meter_provider(resource)
 
     # Ensure it’s registered
     metrics.set_meter_provider(meter_provider)
@@ -116,7 +118,13 @@ def test_instrument_eodag_runs(monkeypatch: pytest.MonkeyPatch) -> None:
     """
 
     class DummyEODAG:
-        pass
+        class State:
+            def __init__(self):
+                self.otel_resource = {}
+                self.dag = {}
+
+        def __init__(self):
+            self.state = self.State()
 
     class DummyInstrumentor:
         def __init__(self, api):