007 add telemetry

app/observability/__init__.py

@@ -0,0 +1,5 @@
+"""Observability setup and helpers."""
+
+from app.observability.setup import setup_observability
+
+__all__ = ["setup_observability"]

app/observability/logging.py

@@ -0,0 +1,48 @@
+"""Structured logging setup with request and trace correlation."""
+
+import logging
+from typing import Any
+
+import structlog
+from opentelemetry import trace
+from opentelemetry.trace import SpanContext
+
+
+def configure_structured_logging(log_level: str) -> None:
+    resolved_level = _resolve_log_level(log_level)
+    logging.basicConfig(level=resolved_level, format="%(message)s")
+
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            structlog.stdlib.add_log_level,
+            add_trace_context,
+            structlog.processors.TimeStamper(fmt="iso", utc=True),
+            structlog.processors.JSONRenderer(),
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.make_filtering_bound_logger(resolved_level),
+        cache_logger_on_first_use=True,
+    )
+
+
+def add_trace_context(
+    _: Any,
+    __: str,
+    event_dict: dict[str, Any],
+) -> dict[str, Any]:
+    span = trace.get_current_span()
+    span_context = span.get_span_context()
+    if _is_valid_span_context(span_context):
+        event_dict["trace_id"] = format(span_context.trace_id, "032x")
+        event_dict["span_id"] = format(span_context.span_id, "016x")
+    return event_dict
+
+
+def _resolve_log_level(log_level: str) -> int:
+    candidate = log_level.strip().upper()
+    return logging.getLevelNamesMapping().get(candidate, logging.INFO)
+
+
+def _is_valid_span_context(span_context: SpanContext) -> bool:
+    return bool(span_context.is_valid and span_context.trace_id and span_context.span_id)

app/observability/metrics.py

@@ -0,0 +1,129 @@
+"""OpenTelemetry metrics setup and recording helpers."""
+
+from typing import Sequence
+
+from opentelemetry import metrics
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import (
+    MetricReader,
+    PeriodicExportingMetricReader,
+)
+from opentelemetry.sdk.resources import Resource
+
+_METER_PROVIDER: MeterProvider | None = None
+_METRICS: "ObservabilityMetrics" | None = None
+_METER_NAME = "g2s.observability"
+
+
+class ObservabilityMetrics:
+    def __init__(
+        self,
+        meter_name: str,
+        *,
+        meter_provider: MeterProvider | None = None,
+    ) -> None:
+        if meter_provider is None:
+            meter = metrics.get_meter(meter_name)
+        else:
+            meter = meter_provider.get_meter(meter_name)
+        self._request_count = meter.create_counter(
+            name="http.server.request.count",
+            unit="1",
+            description="Total number of HTTP requests.",
+        )
+        self._error_count = meter.create_counter(
+            name="http.server.error.count",
+            unit="1",
+            description="Total number of HTTP server errors.",
+        )
+        self._request_latency = meter.create_histogram(
+            name="http.server.request.latency",
+            unit="ms",
+            description="HTTP request latency.",
+        )
+        self._provider_availability = meter.create_histogram(
+            name="delivery.provider.availability",
+            unit="1",
+            description="Provider availability sample (1=available, 0=unavailable).",
+        )
+
+    def record_http_request(
+        self,
+        *,
+        method: str,
+        route: str,
+        status_code: int,
+        duration_ms: float,
+        is_error: bool,
+    ) -> None:
+        attributes = {
+            "http.method": method,
+            "http.route": route,
+            "http.status_code": status_code,
+        }
+        self._request_count.add(1, attributes)
+        self._request_latency.record(duration_ms, attributes)
+        if is_error:
+            self._error_count.add(1, attributes)
+
+    def record_provider_availability(
+        self,
+        *,
+        provider: str,
+        is_available: bool,
+    ) -> None:
+        self._provider_availability.record(
+            1.0 if is_available else 0.0,
+            {"provider": provider},
+        )
+
+
+def setup_metrics(
+    *,
+    service_name: str,
+    otlp_endpoint: str,
+    meter_provider: MeterProvider | None = None,
+    metric_readers: Sequence[MetricReader] | None = None,
+) -> MeterProvider:
+    global _METER_PROVIDER, _METRICS
+
+    if meter_provider is None and _METER_PROVIDER is not None:
+        return _METER_PROVIDER
+
+    if meter_provider is None:
+        readers = list(metric_readers or ())
+        if otlp_endpoint:
+            metric_exporter = OTLPMetricExporter(
+                endpoint=otlp_endpoint,
+                insecure=otlp_endpoint.startswith("http://"),
+            )
+            readers.append(PeriodicExportingMetricReader(metric_exporter))
+        meter_provider = MeterProvider(
+            resource=Resource.create({"service.name": service_name}),
+            metric_readers=readers,
+        )
+
+    _METER_PROVIDER = meter_provider
+    _METRICS = ObservabilityMetrics(
+        _METER_NAME,
+        meter_provider=meter_provider,
+    )
+    return meter_provider
+
+
+def get_metrics() -> ObservabilityMetrics:
+    global _METRICS
+
+    if _METRICS is None:
+        _METRICS = ObservabilityMetrics(_METER_NAME)
+    return _METRICS
+
+
+def reset_metrics_for_tests() -> None:
+    global _METER_PROVIDER, _METRICS
+
+    if _METER_PROVIDER is not None:
+        _METER_PROVIDER.shutdown()
+    _METER_PROVIDER = None
+    _METRICS = None

app/observability/setup.py

@@ -0,0 +1,26 @@
+"""High-level observability setup wiring."""
+
+from fastapi import FastAPI
+
+from app.config import ObservabilityConfig
+from app.observability.logging import configure_structured_logging
+from app.observability.metrics import setup_metrics
+from app.observability.tracing import (
+    instrument_fastapi_app,
+    instrument_httpx_client,
+    setup_tracing,
+)
+
+
+def setup_observability(app: FastAPI, config: ObservabilityConfig) -> None:
+    configure_structured_logging(config.log_level)
+    setup_tracing(
+        service_name=config.service_name,
+        otlp_endpoint=config.otlp_endpoint,
+    )
+    setup_metrics(
+        service_name=config.service_name,
+        otlp_endpoint=config.otlp_endpoint,
+    )
+    instrument_fastapi_app(app)
+    instrument_httpx_client()

app/observability/tracing.py

@@ -0,0 +1,80 @@
+"""OpenTelemetry tracing setup and instrumentation helpers."""
+
+from typing import cast
+
+from fastapi import FastAPI
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.trace import Tracer, TracerProvider as TraceAPIProvider
+
+_TRACER_PROVIDER: TraceAPIProvider | None = None
+_HTTPX_INSTRUMENTED = False
+
+
+def setup_tracing(
+    *,
+    service_name: str,
+    otlp_endpoint: str,
+    tracer_provider: TraceAPIProvider | None = None,
+) -> TraceAPIProvider:
+    global _TRACER_PROVIDER
+
+    if tracer_provider is None and _TRACER_PROVIDER is not None:
+        return _TRACER_PROVIDER
+
+    if tracer_provider is None:
+        tracer_provider = TracerProvider(
+            resource=Resource.create({"service.name": service_name}),
+        )
+        if otlp_endpoint:
+            span_exporter = OTLPSpanExporter(
+                endpoint=otlp_endpoint,
+                insecure=otlp_endpoint.startswith("http://"),
+            )
+            tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
+
+    _TRACER_PROVIDER = tracer_provider
+    return tracer_provider
+
+
+def instrument_fastapi_app(app: FastAPI) -> None:
+    tracer_provider = _TRACER_PROVIDER
+    if tracer_provider is None:
+        return
+    FastAPIInstrumentor.instrument_app(app, tracer_provider=tracer_provider)
+
+
+def instrument_httpx_client() -> None:
+    global _HTTPX_INSTRUMENTED
+
+    tracer_provider = _TRACER_PROVIDER
+    if tracer_provider is None:
+        return
+
+    if _HTTPX_INSTRUMENTED:
+        return
+    HTTPXClientInstrumentor().instrument(tracer_provider=tracer_provider)
+    _HTTPX_INSTRUMENTED = True
+
+
+def get_tracer(name: str) -> Tracer:
+    tracer_provider = _TRACER_PROVIDER
+    if tracer_provider is None:
+        return trace.get_tracer(name)
+    return cast(Tracer, tracer_provider.get_tracer(name))
+
+
+def reset_tracing_for_tests() -> None:
+    global _TRACER_PROVIDER, _HTTPX_INSTRUMENTED
+
+    if _HTTPX_INSTRUMENTED:
+        HTTPXClientInstrumentor().uninstrument()
+    if isinstance(_TRACER_PROVIDER, TracerProvider):
+        _TRACER_PROVIDER.shutdown()
+    _TRACER_PROVIDER = None
+    _HTTPX_INSTRUMENTED = False