015 add tracing

app/config.py

@@ -48,6 +48,13 @@ class AdapterConfig(BaseModel):
     cdek_cache_ttl_seconds: int = Field(default=900, gt=0)
 
 
+class ObservabilityConfig(BaseModel):
+    enabled: bool = False
+    service_name: str = Field(..., min_length=1)
+    otlp_endpoint: str = Field(..., min_length=1)
+    otlp_insecure: bool = True
+
+
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(
         extra="ignore",
@@ -58,6 +65,7 @@ class Settings(BaseSettings):
     business_logic: BusinessLogicConfig = Field(default_factory=BusinessLogicConfig)
     repository: RepositoryConfig = Field(default_factory=RepositoryConfig)
     adapter: AdapterConfig = Field(default_factory=AdapterConfig)
+    observability: ObservabilityConfig
 
     @classmethod
     def settings_customise_sources(
@@ -85,6 +93,7 @@ class _RequiredYamlSections(BaseModel):
     business_logic: dict[str, Any]
     repository: dict[str, Any]
     adapter: dict[str, Any]
+    observability: dict[str, Any]
 
 
 def _resolve_runtime_config_file() -> str:

app/main.py

@@ -5,6 +5,7 @@ from fastapi import FastAPI
 from app.config import Settings, get_settings
 from app.controllers.v1.delivery import router as delivery_router
 from app.runtime.logging import configure_logging
+from app.runtime.tracing import configure_tracing
 
 
 def create_app(settings: Settings | None = None) -> FastAPI:
@@ -15,6 +16,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     application.state.settings = resolved_settings
 
     application.include_router(delivery_router, prefix=resolved_settings.controller.api_prefix)
+    configure_tracing(application, resolved_settings.observability)
     return application
 
 

app/runtime/tracing.py

@@ -0,0 +1,75 @@
+"""Centralized runtime OpenTelemetry tracing bootstrap."""
+
+from fastapi import FastAPI
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+from opentelemetry.instrumentation.redis import RedisInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+from app.config import ObservabilityConfig
+
+_HTTPX_INSTRUMENTOR = HTTPXClientInstrumentor()
+_REDIS_INSTRUMENTOR = RedisInstrumentor()
+_TRACER_PROVIDER: TracerProvider | None = None
+
+
+def configure_tracing(app: FastAPI, observability: ObservabilityConfig) -> None:
+    if not observability.enabled:
+        return
+
+    tracer_provider = _get_or_create_tracer_provider(observability)
+    _instrument_fastapi_app(app, tracer_provider=tracer_provider)
+    _instrument_httpx(tracer_provider=tracer_provider)
+    _instrument_redis(tracer_provider=tracer_provider)
+
+
+def _get_or_create_tracer_provider(observability: ObservabilityConfig) -> TracerProvider:
+    global _TRACER_PROVIDER
+
+    if _TRACER_PROVIDER is not None:
+        return _TRACER_PROVIDER
+
+    tracer_provider = _build_tracer_provider(observability)
+    trace.set_tracer_provider(tracer_provider)
+    _TRACER_PROVIDER = tracer_provider
+    return tracer_provider
+
+
+def _build_tracer_provider(observability: ObservabilityConfig) -> TracerProvider:
+    exporter = _build_span_exporter(observability)
+    tracer_provider = TracerProvider(resource=_build_resource(observability))
+    tracer_provider.add_span_processor(BatchSpanProcessor(exporter))
+    return tracer_provider
+
+
+def _build_span_exporter(observability: ObservabilityConfig) -> OTLPSpanExporter:
+    return OTLPSpanExporter(
+        endpoint=observability.otlp_endpoint,
+        insecure=observability.otlp_insecure,
+    )
+
+
+def _build_resource(observability: ObservabilityConfig) -> Resource:
+    return Resource.create({"service.name": observability.service_name})
+
+
+def _instrument_fastapi_app(app: FastAPI, *, tracer_provider: TracerProvider) -> None:
+    if getattr(app, "_is_instrumented_by_opentelemetry", False):
+        return
+    FastAPIInstrumentor.instrument_app(app, tracer_provider=tracer_provider)
+
+
+def _instrument_httpx(*, tracer_provider: TracerProvider) -> None:
+    if _HTTPX_INSTRUMENTOR.is_instrumented_by_opentelemetry:
+        return
+    _HTTPX_INSTRUMENTOR.instrument(tracer_provider=tracer_provider)
+
+
+def _instrument_redis(*, tracer_provider: TracerProvider) -> None:
+    if _REDIS_INSTRUMENTOR.is_instrumented_by_opentelemetry:
+        return
+    _REDIS_INSTRUMENTOR.instrument(tracer_provider=tracer_provider)