007 add telemetry
This commit is contained in:
Раис Юсупалиев
@@ -0,0 +1,50 @@
|
||||
import structlog
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from structlog.testing import LogCapture
|
||||
from structlog.contextvars import bind_contextvars, clear_contextvars
|
||||
|
||||
from app.observability.logging import add_trace_context
|
||||
from app.observability.tracing import get_tracer, reset_tracing_for_tests, setup_tracing
|
||||
|
||||
|
||||
def test_logging_context_includes_request_id_and_trace_id() -> None:
|
||||
reset_tracing_for_tests()
|
||||
span_exporter = InMemorySpanExporter()
|
||||
tracer_provider = TracerProvider(resource=Resource.create({"service.name": "tests"}))
|
||||
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
|
||||
setup_tracing(
|
||||
service_name="tests",
|
||||
otlp_endpoint="",
|
||||
tracer_provider=tracer_provider,
|
||||
)
|
||||
|
||||
log_capture = LogCapture()
|
||||
structlog.configure(
|
||||
processors=[
|
||||
structlog.contextvars.merge_contextvars,
|
||||
add_trace_context,
|
||||
log_capture,
|
||||
],
|
||||
logger_factory=structlog.stdlib.LoggerFactory(),
|
||||
wrapper_class=structlog.make_filtering_bound_logger(20),
|
||||
cache_logger_on_first_use=False,
|
||||
)
|
||||
logger = structlog.get_logger("test-logger")
|
||||
|
||||
clear_contextvars()
|
||||
bind_contextvars(request_id="req-test-id")
|
||||
with get_tracer(__name__).start_as_current_span("test-span"):
|
||||
logger.info("request_log")
|
||||
|
||||
assert len(log_capture.entries) == 1
|
||||
entry = log_capture.entries[0]
|
||||
assert entry["event"] == "request_log"
|
||||
assert entry["request_id"] == "req-test-id"
|
||||
assert len(entry["trace_id"]) == 32
|
||||
assert len(entry["span_id"]) == 16
|
||||
|
||||
clear_contextvars()
|
||||
reset_tracing_for_tests()
|
||||
@@ -0,0 +1,204 @@
|
||||
import asyncio
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
|
||||
|
||||
from app.config import Settings
|
||||
from app.controllers.v1.delivery import get_aggregator_service
|
||||
from app.observability.metrics import reset_metrics_for_tests, setup_metrics
|
||||
from app.schemas.request import DeliveryEntity, DeliveryRequest
|
||||
from app.schemas.response import DeliveryPrice
|
||||
from app.services.aggregator import AggregatorService, AggregatorServiceError
|
||||
|
||||
|
||||
class ToggleAggregatorService:
|
||||
def __init__(self) -> None:
|
||||
self.should_fail = False
|
||||
|
||||
async def get_all_prices(self, request: DeliveryRequest) -> list[DeliveryPrice]:
|
||||
_ = request
|
||||
if self.should_fail:
|
||||
raise AggregatorServiceError("failed")
|
||||
return []
|
||||
|
||||
|
||||
class StubProvider:
|
||||
def __init__(self, name: str, *, fail: bool) -> None:
|
||||
self.name = name
|
||||
self.fail = fail
|
||||
self.cache_ttl_seconds = 120
|
||||
|
||||
async def get_price(self, request: DeliveryRequest) -> DeliveryPrice:
|
||||
_ = request
|
||||
if self.fail:
|
||||
raise RuntimeError("provider unavailable")
|
||||
return DeliveryPrice(
|
||||
provider=self.name,
|
||||
service_name="economy",
|
||||
price=Decimal("99.90"),
|
||||
currency="RUB",
|
||||
delivery_days_min=2,
|
||||
delivery_days_max=3,
|
||||
)
|
||||
|
||||
|
||||
class StubCache:
|
||||
async def get(self, key: str) -> object | None:
|
||||
_ = key
|
||||
return None
|
||||
|
||||
async def set(self, key: str, value: object, ttl: int | None = None) -> None:
|
||||
_ = key, value, ttl
|
||||
|
||||
|
||||
class StubCacheHit:
|
||||
def __init__(self, payload: object) -> None:
|
||||
self.payload = payload
|
||||
|
||||
async def get(self, key: str) -> object | None:
|
||||
_ = key
|
||||
return self.payload
|
||||
|
||||
async def set(self, key: str, value: object, ttl: int | None = None) -> None:
|
||||
_ = key, value, ttl
|
||||
|
||||
|
||||
def _build_payload() -> dict[str, object]:
|
||||
return {
|
||||
"entity": "individual",
|
||||
"from_city": "Moscow",
|
||||
"to_city": "Kazan",
|
||||
"weight_kg": 2.5,
|
||||
"length_cm": 30.0,
|
||||
"width_cm": 20.0,
|
||||
"height_cm": 10.0,
|
||||
}
|
||||
|
||||
|
||||
def _build_request() -> DeliveryRequest:
|
||||
return DeliveryRequest(
|
||||
entity=DeliveryEntity.INDIVIDUAL,
|
||||
from_city="Moscow",
|
||||
to_city="Kazan",
|
||||
weight_kg=2.5,
|
||||
length_cm=30.0,
|
||||
width_cm=20.0,
|
||||
height_cm=10.0,
|
||||
)
|
||||
|
||||
|
||||
def _iter_data_points(metrics_data: Any, metric_name: str) -> list[Any]:
|
||||
points: list[Any] = []
|
||||
if metrics_data is None:
|
||||
return points
|
||||
for resource_metric in metrics_data.resource_metrics:
|
||||
for scope_metric in resource_metric.scope_metrics:
|
||||
for metric in scope_metric.metrics:
|
||||
if metric.name == metric_name:
|
||||
points.extend(metric.data.data_points)
|
||||
return points
|
||||
|
||||
|
||||
def test_metrics_export_requests_errors_latency_and_provider_availability() -> None:
|
||||
reset_metrics_for_tests()
|
||||
metric_reader = InMemoryMetricReader()
|
||||
meter_provider = MeterProvider(metric_readers=[metric_reader])
|
||||
setup_metrics(
|
||||
service_name="g2s-tests",
|
||||
otlp_endpoint="",
|
||||
meter_provider=meter_provider,
|
||||
)
|
||||
|
||||
from app.main import create_app
|
||||
|
||||
toggle_service = ToggleAggregatorService()
|
||||
app = create_app(
|
||||
settings=Settings(
|
||||
observability={
|
||||
"service_name": "g2s-tests",
|
||||
"otlp_endpoint": "",
|
||||
"log_level": "INFO",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def override_service() -> ToggleAggregatorService:
|
||||
return toggle_service
|
||||
|
||||
app.dependency_overrides[get_aggregator_service] = override_service
|
||||
|
||||
async def run_requests() -> tuple[httpx.Response, httpx.Response]:
|
||||
transport = httpx.ASGITransport(app=app, raise_app_exceptions=False)
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url="http://testserver",
|
||||
) as client:
|
||||
success = await client.post("/api/v1/delivery/price", json=_build_payload())
|
||||
toggle_service.should_fail = True
|
||||
failure = await client.post("/api/v1/delivery/price", json=_build_payload())
|
||||
return success, failure
|
||||
|
||||
success_response, failure_response = asyncio.run(run_requests())
|
||||
assert success_response.status_code == 200
|
||||
assert failure_response.status_code == 503
|
||||
|
||||
availability_service = AggregatorService(
|
||||
providers=[
|
||||
StubProvider("available-provider", fail=False),
|
||||
StubProvider("unavailable-provider", fail=True),
|
||||
],
|
||||
cache=StubCache(),
|
||||
)
|
||||
asyncio.run(availability_service.get_all_prices(_build_request()))
|
||||
|
||||
metrics_data = metric_reader.get_metrics_data()
|
||||
request_points = _iter_data_points(metrics_data, "http.server.request.count")
|
||||
error_points = _iter_data_points(metrics_data, "http.server.error.count")
|
||||
latency_points = _iter_data_points(metrics_data, "http.server.request.latency")
|
||||
availability_points = _iter_data_points(metrics_data, "delivery.provider.availability")
|
||||
|
||||
assert sum(point.value for point in request_points) == 2
|
||||
assert sum(point.value for point in error_points) == 1
|
||||
assert sum(point.count for point in latency_points) == 2
|
||||
assert sum(point.count for point in availability_points) == 2
|
||||
assert sorted(point.sum for point in availability_points) == [0.0, 1.0]
|
||||
|
||||
reset_metrics_for_tests()
|
||||
|
||||
|
||||
def test_provider_availability_not_recorded_on_cache_hit() -> None:
|
||||
reset_metrics_for_tests()
|
||||
metric_reader = InMemoryMetricReader()
|
||||
meter_provider = MeterProvider(metric_readers=[metric_reader])
|
||||
setup_metrics(
|
||||
service_name="g2s-tests",
|
||||
otlp_endpoint="",
|
||||
meter_provider=meter_provider,
|
||||
)
|
||||
|
||||
cached_price = DeliveryPrice(
|
||||
provider="cached-provider",
|
||||
service_name="economy",
|
||||
price=Decimal("95.50"),
|
||||
currency="RUB",
|
||||
delivery_days_min=2,
|
||||
delivery_days_max=3,
|
||||
).model_dump(mode="json")
|
||||
provider = StubProvider("cached-provider", fail=True)
|
||||
service = AggregatorService(
|
||||
providers=[provider],
|
||||
cache=StubCacheHit(cached_price),
|
||||
)
|
||||
|
||||
result = asyncio.run(service.get_all_prices(_build_request()))
|
||||
assert len(result) == 1
|
||||
assert result[0].provider == "cached-provider"
|
||||
|
||||
metrics_data = metric_reader.get_metrics_data()
|
||||
availability_points = _iter_data_points(metrics_data, "delivery.provider.availability")
|
||||
assert availability_points == []
|
||||
|
||||
reset_metrics_for_tests()
|
||||
@@ -0,0 +1,160 @@
|
||||
import asyncio
|
||||
from decimal import Decimal
|
||||
|
||||
import httpx
|
||||
from fastapi import FastAPI
|
||||
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from app.observability.tracing import (
|
||||
instrument_fastapi_app,
|
||||
instrument_httpx_client,
|
||||
reset_tracing_for_tests,
|
||||
setup_tracing,
|
||||
)
|
||||
from app.schemas.request import DeliveryEntity, DeliveryRequest
|
||||
from app.schemas.response import DeliveryPrice
|
||||
from app.services.aggregator import AggregatorService
|
||||
|
||||
|
||||
class StubProvider:
|
||||
name = "stub-provider"
|
||||
cache_ttl_seconds = 120
|
||||
|
||||
async def get_price(self, request: DeliveryRequest) -> DeliveryPrice:
|
||||
_ = request
|
||||
return DeliveryPrice(
|
||||
provider=self.name,
|
||||
service_name="economy",
|
||||
price=Decimal("100.10"),
|
||||
currency="RUB",
|
||||
delivery_days_min=2,
|
||||
delivery_days_max=4,
|
||||
)
|
||||
|
||||
|
||||
class StubCache:
|
||||
async def get(self, key: str) -> object | None:
|
||||
_ = key
|
||||
return None
|
||||
|
||||
async def set(self, key: str, value: object, ttl: int | None = None) -> None:
|
||||
_ = key, value, ttl
|
||||
|
||||
|
||||
def _build_request() -> DeliveryRequest:
|
||||
return DeliveryRequest(
|
||||
entity=DeliveryEntity.INDIVIDUAL,
|
||||
from_city="Moscow",
|
||||
to_city="Kazan",
|
||||
weight_kg=1.5,
|
||||
length_cm=10.0,
|
||||
width_cm=20.0,
|
||||
height_cm=30.0,
|
||||
)
|
||||
|
||||
|
||||
def test_manual_spans_capture_required_attributes() -> None:
|
||||
reset_tracing_for_tests()
|
||||
span_exporter = InMemorySpanExporter()
|
||||
tracer_provider = TracerProvider(resource=Resource.create({"service.name": "tests"}))
|
||||
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
|
||||
setup_tracing(
|
||||
service_name="tests",
|
||||
otlp_endpoint="",
|
||||
tracer_provider=tracer_provider,
|
||||
)
|
||||
|
||||
service = AggregatorService(providers=[StubProvider()], cache=StubCache())
|
||||
asyncio.run(service.get_all_prices(_build_request()))
|
||||
|
||||
spans = span_exporter.get_finished_spans()
|
||||
span_by_name = {span.name: span for span in spans}
|
||||
|
||||
assert "AggregatorService.get_all_prices" in span_by_name
|
||||
assert "DeliveryProvider.get_price" in span_by_name
|
||||
assert "PriceCache.get" in span_by_name
|
||||
assert "PriceCache.set" in span_by_name
|
||||
|
||||
service_span = span_by_name["AggregatorService.get_all_prices"]
|
||||
assert service_span.attributes["from_city"] == "Moscow"
|
||||
assert service_span.attributes["to_city"] == "Kazan"
|
||||
assert service_span.attributes["weight_kg"] == 1.5
|
||||
assert service_span.attributes["tariffs_found"] == 1
|
||||
|
||||
provider_span = span_by_name["DeliveryProvider.get_price"]
|
||||
assert provider_span.attributes["provider"] == "stub-provider"
|
||||
assert provider_span.attributes["cache_hit"] is False
|
||||
assert provider_span.attributes["from_city"] == "Moscow"
|
||||
assert provider_span.attributes["to_city"] == "Kazan"
|
||||
assert provider_span.attributes["weight_kg"] == 1.5
|
||||
|
||||
cache_span = span_by_name["PriceCache.get"]
|
||||
assert cache_span.attributes["cache_hit"] is False
|
||||
|
||||
reset_tracing_for_tests()
|
||||
|
||||
|
||||
def test_fastapi_and_httpx_instrumentation_emit_server_and_client_spans() -> None:
|
||||
reset_tracing_for_tests()
|
||||
span_exporter = InMemorySpanExporter()
|
||||
tracer_provider = TracerProvider(resource=Resource.create({"service.name": "tests"}))
|
||||
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
|
||||
setup_tracing(
|
||||
service_name="tests",
|
||||
otlp_endpoint="",
|
||||
tracer_provider=tracer_provider,
|
||||
)
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/proxy")
|
||||
async def proxy() -> dict[str, int]:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
_ = request
|
||||
return httpx.Response(status_code=200, json={"ok": True})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url="https://provider.test",
|
||||
) as client:
|
||||
HTTPXClientInstrumentor.instrument_client(client, tracer_provider=tracer_provider)
|
||||
response = await client.get("/quote")
|
||||
return {"status_code": response.status_code}
|
||||
|
||||
instrument_fastapi_app(app)
|
||||
instrument_httpx_client()
|
||||
|
||||
async def run_request() -> httpx.Response:
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url="http://testserver",
|
||||
) as client:
|
||||
return await client.get("/proxy")
|
||||
|
||||
response = asyncio.run(run_request())
|
||||
assert response.status_code == 200
|
||||
|
||||
spans = span_exporter.get_finished_spans()
|
||||
assert any(span.kind == SpanKind.SERVER for span in spans)
|
||||
assert any(span.kind == SpanKind.CLIENT for span in spans)
|
||||
assert any(
|
||||
(
|
||||
"provider.test/quote"
|
||||
in (
|
||||
span.attributes.get("http.url")
|
||||
or span.attributes.get("url.full")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
for span in spans
|
||||
if span.kind == SpanKind.CLIENT
|
||||
)
|
||||
|
||||
reset_tracing_for_tests()
|
||||
Reference in New Issue
Block a user