008 add telegram alerts

app/config.py

@@ -3,7 +3,7 @@
 import os
 from functools import lru_cache
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 from pydantic_settings import (
     BaseSettings,
     PydanticBaseSettingsSource,
@@ -50,10 +50,46 @@ class ObservabilityConfig(BaseModel):
     log_level: str = "INFO"
 
 
+class Provider5xxAlertConfig(BaseModel):
+    error_count: int = Field(default=5, ge=1)
+    window_minutes: int = Field(default=5, ge=1)
+
+
+class ProviderP99LatencyAlertConfig(BaseModel):
+    threshold_ms: int = Field(default=5000, ge=1)
+    window_minutes: int = Field(default=10, ge=1)
+
+
+class ProviderUnavailableAlertConfig(BaseModel):
+    duration_minutes: int = Field(default=5, ge=1)
+
+
 class AlertsConfig(BaseModel):
     telegram_enabled: bool = False
     telegram_bot_token: str | None = None
     telegram_chat_id: str | None = None
+    provider_5xx: Provider5xxAlertConfig = Field(default_factory=Provider5xxAlertConfig)
+    provider_p99_latency: ProviderP99LatencyAlertConfig = Field(
+        default_factory=ProviderP99LatencyAlertConfig
+    )
+    provider_unavailable: ProviderUnavailableAlertConfig = Field(
+        default_factory=ProviderUnavailableAlertConfig
+    )
+
+    @model_validator(mode="after")
+    def validate_telegram_credentials(self) -> "AlertsConfig":
+        if not self.telegram_enabled:
+            return self
+
+        if not self.telegram_bot_token:
+            raise ValueError(
+                "alerts.telegram_bot_token is required when alerts.telegram_enabled=true"
+            )
+        if not self.telegram_chat_id:
+            raise ValueError(
+                "alerts.telegram_chat_id is required when alerts.telegram_enabled=true"
+            )
+        return self
 
 
 class Settings(BaseSettings):

config.example.yaml

@@ -31,3 +31,11 @@ alerts:
   telegram_enabled: false
   telegram_bot_token: null
   telegram_chat_id: null
+  provider_5xx:
+    error_count: 5
+    window_minutes: 5
+  provider_p99_latency:
+    threshold_ms: 5000
+    window_minutes: 10
+  provider_unavailable:
+    duration_minutes: 5

docker-compose.yml

@@ -0,0 +1,13 @@
+services:
+  signoz:
+    image: signoz/signoz:0.82.0
+    container_name: signoz
+    environment:
+      G2S_ALERTS__TELEGRAM_BOT_TOKEN: ${G2S_ALERTS__TELEGRAM_BOT_TOKEN:-}
+      G2S_ALERTS__TELEGRAM_CHAT_ID: ${G2S_ALERTS__TELEGRAM_CHAT_ID:-}
+      SIGNOZ_ALERTS_CONTACT_POINT_FILE: /etc/signoz/alerts/signoz_contact_point_telegram.yaml
+      SIGNOZ_ALERTS_RULES_FILE: /etc/signoz/alerts/signoz_alert_rules.yaml
+    ports:
+      - "3301:3301"
+    volumes:
+      - ./infra/alerts:/etc/signoz/alerts:ro

infra/alerts/README.md

@@ -0,0 +1,39 @@
+# SigNoz -> Telegram alerting setup
+
+## Alerts config section example (`config.yaml`)
+
+```yaml
+alerts:
+  telegram_enabled: true
+  telegram_bot_token: "123456789:telegram-bot-token"
+  telegram_chat_id: "-1000000000000"
+  provider_5xx:
+    error_count: 5
+    window_minutes: 5
+  provider_p99_latency:
+    threshold_ms: 5000
+    window_minutes: 10
+  provider_unavailable:
+    duration_minutes: 5
+```
+
+## Routing artifacts
+
+- `signoz_contact_point_telegram.yaml`:
+  - webhook endpoint points to Telegram Bot API `sendMessage`
+  - uses `G2S_ALERTS__TELEGRAM_BOT_TOKEN` and `G2S_ALERTS__TELEGRAM_CHAT_ID`
+- `signoz_alert_rules.yaml`:
+  - provider 5xx: `>= 5 errors / 5m`
+  - provider p99 latency: `> 5000ms / 10m`
+  - provider unavailable: `> 5m`
+
+## Local verification steps
+
+1. Export Telegram variables (must match `config.yaml -> alerts` values):
+   - `export G2S_ALERTS__TELEGRAM_BOT_TOKEN="<bot_token>"`
+   - `export G2S_ALERTS__TELEGRAM_CHAT_ID="<chat_id>"`
+2. Validate compose syntax:
+   - `docker compose config`
+3. In SigNoz UI, create a webhook contact point using `infra/alerts/signoz_contact_point_telegram.yaml`.
+4. Create three alert rules from `infra/alerts/signoz_alert_rules.yaml`.
+5. Use SigNoz "Test alert" action and verify message delivery in Telegram chat.

infra/alerts/signoz_alert_rules.yaml

@@ -0,0 +1,41 @@
+rules:
+  - name: "provider-5xx-errors"
+    description: "Provider returned >= 5 server errors in 5 minutes."
+    severity: "critical"
+    condition:
+      query: |
+        sum(increase(g2s_provider_errors_total{status_code=~"5.."}[5m])) by (provider) >= 5
+      evaluate_for: "0m"
+      threshold:
+        error_count: 5
+        window_minutes: 5
+    annotations:
+      summary: "Provider {{ $labels.provider }} returned >= 5 5xx responses during last 5m."
+
+  - name: "provider-p99-latency"
+    description: "Provider p99 latency is above 5000ms for 10 minutes."
+    severity: "warning"
+    condition:
+      query: |
+        histogram_quantile(
+          0.99,
+          sum(rate(g2s_provider_latency_ms_bucket[10m])) by (provider, le)
+        ) > 5000
+      evaluate_for: "10m"
+      threshold:
+        threshold_ms: 5000
+        window_minutes: 10
+    annotations:
+      summary: "Provider {{ $labels.provider }} p99 latency > 5000ms for 10m."
+
+  - name: "provider-unavailable"
+    description: "Provider is unavailable for more than 5 minutes."
+    severity: "critical"
+    condition:
+      query: |
+        max_over_time(g2s_provider_available[5m]) == 0
+      evaluate_for: "5m"
+      threshold:
+        duration_minutes: 5
+    annotations:
+      summary: "Provider {{ $labels.provider }} is unavailable for more than 5m."

infra/alerts/signoz_contact_point_telegram.yaml

@@ -0,0 +1,14 @@
+contact_point:
+  name: "telegram-webhook"
+  type: "webhook"
+  settings:
+    method: "POST"
+    url: "https://api.telegram.org/bot${G2S_ALERTS__TELEGRAM_BOT_TOKEN}/sendMessage"
+    headers:
+      Content-Type: "application/json"
+    body: |
+      {
+        "chat_id": "${G2S_ALERTS__TELEGRAM_CHAT_ID}",
+        "text": "[{{ .Status }}] {{ .RuleName }}\nProvider: {{ index .Labels \"provider\" }}\nSummary: {{ index .Annotations \"summary\" }}",
+        "disable_notification": false
+      }

spec/index.md

@@ -1,7 +1,7 @@
 # Spec Tasks Index
 
 > ⚠️ This file is generated. Do not edit manually.
-> Generated at (UTC): `2026-03-08T08:42:58+00:00`
+> Generated at (UTC): `2026-03-08T09:04:29+00:00`
 
 ## Tasks
 
@@ -15,11 +15,11 @@
 | 005 | DONE | 2026-03-07 | Implement aggregator service workflow | `spec/tasks/005_implement_aggregator_service_workflow.md` |
 | 006 | DONE | 2026-03-07 | Add delivery price controller endpoint | `spec/tasks/006_add_delivery_price_controller.md` |
 | 007 | DONE | 2026-03-07 | Add observability correlation and telemetry | `spec/tasks/007_add_observability_correlation.md` |
-| 008 | TODO | 2026-03-07 | Add SigNoz to Telegram alerting configuration | `spec/tasks/008_add_signoz_telegram_alerting.md` |
+| 008 | DONE | 2026-03-07 | Add SigNoz to Telegram alerting configuration | `spec/tasks/008_add_signoz_telegram_alerting.md` |
 | 009 | TODO | 2026-03-07 | Add local infrastructure stack | `spec/tasks/009_add_local_infra_stack.md` |
 
 ## Summary
 
 - Total: **10**
-- TODO: **2**
-- DONE: **8**
+- TODO: **1**
+- DONE: **9**

spec/tasks/008_add_signoz_telegram_alerting.md

@@ -1,7 +1,7 @@
 ---
 id: 008
 title: Add SigNoz to Telegram alerting configuration
-status: TODO
+status: DONE
 created: 2026-03-07
 ---
 

tests/config/test_alerts_config.py

@@ -0,0 +1,93 @@
+import pytest
+from pydantic import ValidationError
+
+from app.config import AlertsConfig, Settings
+
+
+def _clear_alert_env(monkeypatch: pytest.MonkeyPatch) -> None:
+    for name in (
+        "G2S_ALERTS__TELEGRAM_ENABLED",
+        "G2S_ALERTS__TELEGRAM_BOT_TOKEN",
+        "G2S_ALERTS__TELEGRAM_CHAT_ID",
+    ):
+        monkeypatch.delenv(name, raising=False)
+
+
+def test_alerts_settings_are_loaded_from_yaml(tmp_path, monkeypatch) -> None:
+    _clear_alert_env(monkeypatch)
+    config_file = tmp_path / "config.yaml"
+    config_file.write_text(
+        """
+alerts:
+  telegram_enabled: true
+  telegram_bot_token: "bot-token"
+  telegram_chat_id: "-100777"
+  provider_5xx:
+    error_count: 7
+    window_minutes: 6
+  provider_p99_latency:
+    threshold_ms: 5200
+    window_minutes: 11
+  provider_unavailable:
+    duration_minutes: 8
+""".strip(),
+        encoding="utf-8",
+    )
+    monkeypatch.setenv("G2S_CONFIG_FILE", str(config_file))
+
+    settings = Settings()
+
+    assert settings.alerts.telegram_enabled is True
+    assert settings.alerts.telegram_bot_token == "bot-token"
+    assert settings.alerts.telegram_chat_id == "-100777"
+    assert settings.alerts.provider_5xx.error_count == 7
+    assert settings.alerts.provider_5xx.window_minutes == 6
+    assert settings.alerts.provider_p99_latency.threshold_ms == 5200
+    assert settings.alerts.provider_p99_latency.window_minutes == 11
+    assert settings.alerts.provider_unavailable.duration_minutes == 8
+
+
+@pytest.mark.parametrize(
+    ("yaml_body", "error_message"),
+    [
+        (
+            """
+alerts:
+  telegram_enabled: true
+  telegram_chat_id: "-100777"
+""".strip(),
+            "alerts.telegram_bot_token is required when alerts.telegram_enabled=true",
+        ),
+        (
+            """
+alerts:
+  telegram_enabled: true
+  telegram_bot_token: "bot-token"
+""".strip(),
+            "alerts.telegram_chat_id is required when alerts.telegram_enabled=true",
+        ),
+    ],
+)
+def test_enabled_telegram_requires_credentials(
+    tmp_path,
+    monkeypatch,
+    yaml_body: str,
+    error_message: str,
+) -> None:
+    _clear_alert_env(monkeypatch)
+    config_file = tmp_path / "config.yaml"
+    config_file.write_text(yaml_body, encoding="utf-8")
+    monkeypatch.setenv("G2S_CONFIG_FILE", str(config_file))
+
+    with pytest.raises(ValidationError, match=error_message):
+        Settings()
+
+
+def test_alert_condition_defaults_match_required_thresholds() -> None:
+    alerts = AlertsConfig()
+
+    assert alerts.provider_5xx.error_count == 5
+    assert alerts.provider_5xx.window_minutes == 5
+    assert alerts.provider_p99_latency.threshold_ms == 5000
+    assert alerts.provider_p99_latency.window_minutes == 10
+    assert alerts.provider_unavailable.duration_minutes == 5

tests/config/test_config_sections.py

@@ -15,6 +15,8 @@ def test_configuration_sections_are_loaded_from_env(monkeypatch) -> None:
     monkeypatch.setenv("G2S_ADAPTER__CDEK_CACHE_TTL_SECONDS", "780")
     monkeypatch.setenv("G2S_OBSERVABILITY__SERVICE_NAME", "g2s-test")
     monkeypatch.setenv("G2S_ALERTS__TELEGRAM_ENABLED", "true")
+    monkeypatch.setenv("G2S_ALERTS__TELEGRAM_BOT_TOKEN", "bot-token")
+    monkeypatch.setenv("G2S_ALERTS__TELEGRAM_CHAT_ID", "-100")
 
     settings = Settings()
 
@@ -57,6 +59,8 @@ observability:
   service_name: "g2s-yaml"
 alerts:
   telegram_enabled: true
+  telegram_bot_token: "yaml-bot-token"
+  telegram_chat_id: "-100100"
 """.strip(),
         encoding="utf-8",
     )
@@ -77,6 +81,8 @@ alerts:
     assert settings.adapter.cdek_cache_ttl_seconds == 810
     assert settings.observability.service_name == "g2s-yaml"
     assert settings.alerts.telegram_enabled is True
+    assert settings.alerts.telegram_bot_token == "yaml-bot-token"
+    assert settings.alerts.telegram_chat_id == "-100100"
 
 
 def test_get_settings_returns_cached_instance(monkeypatch) -> None: