From 34d7a4c3369f65af0404b5450407ec5495521519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D0=B8=D1=81=20=D0=AE=D1=81=D1=83=D0=BF=D0=B0?= =?UTF-8?q?=D0=BB=D0=B8=D0=B5=D0=B2?= Date: Sun, 8 Mar 2026 12:04:42 +0300 Subject: [PATCH] 008 add telegram alerts --- app/config.py | 38 +++++++- config.example.yaml | 8 ++ docker-compose.yml | 13 +++ infra/alerts/README.md | 39 ++++++++ infra/alerts/signoz_alert_rules.yaml | 41 ++++++++ .../alerts/signoz_contact_point_telegram.yaml | 14 +++ spec/index.md | 8 +- .../tasks/008_add_signoz_telegram_alerting.md | 2 +- tests/config/test_alerts_config.py | 93 +++++++++++++++++++ tests/config/test_config_sections.py | 6 ++ 10 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 docker-compose.yml create mode 100644 infra/alerts/README.md create mode 100644 infra/alerts/signoz_alert_rules.yaml create mode 100644 infra/alerts/signoz_contact_point_telegram.yaml create mode 100644 tests/config/test_alerts_config.py diff --git a/app/config.py b/app/config.py index 52a555c..978497d 100644 --- a/app/config.py +++ b/app/config.py @@ -3,7 +3,7 @@ import os from functools import lru_cache -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator from pydantic_settings import ( BaseSettings, PydanticBaseSettingsSource, @@ -50,10 +50,46 @@ class ObservabilityConfig(BaseModel): log_level: str = "INFO" +class Provider5xxAlertConfig(BaseModel): + error_count: int = Field(default=5, ge=1) + window_minutes: int = Field(default=5, ge=1) + + +class ProviderP99LatencyAlertConfig(BaseModel): + threshold_ms: int = Field(default=5000, ge=1) + window_minutes: int = Field(default=10, ge=1) + + +class ProviderUnavailableAlertConfig(BaseModel): + duration_minutes: int = Field(default=5, ge=1) + + class AlertsConfig(BaseModel): telegram_enabled: bool = False telegram_bot_token: str | None = None telegram_chat_id: str | None = None + provider_5xx: Provider5xxAlertConfig = Field(default_factory=Provider5xxAlertConfig) + provider_p99_latency: ProviderP99LatencyAlertConfig = Field( + default_factory=ProviderP99LatencyAlertConfig + ) + provider_unavailable: ProviderUnavailableAlertConfig = Field( + default_factory=ProviderUnavailableAlertConfig + ) + + @model_validator(mode="after") + def validate_telegram_credentials(self) -> "AlertsConfig": + if not self.telegram_enabled: + return self + + if not self.telegram_bot_token: + raise ValueError( + "alerts.telegram_bot_token is required when alerts.telegram_enabled=true" + ) + if not self.telegram_chat_id: + raise ValueError( + "alerts.telegram_chat_id is required when alerts.telegram_enabled=true" + ) + return self class Settings(BaseSettings): diff --git a/config.example.yaml b/config.example.yaml index 71e9969..88712ef 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -31,3 +31,11 @@ alerts: telegram_enabled: false telegram_bot_token: null telegram_chat_id: null + provider_5xx: + error_count: 5 + window_minutes: 5 + provider_p99_latency: + threshold_ms: 5000 + window_minutes: 10 + provider_unavailable: + duration_minutes: 5 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f5dbb9d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +services: + signoz: + image: signoz/signoz:0.82.0 + container_name: signoz + environment: + G2S_ALERTS__TELEGRAM_BOT_TOKEN: ${G2S_ALERTS__TELEGRAM_BOT_TOKEN:-} + G2S_ALERTS__TELEGRAM_CHAT_ID: ${G2S_ALERTS__TELEGRAM_CHAT_ID:-} + SIGNOZ_ALERTS_CONTACT_POINT_FILE: /etc/signoz/alerts/signoz_contact_point_telegram.yaml + SIGNOZ_ALERTS_RULES_FILE: /etc/signoz/alerts/signoz_alert_rules.yaml + ports: + - "3301:3301" + volumes: + - ./infra/alerts:/etc/signoz/alerts:ro diff --git a/infra/alerts/README.md b/infra/alerts/README.md new file mode 100644 index 0000000..28f718b --- /dev/null +++ b/infra/alerts/README.md @@ -0,0 +1,39 @@ +# SigNoz -> Telegram alerting setup + +## Alerts config section example (`config.yaml`) + +```yaml +alerts: + telegram_enabled: true + telegram_bot_token: "123456789:telegram-bot-token" + telegram_chat_id: "-1000000000000" + provider_5xx: + error_count: 5 + window_minutes: 5 + provider_p99_latency: + threshold_ms: 5000 + window_minutes: 10 + provider_unavailable: + duration_minutes: 5 +``` + +## Routing artifacts + +- `signoz_contact_point_telegram.yaml`: + - webhook endpoint points to Telegram Bot API `sendMessage` + - uses `G2S_ALERTS__TELEGRAM_BOT_TOKEN` and `G2S_ALERTS__TELEGRAM_CHAT_ID` +- `signoz_alert_rules.yaml`: + - provider 5xx: `>= 5 errors / 5m` + - provider p99 latency: `> 5000ms / 10m` + - provider unavailable: `> 5m` + +## Local verification steps + +1. Export Telegram variables (must match `config.yaml -> alerts` values): + - `export G2S_ALERTS__TELEGRAM_BOT_TOKEN=""` + - `export G2S_ALERTS__TELEGRAM_CHAT_ID=""` +2. Validate compose syntax: + - `docker compose config` +3. In SigNoz UI, create a webhook contact point using `infra/alerts/signoz_contact_point_telegram.yaml`. +4. Create three alert rules from `infra/alerts/signoz_alert_rules.yaml`. +5. Use SigNoz "Test alert" action and verify message delivery in Telegram chat. diff --git a/infra/alerts/signoz_alert_rules.yaml b/infra/alerts/signoz_alert_rules.yaml new file mode 100644 index 0000000..ab0995b --- /dev/null +++ b/infra/alerts/signoz_alert_rules.yaml @@ -0,0 +1,41 @@ +rules: + - name: "provider-5xx-errors" + description: "Provider returned >= 5 server errors in 5 minutes." + severity: "critical" + condition: + query: | + sum(increase(g2s_provider_errors_total{status_code=~"5.."}[5m])) by (provider) >= 5 + evaluate_for: "0m" + threshold: + error_count: 5 + window_minutes: 5 + annotations: + summary: "Provider {{ $labels.provider }} returned >= 5 5xx responses during last 5m." + + - name: "provider-p99-latency" + description: "Provider p99 latency is above 5000ms for 10 minutes." + severity: "warning" + condition: + query: | + histogram_quantile( + 0.99, + sum(rate(g2s_provider_latency_ms_bucket[10m])) by (provider, le) + ) > 5000 + evaluate_for: "10m" + threshold: + threshold_ms: 5000 + window_minutes: 10 + annotations: + summary: "Provider {{ $labels.provider }} p99 latency > 5000ms for 10m." + + - name: "provider-unavailable" + description: "Provider is unavailable for more than 5 minutes." + severity: "critical" + condition: + query: | + max_over_time(g2s_provider_available[5m]) == 0 + evaluate_for: "5m" + threshold: + duration_minutes: 5 + annotations: + summary: "Provider {{ $labels.provider }} is unavailable for more than 5m." diff --git a/infra/alerts/signoz_contact_point_telegram.yaml b/infra/alerts/signoz_contact_point_telegram.yaml new file mode 100644 index 0000000..fca8db1 --- /dev/null +++ b/infra/alerts/signoz_contact_point_telegram.yaml @@ -0,0 +1,14 @@ +contact_point: + name: "telegram-webhook" + type: "webhook" + settings: + method: "POST" + url: "https://api.telegram.org/bot${G2S_ALERTS__TELEGRAM_BOT_TOKEN}/sendMessage" + headers: + Content-Type: "application/json" + body: | + { + "chat_id": "${G2S_ALERTS__TELEGRAM_CHAT_ID}", + "text": "[{{ .Status }}] {{ .RuleName }}\nProvider: {{ index .Labels \"provider\" }}\nSummary: {{ index .Annotations \"summary\" }}", + "disable_notification": false + } diff --git a/spec/index.md b/spec/index.md index 996b2ba..ac7aa47 100644 --- a/spec/index.md +++ b/spec/index.md @@ -1,7 +1,7 @@ # Spec Tasks Index > ⚠️ This file is generated. Do not edit manually. -> Generated at (UTC): `2026-03-08T08:42:58+00:00` +> Generated at (UTC): `2026-03-08T09:04:29+00:00` ## Tasks @@ -15,11 +15,11 @@ | 005 | DONE | 2026-03-07 | Implement aggregator service workflow | `spec/tasks/005_implement_aggregator_service_workflow.md` | | 006 | DONE | 2026-03-07 | Add delivery price controller endpoint | `spec/tasks/006_add_delivery_price_controller.md` | | 007 | DONE | 2026-03-07 | Add observability correlation and telemetry | `spec/tasks/007_add_observability_correlation.md` | -| 008 | TODO | 2026-03-07 | Add SigNoz to Telegram alerting configuration | `spec/tasks/008_add_signoz_telegram_alerting.md` | +| 008 | DONE | 2026-03-07 | Add SigNoz to Telegram alerting configuration | `spec/tasks/008_add_signoz_telegram_alerting.md` | | 009 | TODO | 2026-03-07 | Add local infrastructure stack | `spec/tasks/009_add_local_infra_stack.md` | ## Summary - Total: **10** -- TODO: **2** -- DONE: **8** +- TODO: **1** +- DONE: **9** diff --git a/spec/tasks/008_add_signoz_telegram_alerting.md b/spec/tasks/008_add_signoz_telegram_alerting.md index 98901f9..195a239 100644 --- a/spec/tasks/008_add_signoz_telegram_alerting.md +++ b/spec/tasks/008_add_signoz_telegram_alerting.md @@ -1,7 +1,7 @@ --- id: 008 title: Add SigNoz to Telegram alerting configuration -status: TODO +status: DONE created: 2026-03-07 --- diff --git a/tests/config/test_alerts_config.py b/tests/config/test_alerts_config.py new file mode 100644 index 0000000..2cf8aac --- /dev/null +++ b/tests/config/test_alerts_config.py @@ -0,0 +1,93 @@ +import pytest +from pydantic import ValidationError + +from app.config import AlertsConfig, Settings + + +def _clear_alert_env(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "G2S_ALERTS__TELEGRAM_ENABLED", + "G2S_ALERTS__TELEGRAM_BOT_TOKEN", + "G2S_ALERTS__TELEGRAM_CHAT_ID", + ): + monkeypatch.delenv(name, raising=False) + + +def test_alerts_settings_are_loaded_from_yaml(tmp_path, monkeypatch) -> None: + _clear_alert_env(monkeypatch) + config_file = tmp_path / "config.yaml" + config_file.write_text( + """ +alerts: + telegram_enabled: true + telegram_bot_token: "bot-token" + telegram_chat_id: "-100777" + provider_5xx: + error_count: 7 + window_minutes: 6 + provider_p99_latency: + threshold_ms: 5200 + window_minutes: 11 + provider_unavailable: + duration_minutes: 8 +""".strip(), + encoding="utf-8", + ) + monkeypatch.setenv("G2S_CONFIG_FILE", str(config_file)) + + settings = Settings() + + assert settings.alerts.telegram_enabled is True + assert settings.alerts.telegram_bot_token == "bot-token" + assert settings.alerts.telegram_chat_id == "-100777" + assert settings.alerts.provider_5xx.error_count == 7 + assert settings.alerts.provider_5xx.window_minutes == 6 + assert settings.alerts.provider_p99_latency.threshold_ms == 5200 + assert settings.alerts.provider_p99_latency.window_minutes == 11 + assert settings.alerts.provider_unavailable.duration_minutes == 8 + + +@pytest.mark.parametrize( + ("yaml_body", "error_message"), + [ + ( + """ +alerts: + telegram_enabled: true + telegram_chat_id: "-100777" +""".strip(), + "alerts.telegram_bot_token is required when alerts.telegram_enabled=true", + ), + ( + """ +alerts: + telegram_enabled: true + telegram_bot_token: "bot-token" +""".strip(), + "alerts.telegram_chat_id is required when alerts.telegram_enabled=true", + ), + ], +) +def test_enabled_telegram_requires_credentials( + tmp_path, + monkeypatch, + yaml_body: str, + error_message: str, +) -> None: + _clear_alert_env(monkeypatch) + config_file = tmp_path / "config.yaml" + config_file.write_text(yaml_body, encoding="utf-8") + monkeypatch.setenv("G2S_CONFIG_FILE", str(config_file)) + + with pytest.raises(ValidationError, match=error_message): + Settings() + + +def test_alert_condition_defaults_match_required_thresholds() -> None: + alerts = AlertsConfig() + + assert alerts.provider_5xx.error_count == 5 + assert alerts.provider_5xx.window_minutes == 5 + assert alerts.provider_p99_latency.threshold_ms == 5000 + assert alerts.provider_p99_latency.window_minutes == 10 + assert alerts.provider_unavailable.duration_minutes == 5 diff --git a/tests/config/test_config_sections.py b/tests/config/test_config_sections.py index f531400..27e11ce 100644 --- a/tests/config/test_config_sections.py +++ b/tests/config/test_config_sections.py @@ -15,6 +15,8 @@ def test_configuration_sections_are_loaded_from_env(monkeypatch) -> None: monkeypatch.setenv("G2S_ADAPTER__CDEK_CACHE_TTL_SECONDS", "780") monkeypatch.setenv("G2S_OBSERVABILITY__SERVICE_NAME", "g2s-test") monkeypatch.setenv("G2S_ALERTS__TELEGRAM_ENABLED", "true") + monkeypatch.setenv("G2S_ALERTS__TELEGRAM_BOT_TOKEN", "bot-token") + monkeypatch.setenv("G2S_ALERTS__TELEGRAM_CHAT_ID", "-100") settings = Settings() @@ -57,6 +59,8 @@ observability: service_name: "g2s-yaml" alerts: telegram_enabled: true + telegram_bot_token: "yaml-bot-token" + telegram_chat_id: "-100100" """.strip(), encoding="utf-8", ) @@ -77,6 +81,8 @@ alerts: assert settings.adapter.cdek_cache_ttl_seconds == 810 assert settings.observability.service_name == "g2s-yaml" assert settings.alerts.telegram_enabled is True + assert settings.alerts.telegram_bot_token == "yaml-bot-token" + assert settings.alerts.telegram_chat_id == "-100100" def test_get_settings_returns_cached_instance(monkeypatch) -> None: