008 add telegram alerts

2026-03-08 12:04:42 +03:00
parent 8d47917ba2
commit 34d7a4c336
10 changed files with 256 additions and 6 deletions
@@ -0,0 +1,39 @@
+# SigNoz -> Telegram alerting setup
+
+## Alerts config section example (`config.yaml`)
+
+```yaml
+alerts:
+  telegram_enabled: true
+  telegram_bot_token: "123456789:telegram-bot-token"
+  telegram_chat_id: "-1000000000000"
+  provider_5xx:
+    error_count: 5
+    window_minutes: 5
+  provider_p99_latency:
+    threshold_ms: 5000
+    window_minutes: 10
+  provider_unavailable:
+    duration_minutes: 5
+```
+
+## Routing artifacts
+
+- `signoz_contact_point_telegram.yaml`:
+  - webhook endpoint points to Telegram Bot API `sendMessage`
+  - uses `G2S_ALERTS__TELEGRAM_BOT_TOKEN` and `G2S_ALERTS__TELEGRAM_CHAT_ID`
+- `signoz_alert_rules.yaml`:
+  - provider 5xx: `>= 5 errors / 5m`
+  - provider p99 latency: `> 5000ms / 10m`
+  - provider unavailable: `> 5m`
+
+## Local verification steps
+
+1. Export Telegram variables (must match `config.yaml -> alerts` values):
+   - `export G2S_ALERTS__TELEGRAM_BOT_TOKEN="<bot_token>"`
+   - `export G2S_ALERTS__TELEGRAM_CHAT_ID="<chat_id>"`
+2. Validate compose syntax:
+   - `docker compose config`
+3. In SigNoz UI, create a webhook contact point using `infra/alerts/signoz_contact_point_telegram.yaml`.
+4. Create three alert rules from `infra/alerts/signoz_alert_rules.yaml`.
+5. Use SigNoz "Test alert" action and verify message delivery in Telegram chat.
@@ -0,0 +1,41 @@
+rules:
+  - name: "provider-5xx-errors"
+    description: "Provider returned >= 5 server errors in 5 minutes."
+    severity: "critical"
+    condition:
+      query: |
+        sum(increase(g2s_provider_errors_total{status_code=~"5.."}[5m])) by (provider) >= 5
+      evaluate_for: "0m"
+      threshold:
+        error_count: 5
+        window_minutes: 5
+    annotations:
+      summary: "Provider {{ $labels.provider }} returned >= 5 5xx responses during last 5m."
+
+  - name: "provider-p99-latency"
+    description: "Provider p99 latency is above 5000ms for 10 minutes."
+    severity: "warning"
+    condition:
+      query: |
+        histogram_quantile(
+          0.99,
+          sum(rate(g2s_provider_latency_ms_bucket[10m])) by (provider, le)
+        ) > 5000
+      evaluate_for: "10m"
+      threshold:
+        threshold_ms: 5000
+        window_minutes: 10
+    annotations:
+      summary: "Provider {{ $labels.provider }} p99 latency > 5000ms for 10m."
+
+  - name: "provider-unavailable"
+    description: "Provider is unavailable for more than 5 minutes."
+    severity: "critical"
+    condition:
+      query: |
+        max_over_time(g2s_provider_available[5m]) == 0
+      evaluate_for: "5m"
+      threshold:
+        duration_minutes: 5
+    annotations:
+      summary: "Provider {{ $labels.provider }} is unavailable for more than 5m."
@@ -0,0 +1,14 @@
+contact_point:
+  name: "telegram-webhook"
+  type: "webhook"
+  settings:
+    method: "POST"
+    url: "https://api.telegram.org/bot${G2S_ALERTS__TELEGRAM_BOT_TOKEN}/sendMessage"
+    headers:
+      Content-Type: "application/json"
+    body: |
+      {
+        "chat_id": "${G2S_ALERTS__TELEGRAM_CHAT_ID}",
+        "text": "[{{ .Status }}] {{ .RuleName }}\nProvider: {{ index .Labels \"provider\" }}\nSummary: {{ index .Annotations \"summary\" }}",
+        "disable_notification": false
+      }