g2s-aggregator/infra/alerts/signoz_alert_rules.yaml

rules:
  - name: "provider-5xx-errors"
    description: "Provider returned >= 5 server errors in 5 minutes."
    severity: "critical"
    condition:
      query: |
        sum(increase(g2s_provider_errors_total{status_code=~"5.."}[5m])) by (provider) >= 5
      evaluate_for: "0m"
      threshold:
        error_count: 5
        window_minutes: 5
    annotations:
      summary: "Provider {{ $labels.provider }} returned >= 5 5xx responses during last 5m."

  - name: "provider-p99-latency"
    description: "Provider p99 latency is above 5000ms for 10 minutes."
    severity: "warning"
    condition:
      query: |
        histogram_quantile(
          0.99,
          sum(rate(g2s_provider_latency_ms_bucket[10m])) by (provider, le)
        ) > 5000
      evaluate_for: "10m"
      threshold:
        threshold_ms: 5000
        window_minutes: 10
    annotations:
      summary: "Provider {{ $labels.provider }} p99 latency > 5000ms for 10m."

  - name: "provider-unavailable"
    description: "Provider is unavailable for more than 5 minutes."
    severity: "critical"
    condition:
      query: |
        max_over_time(g2s_provider_available[5m]) == 0
      evaluate_for: "5m"
      threshold:
        duration_minutes: 5
    annotations:
      summary: "Provider {{ $labels.provider }} is unavailable for more than 5m."