spb/datacenterindex

1# Alerting rules (evaluated by Prometheus; visible in Grafana → Alerting and on the overview dashboard).2# No Alertmanager is deployed: these are state indicators, not pages.3groups:4  - name: dci5    rules:6      - alert: TargetDown7        expr: up == 08        for: 3m9        labels: { severity: serious }10        annotations: { summary: "{{ $labels.job }} {{ $labels.instance }} is down" }1112      - alert: ApiHighLatency13        expr: histogram_quantile(0.95, sum(rate(dci_api_request_duration_seconds_bucket[5m])) by (le)) > 114        for: 10m15        labels: { severity: warning }16        annotations: { summary: "API p95 latency above 1 s for 10 min" }1718      - alert: ApiErrorRate19        expr: sum(rate(dci_api_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(dci_api_requests_total[5m])), 0.001) > 0.0520        for: 10m21        labels: { severity: serious }22        annotations: { summary: "API 5xx ratio above 5 %" }2324      - alert: CrawlStalled25        expr: sum(rate(dci_crawl_fetches_total[30m])) == 0 and sum(dci_queue_jobs{state="waiting"}) > 026        for: 30m27        labels: { severity: warning }28        annotations: { summary: "Jobs are waiting but no fetch completed in 30 min" }2930      - alert: PremiumBudgetNearlyExhausted31        # shared (Redis) usage today vs. the configured daily cap — survives worker restarts, unlike increase()32        expr: max by (provider) (dci_crawl_daily_credits_used) / on (provider) max by (provider) (dci_crawl_daily_budget) > 0.933        for: 5m34        labels: { severity: warning }35        annotations: { summary: "{{ $labels.provider }} premium credits above 90 % of the daily budget" }3637      - alert: SchedulerStale38        expr: (time() - max(dci_scheduler_last_tick_timestamp_seconds{role="scheduler"})) > 60039        for: 5m40        labels: { severity: serious }41        annotations: { summary: "No successful scheduler pass in 10 min (dedicated scheduler)" }4243      - alert: WorkerDown44        expr: max(dci_worker_up{instance_name="worker"}) == 0 or absent(dci_worker_up{instance_name="worker"})45        for: 5m46        labels: { severity: serious }47        annotations: { summary: "The crawl worker on BHS64b is not accepting jobs" }4849      - alert: CrawlRunsFailing50        expr: sum(increase(dci_crawl_runs_total{status="failed"}[1h])) > 351        for: 10m52        labels: { severity: warning }53        annotations: { summary: "More than 3 connector runs failed in the last hour" }5455      - alert: FetchErrorRateHigh56        expr: sum(rate(dci_crawl_fetches_total{outcome=~"error|blocked"}[30m])) / clamp_min(sum(rate(dci_crawl_fetches_total[30m])), 0.001) > 0.357        for: 30m58        labels: { severity: warning }59        annotations: { summary: "More than 30 % of fetches end in error/blocked over 30 min" }6061      - alert: QueueBacklog62        expr: sum(dci_queue_jobs{queue="crawl",state=~"waiting|prioritized"}) > 5063        for: 30m64        labels: { severity: warning }65        annotations: { summary: "Crawl queue backlog above 50 jobs for 30 min — add a worker (profile scale) or raise DCI_CRAWL_CONCURRENCY" }6667      - alert: DiskAlmostFull68        expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) < 0.1069        for: 15m70        labels: { severity: critical }71        annotations: { summary: "Root filesystem below 10 % free on {{ $labels.node }}" }7273      - alert: MemoryPressure74        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 0.9275        for: 15m76        labels: { severity: serious }77        annotations: { summary: "Less than 8 % memory available on {{ $labels.node }}" }7879      - alert: PostgresConnectionsHigh80        expr: sum(pg_stat_activity_count) > 17081        for: 10m82        labels: { severity: warning }83        annotations: { summary: "Postgres connections above 170/200" }84