# Alerting rules (evaluated by Prometheus; visible in Grafana → Alerting and on the overview dashboard).
# No Alertmanager is deployed: these are state indicators, not pages.
groups:
  - name: dci
    rules:
      - alert: TargetDown
        expr: up == 0
        for: 3m
        labels: { severity: serious }
        annotations: { summary: "{{ $labels.job }} {{ $labels.instance }} is down" }

      - alert: ApiHighLatency
        expr: histogram_quantile(0.95, sum(rate(dci_api_request_duration_seconds_bucket[5m])) by (le)) > 1
        for: 10m
        labels: { severity: warning }
        annotations: { summary: "API p95 latency above 1 s for 10 min" }

      - alert: ApiErrorRate
        expr: sum(rate(dci_api_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(dci_api_requests_total[5m])), 0.001) > 0.05
        for: 10m
        labels: { severity: serious }
        annotations: { summary: "API 5xx ratio above 5 %" }

      - alert: CrawlStalled
        expr: sum(rate(dci_crawl_fetches_total[30m])) == 0 and sum(dci_queue_jobs{state="waiting"}) > 0
        for: 30m
        labels: { severity: warning }
        annotations: { summary: "Jobs are waiting but no fetch completed in 30 min" }

      - alert: PremiumBudgetNearlyExhausted
        # shared (Redis) usage today vs. the configured daily cap — survives worker restarts, unlike increase()
        expr: max by (provider) (dci_crawl_daily_credits_used) / on (provider) max by (provider) (dci_crawl_daily_budget) > 0.9
        for: 5m
        labels: { severity: warning }
        annotations: { summary: "{{ $labels.provider }} premium credits above 90 % of the daily budget" }

      - alert: SchedulerStale
        expr: (time() - max(dci_scheduler_last_tick_timestamp_seconds{role="scheduler"})) > 600
        for: 5m
        labels: { severity: serious }
        annotations: { summary: "No successful scheduler pass in 10 min (dedicated scheduler)" }

      - alert: WorkerDown
        expr: max(dci_worker_up{instance_name="worker"}) == 0 or absent(dci_worker_up{instance_name="worker"})
        for: 5m
        labels: { severity: serious }
        annotations: { summary: "The crawl worker on BHS64b is not accepting jobs" }

      - alert: CrawlRunsFailing
        expr: sum(increase(dci_crawl_runs_total{status="failed"}[1h])) > 3
        for: 10m
        labels: { severity: warning }
        annotations: { summary: "More than 3 connector runs failed in the last hour" }

      - alert: FetchErrorRateHigh
        expr: sum(rate(dci_crawl_fetches_total{outcome=~"error|blocked"}[30m])) / clamp_min(sum(rate(dci_crawl_fetches_total[30m])), 0.001) > 0.3
        for: 30m
        labels: { severity: warning }
        annotations: { summary: "More than 30 % of fetches end in error/blocked over 30 min" }

      - alert: QueueBacklog
        expr: sum(dci_queue_jobs{queue="crawl",state=~"waiting|prioritized"}) > 50
        for: 30m
        labels: { severity: warning }
        annotations: { summary: "Crawl queue backlog above 50 jobs for 30 min — add a worker (profile scale) or raise DCI_CRAWL_CONCURRENCY" }

      - alert: DiskAlmostFull
        expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) < 0.10
        for: 15m
        labels: { severity: critical }
        annotations: { summary: "Root filesystem below 10 % free on {{ $labels.node }}" }

      - alert: MemoryPressure
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 0.92
        for: 15m
        labels: { severity: serious }
        annotations: { summary: "Less than 8 % memory available on {{ $labels.node }}" }

      - alert: PostgresConnectionsHigh
        expr: sum(pg_stat_activity_count) > 170
        for: 10m
        labels: { severity: warning }
        annotations: { summary: "Postgres connections above 170/200" }