Repository Public
1package alerts23import (4 "context"5 "errors"6 "fmt"7 "math"8 "sort"9 "strings"10 "time"1112 "github.com/google/uuid"13 "github.com/jackc/pgx/v5"14 "github.com/jackc/pgx/v5/pgxpool"1516 "clustr.spboucher.ai/mcc/internal/events"17 "clustr.spboucher.ai/mcc/internal/ids"18)1920// Engine evaluates every enabled rule. It is stateless: everything it needs21// to continue after a restart is in the alerts table.22type Engine struct {23 Pool *pgxpool.Pool24 Now func() time.Time25 FreshAfter time.Duration // data older than this is not evaluated (default 30 s)26}2728func (e *Engine) now() time.Time {29 if e.Now != nil {30 return e.Now()31 }32 return time.Now().UTC()33}3435func (e *Engine) fresh() time.Duration {36 if e.FreshAfter > 0 {37 return e.FreshAfter38 }39 return 30 * time.Second40}4142type nodeInfo struct {43 id uuid.UUID44 alias string45 tags []string46 maintenance bool47 hasAgent bool48 enrolledAt *time.Time49 certAfter *time.Time50 lastReport *time.Time // greatest(agent last_seen, node_state.received_at)51 stateAt *time.Time52 values map[string]any53}5455type serviceInfo struct {56 node uuid.UUID57 label string58 state string59}6061type silence struct {62 rule *uuid.UUID63 scope Scope64}6566type snapshot struct {67 now time.Time68 nodes map[uuid.UUID]*nodeInfo69 order []uuid.UUID70 services []serviceInfo71 silences []silence72 denied map[string]float64 // lazily loaded73 newDev map[string]float6474}7576// instance is the current observation of one (rule, node, label).77type instance struct {78 node *nodeInfo79 label string80 value float6481 evaluable bool // false: no fresh data — never opens, never confirms82 firing bool // condition holds83 clear bool // condition is clear (beyond the hysteresis band)84 since *time.Time // absence rules: when the condition really started85 what string // human description of the observation86}8788func fingerprint(rule uuid.UUID, node *nodeInfo, label string) string {89 n := "-"90 if node != nil {91 n = node.id.String()92 }93 return rule.String() + "/" + n + "/" + label94}9596func (e *Engine) load(ctx context.Context, now time.Time) (*snapshot, error) {97 s := &snapshot{now: now, nodes: map[uuid.UUID]*nodeInfo{}}98 rows, err := e.Pool.Query(ctx, `select n.id, n.alias::text, n.tags, n.maintenance, a.id is not null, a.enrolled_at, a.cert_not_after,99 greatest(a.last_seen_at, ns.received_at), ns.received_at, ns.values100 from nodes n left join agents a on a.node_id=n.id and a.revoked_at is null left join node_state ns on ns.node_id=n.id101 where n.retired_at is null order by n.alias`)102 if err != nil {103 return nil, err104 }105 defer rows.Close()106 for rows.Next() {107 n := &nodeInfo{}108 if err := rows.Scan(&n.id, &n.alias, &n.tags, &n.maintenance, &n.hasAgent, &n.enrolledAt, &n.certAfter, &n.lastReport, &n.stateAt, &n.values); err != nil {109 return nil, err110 }111 s.nodes[n.id] = n112 s.order = append(s.order, n.id)113 }114 if err := rows.Err(); err != nil {115 return nil, err116 }117 srows, err := e.Pool.Query(ctx, `select node_id, manager::text || ':' || name, state from services where managed and gone_at is null`)118 if err != nil {119 return nil, err120 }121 if s.services, err = pgx.CollectRows(srows, func(r pgx.CollectableRow) (serviceInfo, error) {122 var v serviceInfo123 return v, r.Scan(&v.node, &v.label, &v.state)124 }); err != nil {125 return nil, err126 }127 sl, err := e.Pool.Query(ctx, `select rule_id, scope from silences where starts_at <= $1 and ends_at > $1`, now)128 if err != nil {129 return nil, err130 }131 s.silences, err = pgx.CollectRows(sl, func(r pgx.CollectableRow) (silence, error) {132 var v silence133 return v, r.Scan(&v.rule, &v.scope)134 })135 return s, err136}137138func (s *snapshot) isFresh(n *nodeInfo, max time.Duration) bool {139 return n.stateAt != nil && s.now.Sub(*n.stateAt) <= max140}141142// suppression says why notifications are withheld for this alert, if they are.143func (s *snapshot) suppression(rule uuid.UUID, n *nodeInfo) string {144 if n != nil && n.maintenance {145 return "maintenance"146 }147 for _, sl := range s.silences {148 if sl.rule != nil && *sl.rule != rule {149 continue150 }151 var id *uuid.UUID152 var tags []string153 if n != nil {154 id, tags = &n.id, n.tags155 }156 if sl.scope.Matches(id, tags) {157 return "silence"158 }159 }160 return ""161}162163func num(v any) (float64, bool) {164 f, ok := v.(float64)165 return f, ok && !math.IsNaN(f) && !math.IsInf(f, 0)166}167168// series returns label -> value for a metric in a node snapshot ("name" or "name{label}" keys).169func series(values map[string]any, metric string) map[string]float64 {170 out := map[string]float64{}171 if v, ok := num(values[metric]); ok {172 out[""] = v173 }174 // telemetry-ingest stores labeled metrics nested: "net.rx": {"en0": 1.0}.175 if sub, ok := values[metric].(map[string]any); ok {176 for label, raw := range sub {177 if v, ok := num(raw); ok {178 out[label] = v179 }180 }181 }182 prefix := metric + "{"183 for k, raw := range values {184 if strings.HasPrefix(k, prefix) && strings.HasSuffix(k, "}") {185 if v, ok := num(raw); ok {186 out[k[len(prefix):len(k)-1]] = v187 }188 }189 }190 return out191}192193// nodeSeries resolves catalog and derived node metrics.194func nodeSeries(values map[string]any, metric string) map[string]float64 {195 ratio := func(used, total string) map[string]float64 {196 u, t := series(values, used), series(values, total)197 out := map[string]float64{}198 for l, uv := range u {199 if tv, ok := t[l]; ok && tv > 0 {200 out[l] = 100 * uv / tv201 }202 }203 return out204 }205 switch metric {206 case MetricDiskPct:207 return ratio("disk.used", "disk.total")208 case MetricSwapPct:209 return ratio("swap.used", "swap.total")210 }211 return series(values, metric)212}213214func labelOK(match, label string) bool { return match == "" || match == "*" || match == label }215216func (e *Engine) instances(ctx context.Context, r Rule, s *snapshot) (map[string]instance, error) {217 out := map[string]instance{}218 put := func(i instance) { out[fingerprint(r.ID, i.node, i.label)] = i }219 inScope := func(n *nodeInfo) bool { return r.Scope.Matches(&n.id, n.tags) }220 switch r.Kind {221 case "absence":222 for _, id := range s.order {223 n := s.nodes[id]224 if !n.hasAgent || !inScope(n) { // a node that never had an agent cannot "stop reporting"225 continue226 }227 last := n.lastReport228 if last == nil {229 last = n.enrolledAt230 }231 if last == nil {232 continue233 }234 age := s.now.Sub(*last)235 silent := age > e.fresh()236 put(instance{node: n, value: math.Round(age.Seconds()), evaluable: true, firing: silent, clear: !silent, since: last,237 what: fmt.Sprintf("no report for %s", age.Round(time.Second))})238 }239 case "service_state":240 states := r.ServiceStates()241 for _, sv := range s.services {242 n := s.nodes[sv.node]243 if n == nil || !inScope(n) || !labelOK(r.LabelMatch, sv.label) {244 continue245 }246 bad := false247 for _, st := range states {248 bad = bad || st == sv.state249 }250 v := 0.0251 if bad {252 v = 1253 }254 put(instance{node: n, label: sv.label, value: v, evaluable: s.isFresh(n, e.fresh()), firing: bad, clear: !bad, what: "service " + sv.label + " is " + sv.state})255 }256 case "threshold":257 switch r.Metric {258 case MetricCertDays:259 for _, id := range s.order {260 n := s.nodes[id]261 if !n.hasAgent || n.certAfter == nil || !inScope(n) {262 continue263 }264 d := n.certAfter.Sub(s.now).Hours() / 24265 put(instance{node: n, value: math.Round(d*10) / 10, evaluable: true, firing: r.Firing(d), clear: r.Clear(d), what: fmt.Sprintf("agent certificate expires in %.1f days", d)})266 }267 case MetricDenied10m, MetricNewDevice:268 if !r.Scope.All { // cluster-wide metrics belong to no node269 return out, nil270 }271 m, err := e.clusterMetric(ctx, r.Metric, s)272 if err != nil {273 return nil, err274 }275 for label, v := range m {276 if labelOK(r.LabelMatch, label) {277 put(instance{label: label, value: v, evaluable: true, firing: r.Firing(v), clear: r.Clear(v), what: fmt.Sprintf("%s %s = %g", r.Metric, label, v)})278 }279 }280 default:281 for _, id := range s.order {282 n := s.nodes[id]283 if !inScope(n) {284 continue285 }286 fresh := s.isFresh(n, e.fresh())287 for label, v := range nodeSeries(n.values, r.Metric) {288 if !labelOK(r.LabelMatch, label) {289 continue290 }291 what := fmt.Sprintf("%s = %.4g", r.Metric, v)292 if label != "" {293 what = fmt.Sprintf("%s{%s} = %.4g", r.Metric, label, v)294 }295 put(instance{node: n, label: label, value: v, evaluable: fresh, firing: fresh && r.Firing(v), clear: r.Clear(v), what: what})296 }297 }298 }299 }300 return out, nil301}302303// clusterMetric computes the audit-derived metrics (simple anomaly detection).304func (e *Engine) clusterMetric(ctx context.Context, metric string, s *snapshot) (map[string]float64, error) {305 switch metric {306 case MetricDenied10m:307 if s.denied == nil {308 s.denied = map[string]float64{}309 rows, err := e.Pool.Query(ctx, `select 'actor:' || actor_label, count(*) from audit_log310 where outcome='denied' and ts > $1 and ts <= $2 and actor_label <> '' group by 1311 union all312 select 'ip:' || host(ip), count(*) from audit_log where outcome='denied' and ts > $1 and ts <= $2 and ip is not null group by 1`,313 s.now.Add(-deniedWindowMin*time.Minute), s.now)314 if err != nil {315 return nil, err316 }317 defer rows.Close()318 for rows.Next() {319 var l string320 var c float64321 if err := rows.Scan(&l, &c); err != nil {322 return nil, err323 }324 s.denied[l] = c325 }326 if err := rows.Err(); err != nil {327 return nil, err328 }329 }330 return s.denied, nil331 case MetricNewDevice:332 if s.newDev == nil {333 s.newDev = map[string]float64{}334 // A sign-in whose device was created by that very sign-in, for an account that already had another device.335 rows, err := e.Pool.Query(ctx, `select u.handle::text, count(*) from audit_log l336 join users u on u.id=l.actor_id and u.role in ('owner','admin')337 join devices d on d.id=l.device_id338 where l.action='auth.login' and l.outcome='success' and l.ts > $1 and l.ts <= $2339 and d.created_at >= l.ts - interval '1 minute'340 and exists (select 1 from devices o where o.user_id=u.id and o.id<>d.id and o.created_at < d.created_at)341 group by 1`, s.now.Add(-newDeviceWindowMn*time.Minute), s.now)342 if err != nil {343 return nil, err344 }345 defer rows.Close()346 for rows.Next() {347 var l string348 var c float64349 if err := rows.Scan(&l, &c); err != nil {350 return nil, err351 }352 s.newDev[l] = c353 }354 if err := rows.Err(); err != nil {355 return nil, err356 }357 }358 return s.newDev, nil359 }360 return nil, nil361}362363// Evaluate runs one evaluation of every rule. Safe to call concurrently.364func (e *Engine) Evaluate(ctx context.Context) error {365 now := e.now()366 s, err := e.load(ctx, now)367 if err != nil {368 return err369 }370 rr, err := e.Pool.Query(ctx, `select `+RuleColumns+` from alert_rules where deleted_at is null order by name`)371 if err != nil {372 return err373 }374 rules, err := pgx.CollectRows(rr, func(r pgx.CollectableRow) (Rule, error) { return ScanRule(r) })375 if err != nil {376 return err377 }378 type open struct {379 fp string380 rule uuid.UUID381 }382 or, err := e.Pool.Query(ctx, `select fingerprint, rule_id from alerts where state <> 'resolved'`)383 if err != nil {384 return err385 }386 opens, err := pgx.CollectRows(or, func(r pgx.CollectableRow) (open, error) {387 var o open388 return o, r.Scan(&o.fp, &o.rule)389 })390 if err != nil {391 return err392 }393 for _, r := range rules {394 insts := map[string]instance{}395 if r.Enabled {396 if insts, err = e.instances(ctx, r, s); err != nil {397 return err398 }399 }400 fps := make([]string, 0, len(insts))401 for fp := range insts {402 fps = append(fps, fp)403 }404 sort.Strings(fps)405 for _, fp := range fps {406 if err := e.apply(ctx, r, fp, insts[fp], s, ""); err != nil {407 return err408 }409 }410 // Open alerts the rule no longer produces.411 for _, o := range opens {412 if o.rule != r.ID {413 continue414 }415 if _, ok := insts[o.fp]; ok {416 continue417 }418 gone := instance{}419 why := "no recent data"420 switch {421 case !r.Enabled:422 why = "rule disabled"423 case r.Metric == MetricDenied10m || r.Metric == MetricNewDevice:424 gone = instance{evaluable: true, clear: true} // nothing counted any more: the value is zero425 why = ""426 }427 if err := e.apply(ctx, r, o.fp, gone, s, why); err != nil {428 return err429 }430 }431 }432 return nil433}434435type alertRow struct {436 id uuid.UUID437 state string438 nodeID *uuid.UUID439 started time.Time440 clearSince *time.Time441 suppressed *string442 flaps int443 summary string444 incident *uuid.UUID445 lastValue *float64446}447448// apply advances one alert. goneWhy is set when the rule produced no observation for an open alert.449func (e *Engine) apply(ctx context.Context, r Rule, fp string, in instance, s *snapshot, goneWhy string) error {450 now := s.now451 return pgx.BeginFunc(ctx, e.Pool, func(tx pgx.Tx) error {452 var a alertRow453 err := tx.QueryRow(ctx, `select id, state::text, node_id, started_at, clear_since, suppressed_by, flap_count, summary, incident_id, last_value454 from alerts where fingerprint=$1 and state <> 'resolved' for update`, fp).455 Scan(&a.id, &a.state, &a.nodeID, &a.started, &a.clearSince, &a.suppressed, &a.flaps, &a.summary, &a.incident, &a.lastValue)456 exists := err == nil457 if err != nil && !errors.Is(err, pgx.ErrNoRows) {458 return err459 }460 node := in.node461 if node == nil && a.nodeID != nil {462 node = s.nodes[*a.nodeID]463 }464 supp := s.suppression(r.ID, node)465 if goneWhy == "rule disabled" {466 supp = deref(a.suppressed)467 }468 timeline := func(kind, body string) error {469 _, err := tx.Exec(ctx, `insert into timeline_events(id, alert_id, ts, kind, body) values ($1,$2,$3,$4,$5)`, ids.New(), a.id, now, kind, body)470 return err471 }472 publish := func(state string) error {473 p := map[string]any{"alert_id": a.id, "rule_id": r.ID, "state": state, "severity": r.Severity, "summary": a.summary, "suppressed": supp != ""}474 if node != nil {475 p["node_id"] = node.id476 }477 if a.incident != nil {478 p["incident_id"] = *a.incident479 }480 return events.Publish(ctx, tx, "alert.changed", p)481 }482 fire := func() error {483 if _, err := tx.Exec(ctx, `update alerts set state='firing', fired_at=$2 where id=$1`, a.id, now); err != nil {484 return err485 }486 if err := timeline("fired", in.what); err != nil {487 return err488 }489 switch {490 case supp != "":491 if err := timeline("suppressed", "notification withheld: "+supp); err != nil {492 return err493 }494 case r.Notify:495 if _, err := tx.Exec(ctx, `insert into alert_notifications(alert_id, transition, created_at, next_try_at) values ($1,'firing',$2,$2)`, a.id, now); err != nil {496 return err497 }498 }499 return publish("firing")500 }501502 if !exists {503 if !in.evaluable || !in.firing {504 return nil505 }506 a = alertRow{id: ids.New(), started: now}507 if in.since != nil && in.since.Before(now) {508 a.started = *in.since509 }510 a.summary = r.Name + ": " + in.what511 if node != nil {512 a.summary = r.Name + " on " + node.alias + ": " + in.what513 }514 // A fingerprint that re-opens shortly after resolving is flapping.515 var prevFlaps *int516 if err := tx.QueryRow(ctx, `select flap_count from alerts where fingerprint=$1 and state='resolved' and fired_at is not null and resolved_at > $2517 order by resolved_at desc limit 1`, fp, now.Add(-15*time.Minute)).Scan(&prevFlaps); err != nil && !errors.Is(err, pgx.ErrNoRows) {518 return err519 }520 if prevFlaps != nil {521 a.flaps = *prevFlaps + 1522 }523 var nodeID *uuid.UUID524 if node != nil {525 nodeID = &node.id526 }527 tag, err := tx.Exec(ctx, `insert into alerts(id, rule_id, node_id, label, fingerprint, state, severity, summary, last_value, started_at, last_eval_at, flap_count, suppressed_by)528 values ($1,$2,$3,$4,$5,'pending',$6::severity,$7,$8,$9,$10,$11,nullif($12,''))529 on conflict (fingerprint) where state <> 'resolved' do nothing`,530 a.id, r.ID, nodeID, in.label, fp, r.Severity, a.summary, in.value, a.started, now, a.flaps, supp)531 if err != nil || tag.RowsAffected() == 0 {532 return err // 0 rows: another evaluator opened it first533 }534 kind := "opened"535 if prevFlaps != nil {536 kind = "reopened"537 }538 if err := timeline(kind, in.what); err != nil {539 return err540 }541 if now.Sub(a.started) >= time.Duration(r.ForSeconds)*time.Second {542 return fire()543 }544 return publish("pending")545 }546547 resolve := func(body string, notify bool) error {548 if _, err := tx.Exec(ctx, `update alerts set state='resolved', resolved_at=$2, last_eval_at=$2, suppressed_by=nullif($3,'') where id=$1`, a.id, now, supp); err != nil {549 return err550 }551 if err := timeline("resolved", body); err != nil {552 return err553 }554 if notify && r.Notify && supp == "" {555 if _, err := tx.Exec(ctx, `insert into alert_notifications(alert_id, transition, created_at, next_try_at) values ($1,'resolved',$2,$2)`, a.id, now); err != nil {556 return err557 }558 }559 return publish("resolved")560 }561 noData := "no recent data: the condition can no longer be verified"562 if goneWhy != "" && goneWhy != "no recent data" {563 noData = goneWhy564 }565 if goneWhy == "rule disabled" {566 return resolve("rule disabled", false)567 }568 switch a.state {569 case "pending":570 switch {571 case !in.evaluable:572 return resolve(noData, false)573 case !in.firing:574 return resolve("condition cleared before firing", false)575 case now.Sub(a.started) >= time.Duration(r.ForSeconds)*time.Second:576 if _, err := tx.Exec(ctx, `update alerts set last_value=$2, last_eval_at=$3, suppressed_by=nullif($4,'') where id=$1`, a.id, in.value, now, supp); err != nil {577 return err578 }579 return fire()580 }581 case "firing", "acknowledged":582 clearing := !in.evaluable || in.clear583 switch {584 case clearing:585 since := now586 if a.clearSince != nil {587 since = *a.clearSince588 }589 if now.Sub(since) >= time.Duration(r.ClearSeconds)*time.Second {590 body := "condition cleared"591 if !in.evaluable {592 body = noData593 }594 return resolve(body, true)595 }596 if a.clearSince == nil {597 if _, err := tx.Exec(ctx, `update alerts set clear_since=$2 where id=$1`, a.id, now); err != nil {598 return err599 }600 }601 case a.clearSince != nil:602 // It started to clear and came back: one flap.603 if _, err := tx.Exec(ctx, `update alerts set clear_since=null, flap_count=flap_count+1 where id=$1`, a.id); err != nil {604 return err605 }606 if err := timeline("state_changed", "condition returned before the clear delay (flap)"); err != nil {607 return err608 }609 }610 }611 // Still open: refresh the observation and the suppression.612 var val *float64613 if in.evaluable {614 val = &in.value615 }616 if _, err := tx.Exec(ctx, `update alerts set last_value=coalesce($2,last_value), last_eval_at=$3, suppressed_by=nullif($4,'') where id=$1`, a.id, val, now, supp); err != nil {617 return err618 }619 if supp != deref(a.suppressed) {620 body := "notifications restored"621 if supp != "" {622 body = "notifications withheld: " + supp623 }624 if err := timeline("suppressed", body); err != nil {625 return err626 }627 return publish(a.state)628 }629 return nil630 })631}632633func deref[T any](p *T) (v T) {634 if p != nil {635 v = *p636 }637 return v638}639