Repository Public
1package alerts_test23import (4 "context"5 "fmt"6 "net/netip"7 "strings"8 "sync"9 "testing"10 "time"1112 "github.com/google/uuid"13 "github.com/jackc/pgx/v5/pgxpool"1415 "clustr.spboucher.ai/mcc/internal/alerts"16 "clustr.spboucher.ai/mcc/internal/audit"17 "clustr.spboucher.ai/mcc/internal/ids"18 "clustr.spboucher.ai/mcc/internal/protocol"19 "clustr.spboucher.ai/mcc/internal/testdb"20)2122var bg = context.Background()2324type env struct {25 t *testing.T26 pool *pgxpool.Pool27 clock *testdb.Clock28 eng *alerts.Engine29}3031func newEnv(t *testing.T) *env {32 t.Helper()33 pool, _ := testdb.New(t)34 e := &env{t: t, pool: pool, clock: testdb.NewClock()}35 e.eng = &alerts.Engine{Pool: pool, Now: e.clock.Now, FreshAfter: 30 * time.Second}36 return e37}3839func fp(v float64) *float64 { return &v }4041func (e *env) rule(r alerts.Rule) uuid.UUID {42 e.t.Helper()43 r.ID = ids.New()44 if !r.Scope.All && r.Scope.NodeIDs == nil && r.Scope.Tags == nil {45 r.Scope = alerts.Scope{All: true}46 }47 if r.Severity == "" {48 r.Severity = "warning"49 }50 if r.Name == "" {51 r.Name = "rule " + r.ID.String()[24:]52 }53 if err := r.Validate(); err != nil {54 e.t.Fatal(err)55 }56 if _, err := e.pool.Exec(bg, `insert into alert_rules(id, name, kind, metric, label_match, op, threshold, clear_threshold, for_seconds, clear_seconds, severity, scope, notify)57 values ($1,$2,$3,nullif($4,''),nullif($5,''),nullif($6,''),$7,$8,$9,$10,$11::severity,$12,true)`,58 r.ID, r.Name, r.Kind, r.Metric, r.LabelMatch, r.Op, r.Threshold, r.ClearThreshold, r.ForSeconds, r.ClearSeconds, r.Severity, r.Scope); err != nil {59 e.t.Fatal(err)60 }61 return r.ID62}6364// node adds a node with an agent that reports now.65func (e *env) node(alias string, tags ...string) uuid.UUID {66 id := testdb.Node(e.t, e.pool, alias, tags...)67 testdb.Agent(e.t, e.pool, id, e.clock.Now())68 return id69}7071// report advances the clock by d and stores a fresh snapshot, then evaluates.72func (e *env) report(d time.Duration, node uuid.UUID, values map[string]any) {73 e.t.Helper()74 e.clock.Advance(d)75 testdb.State(e.t, e.pool, node, e.clock.Now(), values)76 e.eval()77}7879func (e *env) eval() {80 e.t.Helper()81 if err := e.eng.Evaluate(bg); err != nil {82 e.t.Fatal(err)83 }84}8586type arow struct {87 ID uuid.UUID88 State string89 Flaps int90 Suppressed string91 Summary string92 Label string93 Value *float6494 Started time.Time95 ClearSince *time.Time96}9798// alerts returns the alerts of a rule, oldest first.99func (e *env) alerts(rule uuid.UUID) []arow {100 e.t.Helper()101 rows, err := e.pool.Query(bg, `select id, state::text, flap_count, coalesce(suppressed_by,''), summary, label, last_value, started_at, clear_since102 from alerts where rule_id=$1 order by started_at, id`, rule)103 if err != nil {104 e.t.Fatal(err)105 }106 defer rows.Close()107 var out []arow108 for rows.Next() {109 var a arow110 if err := rows.Scan(&a.ID, &a.State, &a.Flaps, &a.Suppressed, &a.Summary, &a.Label, &a.Value, &a.Started, &a.ClearSince); err != nil {111 e.t.Fatal(err)112 }113 out = append(out, a)114 }115 return out116}117118func (e *env) one(rule uuid.UUID, state string) arow {119 e.t.Helper()120 as := e.alerts(rule)121 if len(as) == 0 {122 e.t.Fatalf("no alert, want one %s", state)123 }124 a := as[len(as)-1]125 if a.State != state {126 e.t.Fatalf("alert state = %s, want %s (%+v)", a.State, state, a)127 }128 return a129}130131func (e *env) count(sql string, args ...any) int {132 e.t.Helper()133 var n int134 if err := e.pool.QueryRow(bg, sql, args...).Scan(&n); err != nil {135 e.t.Fatal(err)136 }137 return n138}139140func (e *env) timeline(alert uuid.UUID) string {141 rows, _ := e.pool.Query(bg, `select kind from timeline_events where alert_id=$1 order by ts, id`, alert)142 defer rows.Close()143 var ks []string144 for rows.Next() {145 var k string146 rows.Scan(&k)147 ks = append(ks, k)148 }149 return strings.Join(ks, ",")150}151152func cpu(v float64) map[string]any { return map[string]any{"cpu.usage": v} }153154func TestStateMachinePendingFiringHysteresisFlapResolve(t *testing.T) {155 e := newEnv(t)156 n := e.node("T-A")157 r := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(90), ClearThreshold: fp(80), ForSeconds: 60, ClearSeconds: 30, Severity: "critical"})158 notif := func(tr string) int {159 return e.count(`select count(*) from alert_notifications q join alerts a on a.id=q.alert_id where a.rule_id=$1 and q.transition=$2`, r, tr)160 }161162 e.report(0, n, cpu(50))163 if len(e.alerts(r)) != 0 {164 t.Fatal("alert below threshold")165 }166 // A breach shorter than for_seconds never fires and never notifies.167 e.report(10*time.Second, n, cpu(95))168 e.one(r, "pending")169 e.report(10*time.Second, n, cpu(50))170 e.one(r, "resolved")171 if notif("firing")+notif("resolved") != 0 {172 t.Fatal("a pending alert that never fired produced a notification")173 }174175 e.report(10*time.Second, n, cpu(95))176 a := e.one(r, "pending")177 start := e.clock.Now()178 e.report(30*time.Second, n, cpu(96))179 e.one(r, "pending")180 e.report(29*time.Second, n, cpu(97))181 e.one(r, "pending")182 e.report(1*time.Second, n, cpu(97)) // 60 s183 a = e.one(r, "firing")184 if !a.Started.Equal(start) || *a.Value != 97 || !strings.Contains(a.Summary, "T-A") || notif("firing") != 1 {185 t.Fatalf("%+v notif=%d", a, notif("firing"))186 }187 // Inside the hysteresis band (80 < v <= 90): stays firing, does not start clearing.188 e.report(10*time.Second, n, cpu(85))189 if a = e.one(r, "firing"); a.ClearSince != nil {190 t.Fatal("clearing started inside the hysteresis band")191 }192 // Below clear_threshold: clearing starts but needs clear_seconds.193 e.report(10*time.Second, n, cpu(70))194 if a = e.one(r, "firing"); a.ClearSince == nil {195 t.Fatal("clearing not started")196 }197 e.report(20*time.Second, n, cpu(70))198 e.one(r, "firing")199 // Back above before the delay: one flap, the delay restarts.200 e.report(5*time.Second, n, cpu(95))201 if a = e.one(r, "firing"); a.Flaps != 1 || a.ClearSince != nil {202 t.Fatalf("flap not counted: %+v", a)203 }204 e.report(10*time.Second, n, cpu(70))205 e.report(29*time.Second, n, cpu(70))206 e.one(r, "firing")207 e.report(1*time.Second, n, cpu(70))208 a = e.one(r, "resolved")209 if notif("firing") != 1 || notif("resolved") != 1 {210 t.Fatalf("notifications: firing=%d resolved=%d", notif("firing"), notif("resolved"))211 }212 if tl := e.timeline(a.ID); tl != "opened,fired,state_changed,resolved" {213 t.Fatalf("timeline: %s", tl)214 }215 // Re-opening soon after is a flap of the same problem.216 e.report(10*time.Second, n, cpu(99))217 e.report(60*time.Second, n, cpu(99))218 a = e.one(r, "firing")219 if a.Flaps != 2 || !strings.HasPrefix(e.timeline(a.ID), "reopened") {220 t.Fatalf("reopen: %+v %s", a, e.timeline(a.ID))221 }222 if n := e.count(`select count(*) from events_outbox where topic='alert.changed' and payload->>'rule_id'=$1`, r.String()); n < 6 {223 t.Fatalf("alert.changed events: %d", n)224 }225 // Disabling the rule closes what it had open, without notification.226 e.pool.Exec(bg, `update alert_rules set enabled=false where id=$1`, r)227 e.eval()228 e.one(r, "resolved")229 if notif("resolved") != 1 {230 t.Fatal("disabling a rule must not notify")231 }232}233234func TestDedupUnderConcurrentEvaluators(t *testing.T) {235 e := newEnv(t)236 var nodes []uuid.UUID237 for i := 0; i < 5; i++ {238 n := e.node(fmt.Sprintf("T-%d", i))239 testdb.State(t, e.pool, n, e.clock.Now(), cpu(99))240 nodes = append(nodes, n)241 }242 r := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(90), ForSeconds: 0})243 var wg sync.WaitGroup244 errs := make(chan error, 64)245 for i := 0; i < 8; i++ {246 wg.Add(1)247 go func() {248 defer wg.Done()249 for j := 0; j < 4; j++ {250 if err := e.eng.Evaluate(bg); err != nil {251 errs <- err252 }253 }254 }()255 }256 wg.Wait()257 close(errs)258 for err := range errs {259 t.Fatal(err)260 }261 if n := e.count(`select count(*) from alerts where rule_id=$1`, r); n != 5 {262 t.Fatalf("%d alerts for 5 nodes", n)263 }264 if n := e.count(`select count(*) from alerts where rule_id=$1 and state='firing'`, r); n != 5 {265 t.Fatalf("%d firing", n)266 }267 if n := e.count(`select count(*) from alert_notifications`); n != 5 {268 t.Fatalf("%d notifications for 5 firings", n)269 }270 if n := e.count(`select count(*) from timeline_events where kind='fired'`); n != 5 {271 t.Fatalf("%d fired events", n)272 }273}274275func TestSilenceAndMaintenanceSuppression(t *testing.T) {276 e := newEnv(t)277 n1, n2, n3 := e.node("T-1"), e.node("T-2", "lab"), e.node("T-3")278 r := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(90), ForSeconds: 0, ClearSeconds: 0})279 other := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(50), ForSeconds: 0, ClearSeconds: 0})280 now := e.clock.Now()281 sil := func(scope string, rule *uuid.UUID) uuid.UUID {282 id := ids.New()283 if _, err := e.pool.Exec(bg, `insert into silences(id, reason, scope, rule_id, starts_at, ends_at) values ($1,'test',$2::jsonb,$3,$4,$5)`,284 id, scope, rule, now.Add(-time.Minute), now.Add(time.Hour)); err != nil {285 t.Fatal(err)286 }287 return id288 }289 s1 := sil(`{"node_ids":["`+n1.String()+`"]}`, &r) // n1, this rule only290 sil(`{"tags":["lab"]}`, nil) // n2, every rule291 e.pool.Exec(bg, `update nodes set maintenance=true where id=$1`, n3)292 for _, n := range []uuid.UUID{n1, n2, n3} {293 testdb.State(t, e.pool, n, now, cpu(99))294 }295 e.eval()296 got := map[string]string{}297 rows, _ := e.pool.Query(bg, `select n.alias::text || '/' || case when a.rule_id=$1 then 'r' else 'o' end, coalesce(a.suppressed_by,''), a.state::text298 from alerts a join nodes n on n.id=a.node_id`, r)299 for rows.Next() {300 var k, s, st string301 rows.Scan(&k, &s, &st)302 if st != "firing" {303 t.Fatalf("%s is %s: a suppressed alert must still be tracked", k, st)304 }305 got[k] = s306 }307 want := map[string]string{"T-1/r": "silence", "T-1/o": "", "T-2/r": "silence", "T-2/o": "silence", "T-3/r": "maintenance", "T-3/o": "maintenance"}308 for k, v := range want {309 if got[k] != v {310 t.Errorf("%s: suppressed_by=%q, want %q", k, got[k], v)311 }312 }313 _ = other314 // Only the one unsuppressed alert notifies.315 if n := e.count(`select count(*) from alert_notifications`); n != 1 {316 t.Fatalf("%d notifications, want 1", n)317 }318 if n := e.count(`select count(*) from timeline_events where kind='suppressed'`); n != 5 {319 t.Fatalf("%d suppressed timeline events", n)320 }321 // Lifting the silence restores notifications for later transitions and says so.322 e.pool.Exec(bg, `delete from silences where id=$1`, s1)323 e.clock.Advance(10 * time.Second)324 for _, n := range []uuid.UUID{n1, n2, n3} {325 testdb.State(t, e.pool, n, e.clock.Now(), cpu(99))326 }327 e.eval()328 if e.count(`select count(*) from alerts where rule_id=$1 and node_id=$2 and suppressed_by is null and state='firing'`, r, n1) != 1 {329 t.Fatal("silence removal not reflected")330 }331 // A resolution while in maintenance does not notify either.332 testdb.State(t, e.pool, n3, e.clock.Now(), cpu(10))333 testdb.State(t, e.pool, n1, e.clock.Now(), cpu(10))334 e.eval()335 if n := e.count(`select count(*) from alert_notifications where transition='resolved'`); n != 2 { // n1: both rules336 t.Fatalf("%d resolved notifications, want 2 (none for the node in maintenance)", n)337 }338}339340func TestStaleDataIsNotEvaluable(t *testing.T) {341 e := newEnv(t)342 n := e.node("T-A")343 r := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(90), ForSeconds: 0, ClearSeconds: 60})344 // A stale breach never opens an alert.345 testdb.State(t, e.pool, n, e.clock.Now().Add(-31*time.Second), cpu(99))346 e.eval()347 if len(e.alerts(r)) != 0 {348 t.Fatal("a stale value opened an alert")349 }350 e.report(0, n, cpu(99))351 e.one(r, "firing")352 // The node goes silent with its last value still at 99: the alert must not be kept on that value.353 e.clock.Advance(31 * time.Second)354 e.eval()355 a := e.one(r, "firing")356 if a.ClearSince == nil {357 t.Fatal("no-data clearing not started")358 }359 e.clock.Advance(59 * time.Second)360 e.eval()361 e.one(r, "firing")362 e.clock.Advance(2 * time.Second)363 e.eval()364 a = e.one(r, "resolved")365 var body string366 e.pool.QueryRow(bg, `select body from timeline_events where alert_id=$1 and kind='resolved'`, a.ID).Scan(&body)367 if !strings.Contains(body, "no recent data") {368 t.Fatalf("resolution reason: %q", body)369 }370 // Pending + stale: dropped at once.371 r2 := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(95), ForSeconds: 300})372 e.report(0, n, cpu(99))373 e.one(r2, "pending")374 e.clock.Advance(40 * time.Second)375 e.eval()376 e.one(r2, "resolved")377 // However long the stale 99 stays in node_state, nothing fires.378 e.clock.Advance(time.Hour)379 e.eval()380 if e.count(`select count(*) from alerts where state <> 'resolved'`) != 0 {381 t.Fatal("stale data kept or opened an alert")382 }383}384385func TestAbsenceRule(t *testing.T) {386 e := newEnv(t)387 n := e.node("T-A")388 manual := testdb.Node(t, e.pool, "T-MANUAL") // registered by hand, never had an agent389 _ = manual390 r := e.rule(alerts.Rule{Kind: "absence", ForSeconds: 120, ClearSeconds: 30, Severity: "critical"})391 last := e.clock.Now()392 testdb.State(t, e.pool, n, last, cpu(1))393 e.clock.Advance(20 * time.Second)394 e.eval()395 if len(e.alerts(r)) != 0 {396 t.Fatal("alert while reporting")397 }398 e.clock.Advance(20 * time.Second) // 40 s of silence399 e.eval()400 a := e.one(r, "pending")401 if !a.Started.Equal(last) {402 t.Fatalf("started_at %s, want the last report %s", a.Started, last)403 }404 e.clock.Advance(79 * time.Second) // 119 s405 e.eval()406 e.one(r, "pending")407 e.clock.Advance(2 * time.Second) // 121 s408 e.eval()409 a = e.one(r, "firing")410 if len(e.alerts(r)) != 1 || !strings.Contains(a.Summary, "T-A") {411 t.Fatalf("%+v", e.alerts(r))412 }413 // Reporting again clears it after clear_seconds.414 e.report(10*time.Second, n, cpu(1))415 e.one(r, "firing")416 e.report(15*time.Second, n, cpu(1))417 e.one(r, "firing")418 e.report(15*time.Second, n, cpu(1))419 e.one(r, "resolved")420 // A node that never reported since enrollment is measured from its enrollment.421 n2 := testdb.Node(t, e.pool, "T-NEW")422 testdb.Agent(t, e.pool, n2, e.clock.Now())423 e.pool.Exec(bg, `update agents set last_seen_at=null where node_id=$1`, n2)424 e.clock.Advance(121 * time.Second)425 testdb.State(t, e.pool, n, e.clock.Now(), cpu(1))426 e.eval()427 if e.count(`select count(*) from alerts where rule_id=$1 and node_id=$2 and state='firing'`, r, n2) != 1 {428 t.Fatal("never-reporting node not detected")429 }430 if e.count(`select count(*) from alerts where node_id=$1`, manual) != 0 {431 t.Fatal("a node without agent cannot be absent")432 }433}434435func TestServiceStateRule(t *testing.T) {436 e := newEnv(t)437 n := e.node("T-A")438 svc := testdb.Service(t, e.pool, n, "web", true, "errored")439 testdb.Service(t, e.pool, n, "scratch", false, "errored") // not managed: ignored440 stopped := testdb.Service(t, e.pool, n, "worker", true, "stopped")441 r := e.rule(alerts.Rule{Kind: "service_state", Metric: "errored", ForSeconds: 120, ClearSeconds: 0, Severity: "critical"})442 any := e.rule(alerts.Rule{Kind: "service_state", ForSeconds: 0, ClearSeconds: 0, LabelMatch: "pm2:worker"})443 e.report(0, n, cpu(1))444 a := e.one(r, "pending")445 if a.Label != "pm2:web" || len(e.alerts(r)) != 1 {446 t.Fatalf("%+v", e.alerts(r))447 }448 if b := e.one(any, "firing"); b.Label != "pm2:worker" {449 t.Fatalf("%+v", b)450 }451 e.report(121*time.Second, n, cpu(1))452 e.one(r, "firing")453 e.pool.Exec(bg, `update services set state='online' where id in ($1,$2)`, svc, stopped)454 e.report(10*time.Second, n, cpu(1))455 e.one(r, "resolved")456 e.one(any, "resolved")457 // Service state of a node that stopped reporting is not evaluated.458 e.pool.Exec(bg, `update services set state='errored' where id=$1`, svc)459 e.clock.Advance(5 * time.Minute)460 e.eval()461 if e.count(`select count(*) from alerts where rule_id=$1 and state <> 'resolved'`, r) != 0 {462 t.Fatal("service alert from a silent node")463 }464}465466func TestLabeledAndDerivedMetrics(t *testing.T) {467 e := newEnv(t)468 n := e.node("T-A")469 disk := e.rule(alerts.Rule{Kind: "threshold", Metric: alerts.MetricDiskPct, LabelMatch: "*", Op: ">", Threshold: fp(90), ClearThreshold: fp(88), ForSeconds: 0, ClearSeconds: 0})470 loss := e.rule(alerts.Rule{Kind: "threshold", Metric: "net.loss", LabelMatch: "gateway", Op: ">=", Threshold: fp(34), ForSeconds: 0, ClearSeconds: 0})471 swap := e.rule(alerts.Rule{Kind: "threshold", Metric: alerts.MetricSwapPct, Op: ">", Threshold: fp(80), ForSeconds: 0, ClearSeconds: 0})472 vals := map[string]any{"disk.used{/}": 950.0, "disk.total{/}": 1000.0, "disk.used{/Volumes/Data}": 100.0, "disk.total{/Volumes/Data}": 1000.0,473 "net.loss{gateway}": 0.0, "net.loss{hub}": 100.0, "swap.used": 0.0, "swap.total": 0.0}474 e.report(0, n, vals)475 a := e.one(disk, "firing")476 if a.Label != "/" || *a.Value != 95 || len(e.alerts(disk)) != 1 {477 t.Fatalf("%+v", e.alerts(disk))478 }479 if len(e.alerts(loss)) != 0 || len(e.alerts(swap)) != 0 {480 t.Fatal("loss to another target, or swap with total=0, raised an alert")481 }482 vals["net.loss{gateway}"], vals["swap.used"], vals["swap.total"] = 67.0, 900.0, 1000.0483 vals["disk.used{/}"] = 890.0 // inside the hysteresis band: still firing484 e.report(10*time.Second, n, vals)485 e.one(disk, "firing")486 e.one(loss, "firing")487 if s := e.one(swap, "firing"); *s.Value != 90 {488 t.Fatalf("%+v", s)489 }490 // The volume disappears (unmounted): its alert cannot be verified any more and closes.491 delete(vals, "disk.used{/}")492 delete(vals, "disk.total{/}")493 e.report(10*time.Second, n, vals)494 e.one(disk, "resolved")495}496497func TestServerComputedMetrics(t *testing.T) {498 e := newEnv(t)499 n := e.node("T-A")500 cert := e.rule(alerts.Rule{Kind: "threshold", Metric: alerts.MetricCertDays, Op: "<", Threshold: fp(7), ForSeconds: 0, ClearSeconds: 0})501 denied := e.rule(alerts.Rule{Kind: "threshold", Metric: alerts.MetricDenied10m, LabelMatch: "*", Op: ">=", Threshold: fp(10), ForSeconds: 0, ClearSeconds: 0})502 newdev := e.rule(alerts.Rule{Kind: "threshold", Metric: alerts.MetricNewDevice, LabelMatch: "*", Op: ">=", Threshold: fp(1), ForSeconds: 0, ClearSeconds: 0, Severity: "info"})503 e.eval()504 if e.count(`select count(*) from alerts`) != 0 {505 t.Fatal("alerts without cause")506 }507 // Certificate about to expire (renewal not happening).508 e.pool.Exec(bg, `update agents set cert_not_after=$2 where node_id=$1`, n, e.clock.Now().Add(6*24*time.Hour))509 e.eval()510 e.one(cert, "firing")511 e.pool.Exec(bg, `update agents set cert_not_after=$2 where node_id=$1`, n, e.clock.Now().Add(30*24*time.Hour))512 e.eval()513 e.one(cert, "resolved")514515 // Burst of denied outcomes from one account / one address. (audit timestamps are real time: use a real clock here)516 e.eng.Now = func() time.Time { return time.Now().UTC() }517 mallory := testdb.User(t, e.pool, "mallory", "admin")518 ip := netip.MustParseAddr("203.0.113.9")519 for i := 0; i < 10; i++ {520 if _, err := audit.Append(bg, e.pool, audit.Entry{ActorKind: "user", ActorID: &mallory, ActorLabel: "mallory", IP: &ip, Action: "authz.denied", Outcome: audit.Denied}); err != nil {521 t.Fatal(err)522 }523 }524 audit.Append(bg, e.pool, audit.Entry{ActorKind: "user", ActorLabel: "bob", Action: "authz.denied", Outcome: audit.Denied})525 e.eval()526 as := e.alerts(denied)527 labels := map[string]bool{}528 for _, a := range as {529 labels[a.Label] = a.State == "firing"530 }531 if len(as) != 2 || !labels["actor:mallory"] || !labels["ip:203.0.113.9"] {532 t.Fatalf("denied burst: %+v", as)533 }534 // Sign-in of an admin from a device created by that sign-in, while an older device exists.535 old, fresh := ids.New(), ids.New()536 e.pool.Exec(bg, `insert into devices(id, user_id, kind, created_at) values ($1,$3,'web',now()-interval '30 days'), ($2,$3,'ios',now())`, old, fresh, mallory)537 audit.Append(bg, e.pool, audit.Entry{ActorKind: "user", ActorID: &mallory, ActorLabel: "mallory", DeviceID: &old, Action: "auth.login", Outcome: audit.Success})538 e.eval()539 if len(e.alerts(newdev)) != 0 {540 t.Fatal("sign-in from a known device flagged")541 }542 audit.Append(bg, e.pool, audit.Entry{ActorKind: "user", ActorID: &mallory, ActorLabel: "mallory", DeviceID: &fresh, Action: "auth.login", Outcome: audit.Success})543 e.eval()544 if a := e.one(newdev, "firing"); a.Label != "mallory" {545 t.Fatalf("%+v", a)546 }547 // Viewers and operators are not concerned.548 oscar := testdb.User(t, e.pool, "oscar", "operator")549 o1, o2 := ids.New(), ids.New()550 e.pool.Exec(bg, `insert into devices(id, user_id, kind, created_at) values ($1,$3,'web',now()-interval '30 days'), ($2,$3,'ios',now())`, o1, o2, oscar)551 audit.Append(bg, e.pool, audit.Entry{ActorKind: "user", ActorID: &oscar, ActorLabel: "oscar", DeviceID: &o2, Action: "auth.login", Outcome: audit.Success})552 e.eval()553 if len(e.alerts(newdev)) != 1 {554 t.Fatal("operator sign-in flagged")555 }556 // Once the window has passed, the count is zero and the alerts close.557 e.eng.Now = func() time.Time { return time.Now().UTC().Add(2 * time.Hour) }558 testdb.State(t, e.pool, n, time.Now().UTC().Add(2*time.Hour), cpu(1))559 e.eval()560 if e.count(`select count(*) from alerts where rule_id in ($1,$2) and state <> 'resolved'`, denied, newdev) != 0 {561 t.Fatal("audit-derived alerts did not close")562 }563}564565func TestBuiltinRulesSeededOnceAndEditsPreserved(t *testing.T) {566 e := newEnv(t)567 for i := 0; i < 2; i++ {568 if err := alerts.SeedBuiltins(bg, e.pool); err != nil {569 t.Fatal(err)570 }571 }572 if n := e.count(`select count(*) from alert_rules where builtin and enabled`); n != len(alerts.Builtins) || n != 12 {573 t.Fatalf("%d builtin rules", n)574 }575 for _, name := range []string{"Node offline", "Disk volume above 90 %", "Disk volume above 97 %", "Memory pressure (warning)", "Memory pressure (critical)",576 "Swap above 80 %", "CPU saturated", "Managed service errored", "Agent certificate expiring", "Packet loss to gateway"} {577 if e.count(`select count(*) from alert_rules where name=$1`, name) != 1 {578 t.Errorf("missing builtin %q", name)579 }580 }581 var sev string582 var thr, forS float64583 e.pool.QueryRow(bg, `select severity::text, threshold, for_seconds from alert_rules where name='Packet loss to gateway'`).Scan(&sev, &thr, &forS)584 if sev != "warning" || thr != 34 || forS != 300 {585 t.Fatalf("packet loss rule: %s %v %v", sev, thr, forS)586 }587 // The owner disables one rule and retunes another; a restart must not undo it.588 e.pool.Exec(bg, `update alert_rules set enabled=false where name='CPU saturated'`)589 e.pool.Exec(bg, `update alert_rules set threshold=85, for_seconds=60, notify=false, updated_at=now() where name='Disk volume above 90 %'`)590 if err := alerts.SeedBuiltins(bg, e.pool); err != nil {591 t.Fatal(err)592 }593 if e.count(`select count(*) from alert_rules where name='CPU saturated' and not enabled`) != 1 ||594 e.count(`select count(*) from alert_rules where name='Disk volume above 90 %' and threshold=85 and for_seconds=60 and not notify`) != 1 ||595 e.count(`select count(*) from alert_rules`) != 12 {596 t.Fatal("a restart overwrote or re-enabled an edited builtin rule")597 }598 // The seeded set evaluates cleanly on real-looking data, and the disabled rule stays silent.599 n := e.node("T-A")600 e.report(0, n, map[string]any{"cpu.usage": 99.0, "mem.pressure.level": 4.0, "disk.used{/}": 99.0, "disk.total{/}": 100.0, "swap.used": 9.0, "swap.total": 10.0, "net.loss{gateway}": 100.0})601 e.report(16*time.Minute, n, map[string]any{"cpu.usage": 99.0, "mem.pressure.level": 4.0, "disk.used{/}": 99.0, "disk.total{/}": 100.0, "swap.used": 9.0, "swap.total": 10.0, "net.loss{gateway}": 100.0})602 if e.count(`select count(*) from alerts a join alert_rules r on r.id=a.rule_id where r.name='CPU saturated'`) != 0 {603 t.Fatal("a disabled builtin rule produced an alert")604 }605 if n := e.count(`select count(*) from alerts where state='firing'`); n != 6 { // disk 90 (retuned), disk 97, mem warning, mem critical, swap, packet loss606 t.Fatalf("%d firing alerts from the builtin set, want 6", n)607 }608}609610func TestReplayOnHistory(t *testing.T) {611 e := newEnv(t)612 n1, n2 := e.node("T-A"), e.node("T-B")613 e.node("T-EMPTY")614 base := time.Now().UTC().Truncate(time.Hour).Add(-3 * time.Hour)615 for _, tbl := range []string{"samples_raw", "samples_1m"} {616 if _, err := e.pool.Exec(bg, fmt.Sprintf(`create table %s_t partition of %s for values from ('%s') to ('%s')`, tbl, tbl,617 base.Add(-48*time.Hour).Format(time.RFC3339), base.Add(48*time.Hour).Format(time.RFC3339))); err != nil {618 t.Fatal(err)619 }620 }621 cpuID, _ := protocol.MetricByName("cpu.usage")622 used, _ := protocol.MetricByName("disk.used")623 total, _ := protocol.MetricByName("disk.total")624 put := func(node uuid.UUID, metric int16, label string, at time.Duration, v float64) {625 if _, err := e.pool.Exec(bg, `insert into samples_raw(node_id, metric_id, label, ts, value) values ($1,$2,$3,$4,$5)`, node, metric, label, base.Add(at), v); err != nil {626 t.Fatal(err)627 }628 }629 // T-A: 10 s samples. 0-5 min calm; 5-12 min at 97; 12-20 min calm; a 30 s spike at 25 min; from 30 min at 99 until the data stops at 40 min.630 for s := 0; s <= 40*60; s += 10 {631 v := 20.0632 switch {633 case s >= 5*60 && s < 12*60, s >= 30*60:634 v = 97635 case s >= 25*60 && s < 25*60+30:636 v = 99637 }638 put(n1, cpuID.ID, "", time.Duration(s)*time.Second, v)639 }640 // T-B: breach, then a 10-minute hole in the data, then calm.641 for s := 0; s <= 10*60; s += 10 {642 put(n2, cpuID.ID, "", time.Duration(s)*time.Second, 99)643 }644 for s := 20 * 60; s <= 30*60; s += 10 {645 put(n2, cpuID.ID, "", time.Duration(s)*time.Second, 5)646 }647 rule := alerts.Rule{Name: "draft", Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(90), ClearThreshold: fp(80), ForSeconds: 120, ClearSeconds: 60,648 Severity: "warning", Scope: alerts.Scope{All: true}}649 res, err := alerts.Replay(bg, e.pool, rule, base, base.Add(3*time.Hour))650 if err != nil {651 t.Fatal(err)652 }653 if res.Resolution != "raw" || res.NodesEvaluated != 3 || res.NodesWithoutData != 1 || len(res.Intervals) != 3 {654 t.Fatalf("%+v", res)655 }656 at := func(d time.Duration) time.Time { return base.Add(d) }657 a1, a2, b := res.Intervals[0], res.Intervals[1], res.Intervals[2]658 if a1.NodeAlias != "T-A" || !a1.StartedAt.Equal(at(7*time.Minute)) || a1.EndedAt == nil || !a1.EndedAt.Equal(at(13*time.Minute)) || a1.Peak != 97 {659 t.Fatalf("first interval: %+v", a1)660 }661 // Data stops at 40 min while firing: closed for lack of data, clear_seconds later.662 if !a2.StartedAt.Equal(at(32*time.Minute)) || a2.EndedAt == nil || !a2.EndedAt.Equal(at(41*time.Minute)) {663 t.Fatalf("second interval: %+v", a2)664 }665 // T-B: fired at 2 min; the hole closes it one clear delay after the last sample.666 if b.NodeAlias != "T-B" || !b.StartedAt.Equal(at(2*time.Minute)) || b.EndedAt == nil || !b.EndedAt.Equal(at(11*time.Minute)) {667 t.Fatalf("T-B interval: %+v", b)668 }669 // Scope restricts the nodes.670 rule.Scope = alerts.Scope{NodeIDs: []uuid.UUID{n2}}671 res, _ = alerts.Replay(bg, e.pool, rule, base, base.Add(3*time.Hour))672 if res.NodesEvaluated != 1 || len(res.Intervals) != 1 {673 t.Fatalf("scoped: %+v", res)674 }675 // Still firing at the end of the window: no end.676 rule.Scope = alerts.Scope{All: true}677 res, _ = alerts.Replay(bg, e.pool, rule, base, base.Add(35*time.Minute))678 if last := res.Intervals[len(res.Intervals)-1]; last.NodeAlias != "T-B" && last.EndedAt != nil {679 t.Fatalf("open interval: %+v", res.Intervals)680 }681 var open int682 for _, iv := range res.Intervals {683 if iv.EndedAt == nil {684 open++685 }686 }687 if open != 1 {688 t.Fatalf("%d open intervals, want 1: %+v", open, res.Intervals)689 }690 // Derived percentage on a labeled metric.691 for s := 0; s <= 20*60; s += 60 {692 put(n1, used.ID, "/", time.Duration(s)*time.Second, 960)693 put(n1, total.ID, "/", time.Duration(s)*time.Second, 1000)694 put(n1, used.ID, "/data", time.Duration(s)*time.Second, 10)695 put(n1, total.ID, "/data", time.Duration(s)*time.Second, 1000)696 }697 res, err = alerts.Replay(bg, e.pool, alerts.Rule{Name: "d", Kind: "threshold", Metric: alerts.MetricDiskPct, LabelMatch: "*", Op: ">", Threshold: fp(90),698 ForSeconds: 300, ClearSeconds: 60, Severity: "warning", Scope: alerts.Scope{All: true}}, base, base.Add(time.Hour))699 if err != nil || len(res.Intervals) != 1 || res.Intervals[0].Label != "/" || res.Intervals[0].Peak != 96 || !res.Intervals[0].StartedAt.Equal(at(5*time.Minute)) {700 t.Fatalf("derived: %v %+v", err, res)701 }702 // Bounds and non-replayable rules.703 if _, err := alerts.Replay(bg, e.pool, rule, base, base.Add(40*24*time.Hour)); err == nil {704 t.Fatal("unbounded range accepted")705 }706 if _, err := alerts.Replay(bg, e.pool, alerts.Rule{Kind: "absence"}, base, base.Add(time.Hour)); err != alerts.ErrNotReplayable {707 t.Fatalf("absence replay: %v", err)708 }709 // Longer windows use the aggregates and say so.710 res, err = alerts.Replay(bg, e.pool, rule, base.Add(-24*time.Hour), base.Add(3*time.Hour))711 if err != nil || res.Resolution != "1m" || len(res.Warnings) == 0 {712 t.Fatalf("%v %+v", err, res)713 }714}715716func TestRuleValidation(t *testing.T) {717 ok := alerts.Rule{Name: "x", Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(1), Severity: "info", Scope: alerts.Scope{All: true}}718 if err := ok.Validate(); err != nil {719 t.Fatal(err)720 }721 bad := map[string]func(r *alerts.Rule){722 "unknown metric": func(r *alerts.Rule) { r.Metric = "cpu.magic" },723 "missing op": func(r *alerts.Rule) { r.Op = "" },724 "sql in op": func(r *alerts.Rule) { r.Op = "> 0 or 1=1" },725 "missing threshold": func(r *alerts.Rule) { r.Threshold = nil },726 "clear on wrong side": func(r *alerts.Rule) { r.ClearThreshold = fp(5) },727 "negative for": func(r *alerts.Rule) { r.ForSeconds = -1 },728 "huge clear": func(r *alerts.Rule) { r.ClearSeconds = 90000 },729 "bad severity": func(r *alerts.Rule) { r.Severity = "fatal" },730 "empty scope": func(r *alerts.Rule) { r.Scope = alerts.Scope{} },731 "two scope forms": func(r *alerts.Rule) { r.Scope = alerts.Scope{All: true, Tags: []string{"a"}} },732 "bad kind": func(r *alerts.Rule) { r.Kind = "script" },733 "empty name": func(r *alerts.Rule) { r.Name = " " },734 "absence with metric": func(r *alerts.Rule) { r.Kind = "absence" },735 "service unknown state": func(r *alerts.Rule) { r.Kind, r.Op, r.Threshold, r.Metric = "service_state", "", nil, "exploded" },736 }737 for name, mut := range bad {738 r := ok739 mut(&r)740 if r.Validate() == nil {741 t.Errorf("%s: accepted", name)742 }743 }744}745