SPB Git

Repository Public

engine_test.go 29.3 KB 745 lines go

1package alerts_test23import (4	"context"5	"fmt"6	"net/netip"7	"strings"8	"sync"9	"testing"10	"time"1112	"github.com/google/uuid"13	"github.com/jackc/pgx/v5/pgxpool"1415	"clustr.spboucher.ai/mcc/internal/alerts"16	"clustr.spboucher.ai/mcc/internal/audit"17	"clustr.spboucher.ai/mcc/internal/ids"18	"clustr.spboucher.ai/mcc/internal/protocol"19	"clustr.spboucher.ai/mcc/internal/testdb"20)2122var bg = context.Background()2324type env struct {25	t     *testing.T26	pool  *pgxpool.Pool27	clock *testdb.Clock28	eng   *alerts.Engine29}3031func newEnv(t *testing.T) *env {32	t.Helper()33	pool, _ := testdb.New(t)34	e := &env{t: t, pool: pool, clock: testdb.NewClock()}35	e.eng = &alerts.Engine{Pool: pool, Now: e.clock.Now, FreshAfter: 30 * time.Second}36	return e37}3839func fp(v float64) *float64 { return &v }4041func (e *env) rule(r alerts.Rule) uuid.UUID {42	e.t.Helper()43	r.ID = ids.New()44	if !r.Scope.All && r.Scope.NodeIDs == nil && r.Scope.Tags == nil {45		r.Scope = alerts.Scope{All: true}46	}47	if r.Severity == "" {48		r.Severity = "warning"49	}50	if r.Name == "" {51		r.Name = "rule " + r.ID.String()[24:]52	}53	if err := r.Validate(); err != nil {54		e.t.Fatal(err)55	}56	if _, err := e.pool.Exec(bg, `insert into alert_rules(id, name, kind, metric, label_match, op, threshold, clear_threshold, for_seconds, clear_seconds, severity, scope, notify)57		values ($1,$2,$3,nullif($4,''),nullif($5,''),nullif($6,''),$7,$8,$9,$10,$11::severity,$12,true)`,58		r.ID, r.Name, r.Kind, r.Metric, r.LabelMatch, r.Op, r.Threshold, r.ClearThreshold, r.ForSeconds, r.ClearSeconds, r.Severity, r.Scope); err != nil {59		e.t.Fatal(err)60	}61	return r.ID62}6364// node adds a node with an agent that reports now.65func (e *env) node(alias string, tags ...string) uuid.UUID {66	id := testdb.Node(e.t, e.pool, alias, tags...)67	testdb.Agent(e.t, e.pool, id, e.clock.Now())68	return id69}7071// report advances the clock by d and stores a fresh snapshot, then evaluates.72func (e *env) report(d time.Duration, node uuid.UUID, values map[string]any) {73	e.t.Helper()74	e.clock.Advance(d)75	testdb.State(e.t, e.pool, node, e.clock.Now(), values)76	e.eval()77}7879func (e *env) eval() {80	e.t.Helper()81	if err := e.eng.Evaluate(bg); err != nil {82		e.t.Fatal(err)83	}84}8586type arow struct {87	ID         uuid.UUID88	State      string89	Flaps      int90	Suppressed string91	Summary    string92	Label      string93	Value      *float6494	Started    time.Time95	ClearSince *time.Time96}9798// alerts returns the alerts of a rule, oldest first.99func (e *env) alerts(rule uuid.UUID) []arow {100	e.t.Helper()101	rows, err := e.pool.Query(bg, `select id, state::text, flap_count, coalesce(suppressed_by,''), summary, label, last_value, started_at, clear_since102		from alerts where rule_id=$1 order by started_at, id`, rule)103	if err != nil {104		e.t.Fatal(err)105	}106	defer rows.Close()107	var out []arow108	for rows.Next() {109		var a arow110		if err := rows.Scan(&a.ID, &a.State, &a.Flaps, &a.Suppressed, &a.Summary, &a.Label, &a.Value, &a.Started, &a.ClearSince); err != nil {111			e.t.Fatal(err)112		}113		out = append(out, a)114	}115	return out116}117118func (e *env) one(rule uuid.UUID, state string) arow {119	e.t.Helper()120	as := e.alerts(rule)121	if len(as) == 0 {122		e.t.Fatalf("no alert, want one %s", state)123	}124	a := as[len(as)-1]125	if a.State != state {126		e.t.Fatalf("alert state = %s, want %s (%+v)", a.State, state, a)127	}128	return a129}130131func (e *env) count(sql string, args ...any) int {132	e.t.Helper()133	var n int134	if err := e.pool.QueryRow(bg, sql, args...).Scan(&n); err != nil {135		e.t.Fatal(err)136	}137	return n138}139140func (e *env) timeline(alert uuid.UUID) string {141	rows, _ := e.pool.Query(bg, `select kind from timeline_events where alert_id=$1 order by ts, id`, alert)142	defer rows.Close()143	var ks []string144	for rows.Next() {145		var k string146		rows.Scan(&k)147		ks = append(ks, k)148	}149	return strings.Join(ks, ",")150}151152func cpu(v float64) map[string]any { return map[string]any{"cpu.usage": v} }153154func TestStateMachinePendingFiringHysteresisFlapResolve(t *testing.T) {155	e := newEnv(t)156	n := e.node("T-A")157	r := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(90), ClearThreshold: fp(80), ForSeconds: 60, ClearSeconds: 30, Severity: "critical"})158	notif := func(tr string) int {159		return e.count(`select count(*) from alert_notifications q join alerts a on a.id=q.alert_id where a.rule_id=$1 and q.transition=$2`, r, tr)160	}161162	e.report(0, n, cpu(50))163	if len(e.alerts(r)) != 0 {164		t.Fatal("alert below threshold")165	}166	// A breach shorter than for_seconds never fires and never notifies.167	e.report(10*time.Second, n, cpu(95))168	e.one(r, "pending")169	e.report(10*time.Second, n, cpu(50))170	e.one(r, "resolved")171	if notif("firing")+notif("resolved") != 0 {172		t.Fatal("a pending alert that never fired produced a notification")173	}174175	e.report(10*time.Second, n, cpu(95))176	a := e.one(r, "pending")177	start := e.clock.Now()178	e.report(30*time.Second, n, cpu(96))179	e.one(r, "pending")180	e.report(29*time.Second, n, cpu(97))181	e.one(r, "pending")182	e.report(1*time.Second, n, cpu(97)) // 60 s183	a = e.one(r, "firing")184	if !a.Started.Equal(start) || *a.Value != 97 || !strings.Contains(a.Summary, "T-A") || notif("firing") != 1 {185		t.Fatalf("%+v notif=%d", a, notif("firing"))186	}187	// Inside the hysteresis band (80 < v <= 90): stays firing, does not start clearing.188	e.report(10*time.Second, n, cpu(85))189	if a = e.one(r, "firing"); a.ClearSince != nil {190		t.Fatal("clearing started inside the hysteresis band")191	}192	// Below clear_threshold: clearing starts but needs clear_seconds.193	e.report(10*time.Second, n, cpu(70))194	if a = e.one(r, "firing"); a.ClearSince == nil {195		t.Fatal("clearing not started")196	}197	e.report(20*time.Second, n, cpu(70))198	e.one(r, "firing")199	// Back above before the delay: one flap, the delay restarts.200	e.report(5*time.Second, n, cpu(95))201	if a = e.one(r, "firing"); a.Flaps != 1 || a.ClearSince != nil {202		t.Fatalf("flap not counted: %+v", a)203	}204	e.report(10*time.Second, n, cpu(70))205	e.report(29*time.Second, n, cpu(70))206	e.one(r, "firing")207	e.report(1*time.Second, n, cpu(70))208	a = e.one(r, "resolved")209	if notif("firing") != 1 || notif("resolved") != 1 {210		t.Fatalf("notifications: firing=%d resolved=%d", notif("firing"), notif("resolved"))211	}212	if tl := e.timeline(a.ID); tl != "opened,fired,state_changed,resolved" {213		t.Fatalf("timeline: %s", tl)214	}215	// Re-opening soon after is a flap of the same problem.216	e.report(10*time.Second, n, cpu(99))217	e.report(60*time.Second, n, cpu(99))218	a = e.one(r, "firing")219	if a.Flaps != 2 || !strings.HasPrefix(e.timeline(a.ID), "reopened") {220		t.Fatalf("reopen: %+v %s", a, e.timeline(a.ID))221	}222	if n := e.count(`select count(*) from events_outbox where topic='alert.changed' and payload->>'rule_id'=$1`, r.String()); n < 6 {223		t.Fatalf("alert.changed events: %d", n)224	}225	// Disabling the rule closes what it had open, without notification.226	e.pool.Exec(bg, `update alert_rules set enabled=false where id=$1`, r)227	e.eval()228	e.one(r, "resolved")229	if notif("resolved") != 1 {230		t.Fatal("disabling a rule must not notify")231	}232}233234func TestDedupUnderConcurrentEvaluators(t *testing.T) {235	e := newEnv(t)236	var nodes []uuid.UUID237	for i := 0; i < 5; i++ {238		n := e.node(fmt.Sprintf("T-%d", i))239		testdb.State(t, e.pool, n, e.clock.Now(), cpu(99))240		nodes = append(nodes, n)241	}242	r := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(90), ForSeconds: 0})243	var wg sync.WaitGroup244	errs := make(chan error, 64)245	for i := 0; i < 8; i++ {246		wg.Add(1)247		go func() {248			defer wg.Done()249			for j := 0; j < 4; j++ {250				if err := e.eng.Evaluate(bg); err != nil {251					errs <- err252				}253			}254		}()255	}256	wg.Wait()257	close(errs)258	for err := range errs {259		t.Fatal(err)260	}261	if n := e.count(`select count(*) from alerts where rule_id=$1`, r); n != 5 {262		t.Fatalf("%d alerts for 5 nodes", n)263	}264	if n := e.count(`select count(*) from alerts where rule_id=$1 and state='firing'`, r); n != 5 {265		t.Fatalf("%d firing", n)266	}267	if n := e.count(`select count(*) from alert_notifications`); n != 5 {268		t.Fatalf("%d notifications for 5 firings", n)269	}270	if n := e.count(`select count(*) from timeline_events where kind='fired'`); n != 5 {271		t.Fatalf("%d fired events", n)272	}273}274275func TestSilenceAndMaintenanceSuppression(t *testing.T) {276	e := newEnv(t)277	n1, n2, n3 := e.node("T-1"), e.node("T-2", "lab"), e.node("T-3")278	r := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(90), ForSeconds: 0, ClearSeconds: 0})279	other := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(50), ForSeconds: 0, ClearSeconds: 0})280	now := e.clock.Now()281	sil := func(scope string, rule *uuid.UUID) uuid.UUID {282		id := ids.New()283		if _, err := e.pool.Exec(bg, `insert into silences(id, reason, scope, rule_id, starts_at, ends_at) values ($1,'test',$2::jsonb,$3,$4,$5)`,284			id, scope, rule, now.Add(-time.Minute), now.Add(time.Hour)); err != nil {285			t.Fatal(err)286		}287		return id288	}289	s1 := sil(`{"node_ids":["`+n1.String()+`"]}`, &r) // n1, this rule only290	sil(`{"tags":["lab"]}`, nil)                      // n2, every rule291	e.pool.Exec(bg, `update nodes set maintenance=true where id=$1`, n3)292	for _, n := range []uuid.UUID{n1, n2, n3} {293		testdb.State(t, e.pool, n, now, cpu(99))294	}295	e.eval()296	got := map[string]string{}297	rows, _ := e.pool.Query(bg, `select n.alias::text || '/' || case when a.rule_id=$1 then 'r' else 'o' end, coalesce(a.suppressed_by,''), a.state::text298		from alerts a join nodes n on n.id=a.node_id`, r)299	for rows.Next() {300		var k, s, st string301		rows.Scan(&k, &s, &st)302		if st != "firing" {303			t.Fatalf("%s is %s: a suppressed alert must still be tracked", k, st)304		}305		got[k] = s306	}307	want := map[string]string{"T-1/r": "silence", "T-1/o": "", "T-2/r": "silence", "T-2/o": "silence", "T-3/r": "maintenance", "T-3/o": "maintenance"}308	for k, v := range want {309		if got[k] != v {310			t.Errorf("%s: suppressed_by=%q, want %q", k, got[k], v)311		}312	}313	_ = other314	// Only the one unsuppressed alert notifies.315	if n := e.count(`select count(*) from alert_notifications`); n != 1 {316		t.Fatalf("%d notifications, want 1", n)317	}318	if n := e.count(`select count(*) from timeline_events where kind='suppressed'`); n != 5 {319		t.Fatalf("%d suppressed timeline events", n)320	}321	// Lifting the silence restores notifications for later transitions and says so.322	e.pool.Exec(bg, `delete from silences where id=$1`, s1)323	e.clock.Advance(10 * time.Second)324	for _, n := range []uuid.UUID{n1, n2, n3} {325		testdb.State(t, e.pool, n, e.clock.Now(), cpu(99))326	}327	e.eval()328	if e.count(`select count(*) from alerts where rule_id=$1 and node_id=$2 and suppressed_by is null and state='firing'`, r, n1) != 1 {329		t.Fatal("silence removal not reflected")330	}331	// A resolution while in maintenance does not notify either.332	testdb.State(t, e.pool, n3, e.clock.Now(), cpu(10))333	testdb.State(t, e.pool, n1, e.clock.Now(), cpu(10))334	e.eval()335	if n := e.count(`select count(*) from alert_notifications where transition='resolved'`); n != 2 { // n1: both rules336		t.Fatalf("%d resolved notifications, want 2 (none for the node in maintenance)", n)337	}338}339340func TestStaleDataIsNotEvaluable(t *testing.T) {341	e := newEnv(t)342	n := e.node("T-A")343	r := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(90), ForSeconds: 0, ClearSeconds: 60})344	// A stale breach never opens an alert.345	testdb.State(t, e.pool, n, e.clock.Now().Add(-31*time.Second), cpu(99))346	e.eval()347	if len(e.alerts(r)) != 0 {348		t.Fatal("a stale value opened an alert")349	}350	e.report(0, n, cpu(99))351	e.one(r, "firing")352	// The node goes silent with its last value still at 99: the alert must not be kept on that value.353	e.clock.Advance(31 * time.Second)354	e.eval()355	a := e.one(r, "firing")356	if a.ClearSince == nil {357		t.Fatal("no-data clearing not started")358	}359	e.clock.Advance(59 * time.Second)360	e.eval()361	e.one(r, "firing")362	e.clock.Advance(2 * time.Second)363	e.eval()364	a = e.one(r, "resolved")365	var body string366	e.pool.QueryRow(bg, `select body from timeline_events where alert_id=$1 and kind='resolved'`, a.ID).Scan(&body)367	if !strings.Contains(body, "no recent data") {368		t.Fatalf("resolution reason: %q", body)369	}370	// Pending + stale: dropped at once.371	r2 := e.rule(alerts.Rule{Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(95), ForSeconds: 300})372	e.report(0, n, cpu(99))373	e.one(r2, "pending")374	e.clock.Advance(40 * time.Second)375	e.eval()376	e.one(r2, "resolved")377	// However long the stale 99 stays in node_state, nothing fires.378	e.clock.Advance(time.Hour)379	e.eval()380	if e.count(`select count(*) from alerts where state <> 'resolved'`) != 0 {381		t.Fatal("stale data kept or opened an alert")382	}383}384385func TestAbsenceRule(t *testing.T) {386	e := newEnv(t)387	n := e.node("T-A")388	manual := testdb.Node(t, e.pool, "T-MANUAL") // registered by hand, never had an agent389	_ = manual390	r := e.rule(alerts.Rule{Kind: "absence", ForSeconds: 120, ClearSeconds: 30, Severity: "critical"})391	last := e.clock.Now()392	testdb.State(t, e.pool, n, last, cpu(1))393	e.clock.Advance(20 * time.Second)394	e.eval()395	if len(e.alerts(r)) != 0 {396		t.Fatal("alert while reporting")397	}398	e.clock.Advance(20 * time.Second) // 40 s of silence399	e.eval()400	a := e.one(r, "pending")401	if !a.Started.Equal(last) {402		t.Fatalf("started_at %s, want the last report %s", a.Started, last)403	}404	e.clock.Advance(79 * time.Second) // 119 s405	e.eval()406	e.one(r, "pending")407	e.clock.Advance(2 * time.Second) // 121 s408	e.eval()409	a = e.one(r, "firing")410	if len(e.alerts(r)) != 1 || !strings.Contains(a.Summary, "T-A") {411		t.Fatalf("%+v", e.alerts(r))412	}413	// Reporting again clears it after clear_seconds.414	e.report(10*time.Second, n, cpu(1))415	e.one(r, "firing")416	e.report(15*time.Second, n, cpu(1))417	e.one(r, "firing")418	e.report(15*time.Second, n, cpu(1))419	e.one(r, "resolved")420	// A node that never reported since enrollment is measured from its enrollment.421	n2 := testdb.Node(t, e.pool, "T-NEW")422	testdb.Agent(t, e.pool, n2, e.clock.Now())423	e.pool.Exec(bg, `update agents set last_seen_at=null where node_id=$1`, n2)424	e.clock.Advance(121 * time.Second)425	testdb.State(t, e.pool, n, e.clock.Now(), cpu(1))426	e.eval()427	if e.count(`select count(*) from alerts where rule_id=$1 and node_id=$2 and state='firing'`, r, n2) != 1 {428		t.Fatal("never-reporting node not detected")429	}430	if e.count(`select count(*) from alerts where node_id=$1`, manual) != 0 {431		t.Fatal("a node without agent cannot be absent")432	}433}434435func TestServiceStateRule(t *testing.T) {436	e := newEnv(t)437	n := e.node("T-A")438	svc := testdb.Service(t, e.pool, n, "web", true, "errored")439	testdb.Service(t, e.pool, n, "scratch", false, "errored") // not managed: ignored440	stopped := testdb.Service(t, e.pool, n, "worker", true, "stopped")441	r := e.rule(alerts.Rule{Kind: "service_state", Metric: "errored", ForSeconds: 120, ClearSeconds: 0, Severity: "critical"})442	any := e.rule(alerts.Rule{Kind: "service_state", ForSeconds: 0, ClearSeconds: 0, LabelMatch: "pm2:worker"})443	e.report(0, n, cpu(1))444	a := e.one(r, "pending")445	if a.Label != "pm2:web" || len(e.alerts(r)) != 1 {446		t.Fatalf("%+v", e.alerts(r))447	}448	if b := e.one(any, "firing"); b.Label != "pm2:worker" {449		t.Fatalf("%+v", b)450	}451	e.report(121*time.Second, n, cpu(1))452	e.one(r, "firing")453	e.pool.Exec(bg, `update services set state='online' where id in ($1,$2)`, svc, stopped)454	e.report(10*time.Second, n, cpu(1))455	e.one(r, "resolved")456	e.one(any, "resolved")457	// Service state of a node that stopped reporting is not evaluated.458	e.pool.Exec(bg, `update services set state='errored' where id=$1`, svc)459	e.clock.Advance(5 * time.Minute)460	e.eval()461	if e.count(`select count(*) from alerts where rule_id=$1 and state <> 'resolved'`, r) != 0 {462		t.Fatal("service alert from a silent node")463	}464}465466func TestLabeledAndDerivedMetrics(t *testing.T) {467	e := newEnv(t)468	n := e.node("T-A")469	disk := e.rule(alerts.Rule{Kind: "threshold", Metric: alerts.MetricDiskPct, LabelMatch: "*", Op: ">", Threshold: fp(90), ClearThreshold: fp(88), ForSeconds: 0, ClearSeconds: 0})470	loss := e.rule(alerts.Rule{Kind: "threshold", Metric: "net.loss", LabelMatch: "gateway", Op: ">=", Threshold: fp(34), ForSeconds: 0, ClearSeconds: 0})471	swap := e.rule(alerts.Rule{Kind: "threshold", Metric: alerts.MetricSwapPct, Op: ">", Threshold: fp(80), ForSeconds: 0, ClearSeconds: 0})472	vals := map[string]any{"disk.used{/}": 950.0, "disk.total{/}": 1000.0, "disk.used{/Volumes/Data}": 100.0, "disk.total{/Volumes/Data}": 1000.0,473		"net.loss{gateway}": 0.0, "net.loss{hub}": 100.0, "swap.used": 0.0, "swap.total": 0.0}474	e.report(0, n, vals)475	a := e.one(disk, "firing")476	if a.Label != "/" || *a.Value != 95 || len(e.alerts(disk)) != 1 {477		t.Fatalf("%+v", e.alerts(disk))478	}479	if len(e.alerts(loss)) != 0 || len(e.alerts(swap)) != 0 {480		t.Fatal("loss to another target, or swap with total=0, raised an alert")481	}482	vals["net.loss{gateway}"], vals["swap.used"], vals["swap.total"] = 67.0, 900.0, 1000.0483	vals["disk.used{/}"] = 890.0 // inside the hysteresis band: still firing484	e.report(10*time.Second, n, vals)485	e.one(disk, "firing")486	e.one(loss, "firing")487	if s := e.one(swap, "firing"); *s.Value != 90 {488		t.Fatalf("%+v", s)489	}490	// The volume disappears (unmounted): its alert cannot be verified any more and closes.491	delete(vals, "disk.used{/}")492	delete(vals, "disk.total{/}")493	e.report(10*time.Second, n, vals)494	e.one(disk, "resolved")495}496497func TestServerComputedMetrics(t *testing.T) {498	e := newEnv(t)499	n := e.node("T-A")500	cert := e.rule(alerts.Rule{Kind: "threshold", Metric: alerts.MetricCertDays, Op: "<", Threshold: fp(7), ForSeconds: 0, ClearSeconds: 0})501	denied := e.rule(alerts.Rule{Kind: "threshold", Metric: alerts.MetricDenied10m, LabelMatch: "*", Op: ">=", Threshold: fp(10), ForSeconds: 0, ClearSeconds: 0})502	newdev := e.rule(alerts.Rule{Kind: "threshold", Metric: alerts.MetricNewDevice, LabelMatch: "*", Op: ">=", Threshold: fp(1), ForSeconds: 0, ClearSeconds: 0, Severity: "info"})503	e.eval()504	if e.count(`select count(*) from alerts`) != 0 {505		t.Fatal("alerts without cause")506	}507	// Certificate about to expire (renewal not happening).508	e.pool.Exec(bg, `update agents set cert_not_after=$2 where node_id=$1`, n, e.clock.Now().Add(6*24*time.Hour))509	e.eval()510	e.one(cert, "firing")511	e.pool.Exec(bg, `update agents set cert_not_after=$2 where node_id=$1`, n, e.clock.Now().Add(30*24*time.Hour))512	e.eval()513	e.one(cert, "resolved")514515	// Burst of denied outcomes from one account / one address. (audit timestamps are real time: use a real clock here)516	e.eng.Now = func() time.Time { return time.Now().UTC() }517	mallory := testdb.User(t, e.pool, "mallory", "admin")518	ip := netip.MustParseAddr("203.0.113.9")519	for i := 0; i < 10; i++ {520		if _, err := audit.Append(bg, e.pool, audit.Entry{ActorKind: "user", ActorID: &mallory, ActorLabel: "mallory", IP: &ip, Action: "authz.denied", Outcome: audit.Denied}); err != nil {521			t.Fatal(err)522		}523	}524	audit.Append(bg, e.pool, audit.Entry{ActorKind: "user", ActorLabel: "bob", Action: "authz.denied", Outcome: audit.Denied})525	e.eval()526	as := e.alerts(denied)527	labels := map[string]bool{}528	for _, a := range as {529		labels[a.Label] = a.State == "firing"530	}531	if len(as) != 2 || !labels["actor:mallory"] || !labels["ip:203.0.113.9"] {532		t.Fatalf("denied burst: %+v", as)533	}534	// Sign-in of an admin from a device created by that sign-in, while an older device exists.535	old, fresh := ids.New(), ids.New()536	e.pool.Exec(bg, `insert into devices(id, user_id, kind, created_at) values ($1,$3,'web',now()-interval '30 days'), ($2,$3,'ios',now())`, old, fresh, mallory)537	audit.Append(bg, e.pool, audit.Entry{ActorKind: "user", ActorID: &mallory, ActorLabel: "mallory", DeviceID: &old, Action: "auth.login", Outcome: audit.Success})538	e.eval()539	if len(e.alerts(newdev)) != 0 {540		t.Fatal("sign-in from a known device flagged")541	}542	audit.Append(bg, e.pool, audit.Entry{ActorKind: "user", ActorID: &mallory, ActorLabel: "mallory", DeviceID: &fresh, Action: "auth.login", Outcome: audit.Success})543	e.eval()544	if a := e.one(newdev, "firing"); a.Label != "mallory" {545		t.Fatalf("%+v", a)546	}547	// Viewers and operators are not concerned.548	oscar := testdb.User(t, e.pool, "oscar", "operator")549	o1, o2 := ids.New(), ids.New()550	e.pool.Exec(bg, `insert into devices(id, user_id, kind, created_at) values ($1,$3,'web',now()-interval '30 days'), ($2,$3,'ios',now())`, o1, o2, oscar)551	audit.Append(bg, e.pool, audit.Entry{ActorKind: "user", ActorID: &oscar, ActorLabel: "oscar", DeviceID: &o2, Action: "auth.login", Outcome: audit.Success})552	e.eval()553	if len(e.alerts(newdev)) != 1 {554		t.Fatal("operator sign-in flagged")555	}556	// Once the window has passed, the count is zero and the alerts close.557	e.eng.Now = func() time.Time { return time.Now().UTC().Add(2 * time.Hour) }558	testdb.State(t, e.pool, n, time.Now().UTC().Add(2*time.Hour), cpu(1))559	e.eval()560	if e.count(`select count(*) from alerts where rule_id in ($1,$2) and state <> 'resolved'`, denied, newdev) != 0 {561		t.Fatal("audit-derived alerts did not close")562	}563}564565func TestBuiltinRulesSeededOnceAndEditsPreserved(t *testing.T) {566	e := newEnv(t)567	for i := 0; i < 2; i++ {568		if err := alerts.SeedBuiltins(bg, e.pool); err != nil {569			t.Fatal(err)570		}571	}572	if n := e.count(`select count(*) from alert_rules where builtin and enabled`); n != len(alerts.Builtins) || n != 12 {573		t.Fatalf("%d builtin rules", n)574	}575	for _, name := range []string{"Node offline", "Disk volume above 90 %", "Disk volume above 97 %", "Memory pressure (warning)", "Memory pressure (critical)",576		"Swap above 80 %", "CPU saturated", "Managed service errored", "Agent certificate expiring", "Packet loss to gateway"} {577		if e.count(`select count(*) from alert_rules where name=$1`, name) != 1 {578			t.Errorf("missing builtin %q", name)579		}580	}581	var sev string582	var thr, forS float64583	e.pool.QueryRow(bg, `select severity::text, threshold, for_seconds from alert_rules where name='Packet loss to gateway'`).Scan(&sev, &thr, &forS)584	if sev != "warning" || thr != 34 || forS != 300 {585		t.Fatalf("packet loss rule: %s %v %v", sev, thr, forS)586	}587	// The owner disables one rule and retunes another; a restart must not undo it.588	e.pool.Exec(bg, `update alert_rules set enabled=false where name='CPU saturated'`)589	e.pool.Exec(bg, `update alert_rules set threshold=85, for_seconds=60, notify=false, updated_at=now() where name='Disk volume above 90 %'`)590	if err := alerts.SeedBuiltins(bg, e.pool); err != nil {591		t.Fatal(err)592	}593	if e.count(`select count(*) from alert_rules where name='CPU saturated' and not enabled`) != 1 ||594		e.count(`select count(*) from alert_rules where name='Disk volume above 90 %' and threshold=85 and for_seconds=60 and not notify`) != 1 ||595		e.count(`select count(*) from alert_rules`) != 12 {596		t.Fatal("a restart overwrote or re-enabled an edited builtin rule")597	}598	// The seeded set evaluates cleanly on real-looking data, and the disabled rule stays silent.599	n := e.node("T-A")600	e.report(0, n, map[string]any{"cpu.usage": 99.0, "mem.pressure.level": 4.0, "disk.used{/}": 99.0, "disk.total{/}": 100.0, "swap.used": 9.0, "swap.total": 10.0, "net.loss{gateway}": 100.0})601	e.report(16*time.Minute, n, map[string]any{"cpu.usage": 99.0, "mem.pressure.level": 4.0, "disk.used{/}": 99.0, "disk.total{/}": 100.0, "swap.used": 9.0, "swap.total": 10.0, "net.loss{gateway}": 100.0})602	if e.count(`select count(*) from alerts a join alert_rules r on r.id=a.rule_id where r.name='CPU saturated'`) != 0 {603		t.Fatal("a disabled builtin rule produced an alert")604	}605	if n := e.count(`select count(*) from alerts where state='firing'`); n != 6 { // disk 90 (retuned), disk 97, mem warning, mem critical, swap, packet loss606		t.Fatalf("%d firing alerts from the builtin set, want 6", n)607	}608}609610func TestReplayOnHistory(t *testing.T) {611	e := newEnv(t)612	n1, n2 := e.node("T-A"), e.node("T-B")613	e.node("T-EMPTY")614	base := time.Now().UTC().Truncate(time.Hour).Add(-3 * time.Hour)615	for _, tbl := range []string{"samples_raw", "samples_1m"} {616		if _, err := e.pool.Exec(bg, fmt.Sprintf(`create table %s_t partition of %s for values from ('%s') to ('%s')`, tbl, tbl,617			base.Add(-48*time.Hour).Format(time.RFC3339), base.Add(48*time.Hour).Format(time.RFC3339))); err != nil {618			t.Fatal(err)619		}620	}621	cpuID, _ := protocol.MetricByName("cpu.usage")622	used, _ := protocol.MetricByName("disk.used")623	total, _ := protocol.MetricByName("disk.total")624	put := func(node uuid.UUID, metric int16, label string, at time.Duration, v float64) {625		if _, err := e.pool.Exec(bg, `insert into samples_raw(node_id, metric_id, label, ts, value) values ($1,$2,$3,$4,$5)`, node, metric, label, base.Add(at), v); err != nil {626			t.Fatal(err)627		}628	}629	// T-A: 10 s samples. 0-5 min calm; 5-12 min at 97; 12-20 min calm; a 30 s spike at 25 min; from 30 min at 99 until the data stops at 40 min.630	for s := 0; s <= 40*60; s += 10 {631		v := 20.0632		switch {633		case s >= 5*60 && s < 12*60, s >= 30*60:634			v = 97635		case s >= 25*60 && s < 25*60+30:636			v = 99637		}638		put(n1, cpuID.ID, "", time.Duration(s)*time.Second, v)639	}640	// T-B: breach, then a 10-minute hole in the data, then calm.641	for s := 0; s <= 10*60; s += 10 {642		put(n2, cpuID.ID, "", time.Duration(s)*time.Second, 99)643	}644	for s := 20 * 60; s <= 30*60; s += 10 {645		put(n2, cpuID.ID, "", time.Duration(s)*time.Second, 5)646	}647	rule := alerts.Rule{Name: "draft", Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(90), ClearThreshold: fp(80), ForSeconds: 120, ClearSeconds: 60,648		Severity: "warning", Scope: alerts.Scope{All: true}}649	res, err := alerts.Replay(bg, e.pool, rule, base, base.Add(3*time.Hour))650	if err != nil {651		t.Fatal(err)652	}653	if res.Resolution != "raw" || res.NodesEvaluated != 3 || res.NodesWithoutData != 1 || len(res.Intervals) != 3 {654		t.Fatalf("%+v", res)655	}656	at := func(d time.Duration) time.Time { return base.Add(d) }657	a1, a2, b := res.Intervals[0], res.Intervals[1], res.Intervals[2]658	if a1.NodeAlias != "T-A" || !a1.StartedAt.Equal(at(7*time.Minute)) || a1.EndedAt == nil || !a1.EndedAt.Equal(at(13*time.Minute)) || a1.Peak != 97 {659		t.Fatalf("first interval: %+v", a1)660	}661	// Data stops at 40 min while firing: closed for lack of data, clear_seconds later.662	if !a2.StartedAt.Equal(at(32*time.Minute)) || a2.EndedAt == nil || !a2.EndedAt.Equal(at(41*time.Minute)) {663		t.Fatalf("second interval: %+v", a2)664	}665	// T-B: fired at 2 min; the hole closes it one clear delay after the last sample.666	if b.NodeAlias != "T-B" || !b.StartedAt.Equal(at(2*time.Minute)) || b.EndedAt == nil || !b.EndedAt.Equal(at(11*time.Minute)) {667		t.Fatalf("T-B interval: %+v", b)668	}669	// Scope restricts the nodes.670	rule.Scope = alerts.Scope{NodeIDs: []uuid.UUID{n2}}671	res, _ = alerts.Replay(bg, e.pool, rule, base, base.Add(3*time.Hour))672	if res.NodesEvaluated != 1 || len(res.Intervals) != 1 {673		t.Fatalf("scoped: %+v", res)674	}675	// Still firing at the end of the window: no end.676	rule.Scope = alerts.Scope{All: true}677	res, _ = alerts.Replay(bg, e.pool, rule, base, base.Add(35*time.Minute))678	if last := res.Intervals[len(res.Intervals)-1]; last.NodeAlias != "T-B" && last.EndedAt != nil {679		t.Fatalf("open interval: %+v", res.Intervals)680	}681	var open int682	for _, iv := range res.Intervals {683		if iv.EndedAt == nil {684			open++685		}686	}687	if open != 1 {688		t.Fatalf("%d open intervals, want 1: %+v", open, res.Intervals)689	}690	// Derived percentage on a labeled metric.691	for s := 0; s <= 20*60; s += 60 {692		put(n1, used.ID, "/", time.Duration(s)*time.Second, 960)693		put(n1, total.ID, "/", time.Duration(s)*time.Second, 1000)694		put(n1, used.ID, "/data", time.Duration(s)*time.Second, 10)695		put(n1, total.ID, "/data", time.Duration(s)*time.Second, 1000)696	}697	res, err = alerts.Replay(bg, e.pool, alerts.Rule{Name: "d", Kind: "threshold", Metric: alerts.MetricDiskPct, LabelMatch: "*", Op: ">", Threshold: fp(90),698		ForSeconds: 300, ClearSeconds: 60, Severity: "warning", Scope: alerts.Scope{All: true}}, base, base.Add(time.Hour))699	if err != nil || len(res.Intervals) != 1 || res.Intervals[0].Label != "/" || res.Intervals[0].Peak != 96 || !res.Intervals[0].StartedAt.Equal(at(5*time.Minute)) {700		t.Fatalf("derived: %v %+v", err, res)701	}702	// Bounds and non-replayable rules.703	if _, err := alerts.Replay(bg, e.pool, rule, base, base.Add(40*24*time.Hour)); err == nil {704		t.Fatal("unbounded range accepted")705	}706	if _, err := alerts.Replay(bg, e.pool, alerts.Rule{Kind: "absence"}, base, base.Add(time.Hour)); err != alerts.ErrNotReplayable {707		t.Fatalf("absence replay: %v", err)708	}709	// Longer windows use the aggregates and say so.710	res, err = alerts.Replay(bg, e.pool, rule, base.Add(-24*time.Hour), base.Add(3*time.Hour))711	if err != nil || res.Resolution != "1m" || len(res.Warnings) == 0 {712		t.Fatalf("%v %+v", err, res)713	}714}715716func TestRuleValidation(t *testing.T) {717	ok := alerts.Rule{Name: "x", Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: fp(1), Severity: "info", Scope: alerts.Scope{All: true}}718	if err := ok.Validate(); err != nil {719		t.Fatal(err)720	}721	bad := map[string]func(r *alerts.Rule){722		"unknown metric":        func(r *alerts.Rule) { r.Metric = "cpu.magic" },723		"missing op":            func(r *alerts.Rule) { r.Op = "" },724		"sql in op":             func(r *alerts.Rule) { r.Op = "> 0 or 1=1" },725		"missing threshold":     func(r *alerts.Rule) { r.Threshold = nil },726		"clear on wrong side":   func(r *alerts.Rule) { r.ClearThreshold = fp(5) },727		"negative for":          func(r *alerts.Rule) { r.ForSeconds = -1 },728		"huge clear":            func(r *alerts.Rule) { r.ClearSeconds = 90000 },729		"bad severity":          func(r *alerts.Rule) { r.Severity = "fatal" },730		"empty scope":           func(r *alerts.Rule) { r.Scope = alerts.Scope{} },731		"two scope forms":       func(r *alerts.Rule) { r.Scope = alerts.Scope{All: true, Tags: []string{"a"}} },732		"bad kind":              func(r *alerts.Rule) { r.Kind = "script" },733		"empty name":            func(r *alerts.Rule) { r.Name = " " },734		"absence with metric":   func(r *alerts.Rule) { r.Kind = "absence" },735		"service unknown state": func(r *alerts.Rule) { r.Kind, r.Op, r.Threshold, r.Metric = "service_state", "", nil, "exploded" },736	}737	for name, mut := range bad {738		r := ok739		mut(&r)740		if r.Validate() == nil {741			t.Errorf("%s: accepted", name)742		}743	}744}745
Files main
Menu

Theme