SPB Git

Repository Public

ops_alerts_test.go 15.1 KB 367 lines go

1package api_test23import (4	"context"5	"strings"6	"testing"7	"time"89	"github.com/google/uuid"1011	"clustr.spboucher.ai/mcc/internal/alerts"12	"clustr.spboucher.ai/mcc/internal/testdb"13)1415type alertBody struct {16	ID         string `json:"id"`17	State      string `json:"state"`18	RuleName   string `json:"rule_name"`19	NodeAlias  string `json:"node_alias"`20	AckBy      string `json:"acknowledged_by_display_name"`21	IncidentID string `json:"incident_id"`22	Suppressed string `json:"suppressed_by"`23}2425type ruleBody struct {26	ID        string   `json:"id"`27	Name      string   `json:"name"`28	Enabled   bool     `json:"enabled"`29	Builtin   bool     `json:"builtin"`30	Op        string   `json:"op"`31	Threshold *float64 `json:"threshold"`32	For       int      `json:"for_seconds"`33	Clear     int      `json:"clear_seconds"`34	Severity  string   `json:"severity"`35	Notify    bool     `json:"notify"`36}3738func TestAlertRulesAPI(t *testing.T) {39	e := newOpsEnv(t)40	ctx := context.Background()41	if err := alerts.SeedBuiltins(ctx, e.Pool); err != nil {42		t.Fatal(err)43	}44	var list struct{ Items []ruleBody }45	e.viewer.must("GET", "/alert-rules?limit=200", nil, 200).json(t, &list)46	if len(list.Items) != 12 {47		t.Fatalf("%d rules", len(list.Items))48	}49	var cpuRule ruleBody50	for _, r := range list.Items {51		if !r.Builtin || !r.Enabled {52			t.Fatalf("%+v", r)53		}54		if r.Name == "CPU saturated" {55			cpuRule = r56		}57	}58	if cpuRule.Op != "gt" || *cpuRule.Threshold != 95 || cpuRule.For != 900 {59		t.Fatalf("%+v", cpuRule)60	}61	draft := map[string]any{"name": "Load high", "kind": "threshold", "metric": "load.1", "op": "gte", "threshold": 20, "for_seconds": 0, "clear_seconds": 0,62		"severity": "critical", "scope": map[string]any{"node_ids": []string{e.node.String()}}}63	// Permissions and validation.64	if r := e.oper.do("POST", "/alert-rules", draft); r.Status != 403 {65		t.Fatalf("operator create rule: %d", r.Status)66	}67	for name, mut := range map[string]func(m map[string]any){68		"unknown metric":   func(m map[string]any) { m["metric"] = "nope" },69		"no threshold":     func(m map[string]any) { delete(m, "threshold") },70		"unknown node":     func(m map[string]any) { m["scope"] = map[string]any{"node_ids": []string{uuid.NewString()}} },71		"two scope forms":  func(m map[string]any) { m["scope"] = map[string]any{"all": true, "tags": []string{"x"}} },72		"negative for":     func(m map[string]any) { m["for_seconds"] = -5 },73		"clear wrong side": func(m map[string]any) { m["clear_threshold"] = 30 },74	} {75		m := map[string]any{}76		for k, v := range draft {77			m[k] = v78		}79		mut(m)80		if r := e.admin.do("POST", "/alert-rules", m); r.Status != 400 {81			t.Errorf("%s: %d %s", name, r.Status, r.Raw)82		}83	}84	var rule ruleBody85	e.admin.must("POST", "/alert-rules", draft, 201).json(t, &rule)86	if rule.Builtin || !rule.Enabled || rule.Op != "gte" || rule.Severity != "critical" || !rule.Notify {87		t.Fatalf("%+v", rule)88	}89	if r := e.admin.do("POST", "/alert-rules", draft); r.Status != 409 {90		t.Fatalf("duplicate name: %d", r.Status)91	}92	e.admin.must("PATCH", "/alert-rules/"+rule.ID, map[string]any{"threshold": 10, "notify": false}, 200).json(t, &rule)93	if *rule.Threshold != 10 || rule.Notify {94		t.Fatalf("%+v", rule)95	}96	if r := e.admin.do("PATCH", "/alert-rules/"+rule.ID, map[string]any{"metric": "nope"}); r.Status != 400 {97		t.Fatalf("patch to an invalid rule: %d", r.Status)98	}99	// Built-in rules: tunable and disablable, not renamable, not deletable.100	if r := e.admin.do("PATCH", "/alert-rules/"+cpuRule.ID, map[string]any{"name": "mine"}); r.Status != 409 {101		t.Fatalf("rename builtin: %d", r.Status)102	}103	if r := e.admin.do("PATCH", "/alert-rules/"+cpuRule.ID, map[string]any{"metric": "load.1"}); r.Status != 409 {104		t.Fatalf("change metric of builtin: %d", r.Status)105	}106	e.admin.must("PATCH", "/alert-rules/"+cpuRule.ID, map[string]any{"enabled": false, "threshold": 99, "for_seconds": 60}, 200).json(t, &cpuRule)107	if cpuRule.Enabled || *cpuRule.Threshold != 99 {108		t.Fatalf("%+v", cpuRule)109	}110	if r := e.admin.do("DELETE", "/alert-rules/"+cpuRule.ID, nil); r.Status != 409 {111		t.Fatalf("delete builtin: %d", r.Status)112	}113	alerts.SeedBuiltins(ctx, e.Pool) // "restart"114	e.viewer.must("GET", "/alert-rules/"+cpuRule.ID, nil, 200).json(t, &cpuRule)115	if cpuRule.Enabled || *cpuRule.Threshold != 99 || cpuRule.For != 60 {116		t.Fatalf("restart overwrote an edited builtin: %+v", cpuRule)117	}118	e.viewer.must("GET", "/alert-rules?enabled=false", nil, 200).json(t, &list)119	if len(list.Items) != 1 {120		t.Fatalf("enabled=false: %d", len(list.Items))121	}122123	// Deleting a rule resolves what it had open and keeps the history.124	now := time.Now().UTC()125	testdb.State(t, e.Pool, e.node, now, map[string]any{"load.1": 50.0})126	eng := &alerts.Engine{Pool: e.Pool}127	if err := eng.Evaluate(ctx); err != nil {128		t.Fatal(err)129	}130	var al struct{ Items []alertBody }131	e.viewer.must("GET", "/alerts?open=true&rule_id="+rule.ID, nil, 200).json(t, &al)132	if len(al.Items) != 1 || al.Items[0].State != "firing" || al.Items[0].NodeAlias != "T-ALPHA" || al.Items[0].RuleName != "Load high" {133		t.Fatalf("%+v", al)134	}135	e.admin.must("DELETE", "/alert-rules/"+rule.ID, nil, 204)136	if r := e.viewer.do("GET", "/alert-rules/"+rule.ID, nil); r.Status != 404 {137		t.Fatalf("deleted rule still served: %d", r.Status)138	}139	var det struct {140		Alert    alertBody141		Timeline []struct{ Kind, Body string }142	}143	e.viewer.must("GET", "/alerts/"+al.Items[0].ID, nil, 200).json(t, &det)144	if det.Alert.State != "resolved" || det.Timeline[len(det.Timeline)-1].Body != "rule deleted" {145		t.Fatalf("%+v", det)146	}147	eng.Evaluate(ctx)148	if e.count(`select count(*) from alerts where state <> 'resolved'`) != 0 {149		t.Fatal("a deleted rule still produces alerts")150	}151	e.admin.must("POST", "/alert-rules", draft, 201) // the name is free again152	if n := e.count(`select count(*) from audit_log where action like 'alert_rule.%' and outcome='success'`); n != 5 {153		t.Fatalf("%d audit entries for rule changes", n)154	}155156	// Validation replay endpoint.157	var val struct {158		NodesEvaluated   int `json:"nodes_evaluated"`159		NodesWithoutData int `json:"nodes_without_data"`160		TotalFirings     int `json:"total_firings"`161		Resolution       string162		Intervals        []any163	}164	e.admin.must("POST", "/alert-rules/validate", map[string]any{"rule": draft, "from": now.Add(-time.Hour), "to": now}, 200).json(t, &val)165	if val.NodesEvaluated != 1 || val.NodesWithoutData != 1 || val.TotalFirings != 0 || val.Resolution != "raw" || val.Intervals == nil {166		t.Fatalf("%+v", val)167	}168	if r := e.admin.do("POST", "/alert-rules/validate", map[string]any{"rule": map[string]any{"name": "x", "kind": "absence"}}); r.Status != 422 {169		t.Fatalf("replay of an absence rule: %d %s", r.Status, r.Raw)170	}171	if r := e.admin.do("POST", "/alert-rules/validate", map[string]any{"rule": draft, "from": now.Add(-90 * 24 * time.Hour), "to": now}); r.Status != 400 {172		t.Fatalf("unbounded replay: %d", r.Status)173	}174	if r := e.oper.do("POST", "/alert-rules/validate", map[string]any{"rule": draft}); r.Status != 403 {175		t.Fatalf("operator validate: %d", r.Status)176	}177}178179func TestAlertsSilencesIncidentsAPI(t *testing.T) {180	e := newOpsEnv(t)181	ctx := context.Background()182	var rule ruleBody183	e.admin.must("POST", "/alert-rules", map[string]any{"name": "Load high", "kind": "threshold", "metric": "load.1", "op": "gt", "threshold": 3,184		"for_seconds": 0, "clear_seconds": 0}, 201).json(t, &rule)185	eng := &alerts.Engine{Pool: e.Pool}186	eval := func(load1, load2 float64) {187		now := time.Now().UTC()188		testdb.State(t, e.Pool, e.node, now, map[string]any{"load.1": load1})189		testdb.State(t, e.Pool, e.node2, now, map[string]any{"load.1": load2})190		if err := eng.Evaluate(ctx); err != nil {191			t.Fatal(err)192		}193	}194	eval(9, 9)195	var al struct{ Items []alertBody }196	e.viewer.must("GET", "/alerts?state=firing&node_id="+e.node.String(), nil, 200).json(t, &al)197	if len(al.Items) != 1 {198		t.Fatalf("%+v", al)199	}200	id := al.Items[0].ID201	// Acknowledge / comment / resolve: alerts.ack (operator), not viewer.202	if r := e.viewer.do("POST", "/alerts/"+id+"/acknowledge", nil); r.Status != 403 {203		t.Fatalf("viewer ack: %d", r.Status)204	}205	if r := e.viewer.do("POST", "/alerts/"+id+"/comments", map[string]any{"body": "hi"}); r.Status != 403 {206		t.Fatalf("viewer comment: %d", r.Status)207	}208	if r := e.viewer.do("POST", "/alerts/"+id+"/resolve", nil); r.Status != 403 {209		t.Fatalf("viewer resolve: %d", r.Status)210	}211	var a alertBody212	e.oper.must("POST", "/alerts/"+id+"/acknowledge", map[string]any{"comment": "looking"}, 200).json(t, &a)213	if a.State != "acknowledged" || a.AckBy != "User oscar" {214		t.Fatalf("%+v", a)215	}216	if r := e.oper.do("POST", "/alerts/"+id+"/acknowledge", nil); r.Status != 409 {217		t.Fatalf("ack twice: %d", r.Status)218	}219	if r := e.oper.do("POST", "/alerts/"+id+"/comments", map[string]any{"body": "  "}); r.Status != 400 {220		t.Fatalf("empty comment: %d", r.Status)221	}222	e.oper.must("POST", "/alerts/"+id+"/comments", map[string]any{"body": "runaway indexer"}, 201)223	eval(9, 9) // an acknowledged alert stays acknowledged while the condition holds224	e.viewer.must("GET", "/alerts/"+id, nil, 200).json(t, &struct{ Alert *alertBody }{&a})225	if a.State != "acknowledged" {226		t.Fatalf("%+v", a)227	}228	e.oper.must("POST", "/alerts/"+id+"/resolve", map[string]any{"comment": "killed the indexer"}, 200).json(t, &a)229	if a.State != "resolved" {230		t.Fatalf("%+v", a)231	}232	if r := e.oper.do("POST", "/alerts/"+id+"/resolve", nil); r.Status != 409 {233		t.Fatalf("resolve twice: %d", r.Status)234	}235	var det struct {236		Timeline []struct {237			Kind, Body string238			Actor      string `json:"actor_display_name"`239		}240	}241	e.viewer.must("GET", "/alerts/"+id, nil, 200).json(t, &det)242	var kinds []string243	for _, ev := range det.Timeline {244		kinds = append(kinds, ev.Kind)245	}246	if strings.Join(kinds, ",") != "opened,fired,acknowledged,comment,resolved" || det.Timeline[2].Actor != "User oscar" || det.Timeline[4].Body != "killed the indexer" {247		t.Fatalf("timeline: %+v", det.Timeline)248	}249	// The condition still holds: the rule opens a NEW alert at the next evaluation.250	eval(9, 9)251	e.viewer.must("GET", "/alerts?open=true&node_id="+e.node.String(), nil, 200).json(t, &al)252	if len(al.Items) != 1 || al.Items[0].ID == id {253		t.Fatalf("%+v", al)254	}255	if n := e.count(`select count(*) from audit_log where action in ('alert.acknowledge','alert.comment','alert.resolve') and outcome='success'`); n != 3 {256		t.Fatalf("%d audit entries", n)257	}258259	// Silences.260	end := time.Now().Add(time.Hour).UTC()261	sbody := map[string]any{"reason": "disk swap", "scope": map[string]any{"node_ids": []string{e.node2.String()}}, "ends_at": end}262	if r := e.oper.do("POST", "/silences", sbody); r.Status != 403 {263		t.Fatalf("operator silence: %d", r.Status)264	}265	if r := e.admin.do("POST", "/silences", map[string]any{"reason": "x", "scope": map[string]any{"all": true}, "ends_at": time.Now().Add(-time.Hour)}); r.Status != 400 {266		t.Fatalf("silence in the past: %d", r.Status)267	}268	if r := e.admin.do("POST", "/silences", map[string]any{"reason": "x", "scope": map[string]any{}, "ends_at": end}); r.Status != 400 {269		t.Fatalf("silence without scope: %d", r.Status)270	}271	var s struct {272		ID     string `json:"id"`273		Active bool   `json:"active"`274	}275	e.admin.must("POST", "/silences", sbody, 201).json(t, &s)276	if !s.Active {277		t.Fatalf("%+v", s)278	}279	eval(9, 9)280	e.viewer.must("GET", "/alerts?open=true&node_id="+e.node2.String(), nil, 200).json(t, &al)281	if len(al.Items) != 1 || al.Items[0].Suppressed != "silence" || al.Items[0].State != "firing" {282		t.Fatalf("silenced alert: %+v", al)283	}284	var sl struct{ Items []struct{ ID string } }285	e.viewer.must("GET", "/silences?active=true", nil, 200).json(t, &sl)286	if len(sl.Items) != 1 {287		t.Fatalf("%+v", sl)288	}289	e.admin.must("DELETE", "/silences/"+s.ID, nil, 204)290	if r := e.admin.do("DELETE", "/silences/"+s.ID, nil); r.Status != 404 {291		t.Fatalf("delete twice: %d", r.Status)292	}293	// A silence owned by a maintenance window is not deletable here.294	win, owned := uuid.New(), uuid.New()295	e.Pool.Exec(ctx, `insert into maintenance_windows(id, title, starts_at, ends_at) values ($1,'w',now(),now()+interval '1 hour')`, win)296	e.Pool.Exec(ctx, `insert into silences(id, reason, scope, starts_at, ends_at, maintenance_window_id) values ($1,'w','{"all":true}',now(),now()+interval '1 hour',$2)`, owned, win)297	if r := e.admin.do("DELETE", "/silences/"+owned.String(), nil); r.Status != 409 {298		t.Fatalf("delete of a window silence: %d", r.Status)299	}300	e.Pool.Exec(ctx, `delete from silences`)301302	// Incidents.303	open := al.Items[0].ID304	ibody := map[string]any{"title": "Load storm on the studios", "severity": "critical", "summary": "Both nodes saturated.", "alert_ids": []string{open, id}}305	if r := e.oper.do("POST", "/incidents", ibody); r.Status != 403 {306		t.Fatalf("operator incident: %d", r.Status)307	}308	if r := e.admin.do("POST", "/incidents", map[string]any{"title": "x", "severity": "critical", "alert_ids": []string{uuid.NewString()}}); r.Status != 404 {309		t.Fatalf("incident with unknown alert: %d", r.Status)310	}311	if e.count(`select count(*) from incidents`) != 0 {312		t.Fatal("a refused incident was stored")313	}314	var inc struct {315		ID         string `json:"id"`316		State      string `json:"state"`317		AlertCount int    `json:"alert_count"`318		ResolvedAt string `json:"resolved_at"`319		Owner      string `json:"owner_display_name"`320	}321	e.admin.must("POST", "/incidents", ibody, 201).json(t, &inc)322	if inc.State != "open" || inc.AlertCount != 2 {323		t.Fatalf("%+v", inc)324	}325	e.viewer.must("GET", "/alerts?incident_id="+inc.ID, nil, 200).json(t, &al)326	if len(al.Items) != 2 || al.Items[0].IncidentID != inc.ID {327		t.Fatalf("%+v", al)328	}329	e.oper.must("POST", "/incidents/"+inc.ID+"/comments", map[string]any{"body": "indexer disabled cluster-wide"}, 201)330	if r := e.viewer.do("POST", "/incidents/"+inc.ID+"/comments", map[string]any{"body": "x"}); r.Status != 403 {331		t.Fatalf("viewer incident comment: %d", r.Status)332	}333	var me struct{ User struct{ ID string } }334	e.admin.must("GET", "/me", nil, 200).json(t, &me)335	e.admin.must("PATCH", "/incidents/"+inc.ID, map[string]any{"state": "resolved", "owner_id": me.User.ID, "remove_alert_ids": []string{id}}, 200).json(t, &inc)336	if inc.State != "resolved" || inc.ResolvedAt == "" || inc.AlertCount != 1 || inc.Owner != "User alice" {337		t.Fatalf("%+v", inc)338	}339	if r := e.admin.do("PATCH", "/incidents/"+inc.ID, map[string]any{"owner_id": uuid.NewString()}); r.Status != 400 {340		t.Fatalf("unknown owner: %d", r.Status)341	}342	var idet struct {343		Incident struct{ State string }344		Timeline []struct{ Kind string }345		Alerts   []alertBody346	}347	e.viewer.must("GET", "/incidents/"+inc.ID, nil, 200).json(t, &idet)348	kinds = nil349	for _, ev := range idet.Timeline {350		kinds = append(kinds, ev.Kind)351	}352	if strings.Join(kinds, ",") != "opened,alert_linked,alert_linked,comment,state_changed,owner_changed,alert_unlinked" || len(idet.Alerts) != 1 {353		t.Fatalf("incident detail: %v %d", kinds, len(idet.Alerts))354	}355	var il struct{ Items []struct{ ID string } }356	e.viewer.must("GET", "/incidents?state=resolved&severity=critical", nil, 200).json(t, &il)357	if len(il.Items) != 1 {358		t.Fatalf("%+v", il)359	}360	if e.count(`select count(*) from events_outbox where topic='incident.changed'`) != 2 {361		t.Fatal("incident.changed events missing")362	}363	if n := e.count(`select count(*) from audit_log where action like 'incident.%' or action like 'silence.%'`); n != 5 {364		t.Fatalf("%d audit entries for incidents and silences", n)365	}366}367
Files main
Menu

Theme