Repository Public
1package api_test23import (4 "context"5 "strings"6 "testing"7 "time"89 "github.com/google/uuid"1011 "clustr.spboucher.ai/mcc/internal/alerts"12 "clustr.spboucher.ai/mcc/internal/testdb"13)1415type alertBody struct {16 ID string `json:"id"`17 State string `json:"state"`18 RuleName string `json:"rule_name"`19 NodeAlias string `json:"node_alias"`20 AckBy string `json:"acknowledged_by_display_name"`21 IncidentID string `json:"incident_id"`22 Suppressed string `json:"suppressed_by"`23}2425type ruleBody struct {26 ID string `json:"id"`27 Name string `json:"name"`28 Enabled bool `json:"enabled"`29 Builtin bool `json:"builtin"`30 Op string `json:"op"`31 Threshold *float64 `json:"threshold"`32 For int `json:"for_seconds"`33 Clear int `json:"clear_seconds"`34 Severity string `json:"severity"`35 Notify bool `json:"notify"`36}3738func TestAlertRulesAPI(t *testing.T) {39 e := newOpsEnv(t)40 ctx := context.Background()41 if err := alerts.SeedBuiltins(ctx, e.Pool); err != nil {42 t.Fatal(err)43 }44 var list struct{ Items []ruleBody }45 e.viewer.must("GET", "/alert-rules?limit=200", nil, 200).json(t, &list)46 if len(list.Items) != 12 {47 t.Fatalf("%d rules", len(list.Items))48 }49 var cpuRule ruleBody50 for _, r := range list.Items {51 if !r.Builtin || !r.Enabled {52 t.Fatalf("%+v", r)53 }54 if r.Name == "CPU saturated" {55 cpuRule = r56 }57 }58 if cpuRule.Op != "gt" || *cpuRule.Threshold != 95 || cpuRule.For != 900 {59 t.Fatalf("%+v", cpuRule)60 }61 draft := map[string]any{"name": "Load high", "kind": "threshold", "metric": "load.1", "op": "gte", "threshold": 20, "for_seconds": 0, "clear_seconds": 0,62 "severity": "critical", "scope": map[string]any{"node_ids": []string{e.node.String()}}}63 // Permissions and validation.64 if r := e.oper.do("POST", "/alert-rules", draft); r.Status != 403 {65 t.Fatalf("operator create rule: %d", r.Status)66 }67 for name, mut := range map[string]func(m map[string]any){68 "unknown metric": func(m map[string]any) { m["metric"] = "nope" },69 "no threshold": func(m map[string]any) { delete(m, "threshold") },70 "unknown node": func(m map[string]any) { m["scope"] = map[string]any{"node_ids": []string{uuid.NewString()}} },71 "two scope forms": func(m map[string]any) { m["scope"] = map[string]any{"all": true, "tags": []string{"x"}} },72 "negative for": func(m map[string]any) { m["for_seconds"] = -5 },73 "clear wrong side": func(m map[string]any) { m["clear_threshold"] = 30 },74 } {75 m := map[string]any{}76 for k, v := range draft {77 m[k] = v78 }79 mut(m)80 if r := e.admin.do("POST", "/alert-rules", m); r.Status != 400 {81 t.Errorf("%s: %d %s", name, r.Status, r.Raw)82 }83 }84 var rule ruleBody85 e.admin.must("POST", "/alert-rules", draft, 201).json(t, &rule)86 if rule.Builtin || !rule.Enabled || rule.Op != "gte" || rule.Severity != "critical" || !rule.Notify {87 t.Fatalf("%+v", rule)88 }89 if r := e.admin.do("POST", "/alert-rules", draft); r.Status != 409 {90 t.Fatalf("duplicate name: %d", r.Status)91 }92 e.admin.must("PATCH", "/alert-rules/"+rule.ID, map[string]any{"threshold": 10, "notify": false}, 200).json(t, &rule)93 if *rule.Threshold != 10 || rule.Notify {94 t.Fatalf("%+v", rule)95 }96 if r := e.admin.do("PATCH", "/alert-rules/"+rule.ID, map[string]any{"metric": "nope"}); r.Status != 400 {97 t.Fatalf("patch to an invalid rule: %d", r.Status)98 }99 // Built-in rules: tunable and disablable, not renamable, not deletable.100 if r := e.admin.do("PATCH", "/alert-rules/"+cpuRule.ID, map[string]any{"name": "mine"}); r.Status != 409 {101 t.Fatalf("rename builtin: %d", r.Status)102 }103 if r := e.admin.do("PATCH", "/alert-rules/"+cpuRule.ID, map[string]any{"metric": "load.1"}); r.Status != 409 {104 t.Fatalf("change metric of builtin: %d", r.Status)105 }106 e.admin.must("PATCH", "/alert-rules/"+cpuRule.ID, map[string]any{"enabled": false, "threshold": 99, "for_seconds": 60}, 200).json(t, &cpuRule)107 if cpuRule.Enabled || *cpuRule.Threshold != 99 {108 t.Fatalf("%+v", cpuRule)109 }110 if r := e.admin.do("DELETE", "/alert-rules/"+cpuRule.ID, nil); r.Status != 409 {111 t.Fatalf("delete builtin: %d", r.Status)112 }113 alerts.SeedBuiltins(ctx, e.Pool) // "restart"114 e.viewer.must("GET", "/alert-rules/"+cpuRule.ID, nil, 200).json(t, &cpuRule)115 if cpuRule.Enabled || *cpuRule.Threshold != 99 || cpuRule.For != 60 {116 t.Fatalf("restart overwrote an edited builtin: %+v", cpuRule)117 }118 e.viewer.must("GET", "/alert-rules?enabled=false", nil, 200).json(t, &list)119 if len(list.Items) != 1 {120 t.Fatalf("enabled=false: %d", len(list.Items))121 }122123 // Deleting a rule resolves what it had open and keeps the history.124 now := time.Now().UTC()125 testdb.State(t, e.Pool, e.node, now, map[string]any{"load.1": 50.0})126 eng := &alerts.Engine{Pool: e.Pool}127 if err := eng.Evaluate(ctx); err != nil {128 t.Fatal(err)129 }130 var al struct{ Items []alertBody }131 e.viewer.must("GET", "/alerts?open=true&rule_id="+rule.ID, nil, 200).json(t, &al)132 if len(al.Items) != 1 || al.Items[0].State != "firing" || al.Items[0].NodeAlias != "T-ALPHA" || al.Items[0].RuleName != "Load high" {133 t.Fatalf("%+v", al)134 }135 e.admin.must("DELETE", "/alert-rules/"+rule.ID, nil, 204)136 if r := e.viewer.do("GET", "/alert-rules/"+rule.ID, nil); r.Status != 404 {137 t.Fatalf("deleted rule still served: %d", r.Status)138 }139 var det struct {140 Alert alertBody141 Timeline []struct{ Kind, Body string }142 }143 e.viewer.must("GET", "/alerts/"+al.Items[0].ID, nil, 200).json(t, &det)144 if det.Alert.State != "resolved" || det.Timeline[len(det.Timeline)-1].Body != "rule deleted" {145 t.Fatalf("%+v", det)146 }147 eng.Evaluate(ctx)148 if e.count(`select count(*) from alerts where state <> 'resolved'`) != 0 {149 t.Fatal("a deleted rule still produces alerts")150 }151 e.admin.must("POST", "/alert-rules", draft, 201) // the name is free again152 if n := e.count(`select count(*) from audit_log where action like 'alert_rule.%' and outcome='success'`); n != 5 {153 t.Fatalf("%d audit entries for rule changes", n)154 }155156 // Validation replay endpoint.157 var val struct {158 NodesEvaluated int `json:"nodes_evaluated"`159 NodesWithoutData int `json:"nodes_without_data"`160 TotalFirings int `json:"total_firings"`161 Resolution string162 Intervals []any163 }164 e.admin.must("POST", "/alert-rules/validate", map[string]any{"rule": draft, "from": now.Add(-time.Hour), "to": now}, 200).json(t, &val)165 if val.NodesEvaluated != 1 || val.NodesWithoutData != 1 || val.TotalFirings != 0 || val.Resolution != "raw" || val.Intervals == nil {166 t.Fatalf("%+v", val)167 }168 if r := e.admin.do("POST", "/alert-rules/validate", map[string]any{"rule": map[string]any{"name": "x", "kind": "absence"}}); r.Status != 422 {169 t.Fatalf("replay of an absence rule: %d %s", r.Status, r.Raw)170 }171 if r := e.admin.do("POST", "/alert-rules/validate", map[string]any{"rule": draft, "from": now.Add(-90 * 24 * time.Hour), "to": now}); r.Status != 400 {172 t.Fatalf("unbounded replay: %d", r.Status)173 }174 if r := e.oper.do("POST", "/alert-rules/validate", map[string]any{"rule": draft}); r.Status != 403 {175 t.Fatalf("operator validate: %d", r.Status)176 }177}178179func TestAlertsSilencesIncidentsAPI(t *testing.T) {180 e := newOpsEnv(t)181 ctx := context.Background()182 var rule ruleBody183 e.admin.must("POST", "/alert-rules", map[string]any{"name": "Load high", "kind": "threshold", "metric": "load.1", "op": "gt", "threshold": 3,184 "for_seconds": 0, "clear_seconds": 0}, 201).json(t, &rule)185 eng := &alerts.Engine{Pool: e.Pool}186 eval := func(load1, load2 float64) {187 now := time.Now().UTC()188 testdb.State(t, e.Pool, e.node, now, map[string]any{"load.1": load1})189 testdb.State(t, e.Pool, e.node2, now, map[string]any{"load.1": load2})190 if err := eng.Evaluate(ctx); err != nil {191 t.Fatal(err)192 }193 }194 eval(9, 9)195 var al struct{ Items []alertBody }196 e.viewer.must("GET", "/alerts?state=firing&node_id="+e.node.String(), nil, 200).json(t, &al)197 if len(al.Items) != 1 {198 t.Fatalf("%+v", al)199 }200 id := al.Items[0].ID201 // Acknowledge / comment / resolve: alerts.ack (operator), not viewer.202 if r := e.viewer.do("POST", "/alerts/"+id+"/acknowledge", nil); r.Status != 403 {203 t.Fatalf("viewer ack: %d", r.Status)204 }205 if r := e.viewer.do("POST", "/alerts/"+id+"/comments", map[string]any{"body": "hi"}); r.Status != 403 {206 t.Fatalf("viewer comment: %d", r.Status)207 }208 if r := e.viewer.do("POST", "/alerts/"+id+"/resolve", nil); r.Status != 403 {209 t.Fatalf("viewer resolve: %d", r.Status)210 }211 var a alertBody212 e.oper.must("POST", "/alerts/"+id+"/acknowledge", map[string]any{"comment": "looking"}, 200).json(t, &a)213 if a.State != "acknowledged" || a.AckBy != "User oscar" {214 t.Fatalf("%+v", a)215 }216 if r := e.oper.do("POST", "/alerts/"+id+"/acknowledge", nil); r.Status != 409 {217 t.Fatalf("ack twice: %d", r.Status)218 }219 if r := e.oper.do("POST", "/alerts/"+id+"/comments", map[string]any{"body": " "}); r.Status != 400 {220 t.Fatalf("empty comment: %d", r.Status)221 }222 e.oper.must("POST", "/alerts/"+id+"/comments", map[string]any{"body": "runaway indexer"}, 201)223 eval(9, 9) // an acknowledged alert stays acknowledged while the condition holds224 e.viewer.must("GET", "/alerts/"+id, nil, 200).json(t, &struct{ Alert *alertBody }{&a})225 if a.State != "acknowledged" {226 t.Fatalf("%+v", a)227 }228 e.oper.must("POST", "/alerts/"+id+"/resolve", map[string]any{"comment": "killed the indexer"}, 200).json(t, &a)229 if a.State != "resolved" {230 t.Fatalf("%+v", a)231 }232 if r := e.oper.do("POST", "/alerts/"+id+"/resolve", nil); r.Status != 409 {233 t.Fatalf("resolve twice: %d", r.Status)234 }235 var det struct {236 Timeline []struct {237 Kind, Body string238 Actor string `json:"actor_display_name"`239 }240 }241 e.viewer.must("GET", "/alerts/"+id, nil, 200).json(t, &det)242 var kinds []string243 for _, ev := range det.Timeline {244 kinds = append(kinds, ev.Kind)245 }246 if strings.Join(kinds, ",") != "opened,fired,acknowledged,comment,resolved" || det.Timeline[2].Actor != "User oscar" || det.Timeline[4].Body != "killed the indexer" {247 t.Fatalf("timeline: %+v", det.Timeline)248 }249 // The condition still holds: the rule opens a NEW alert at the next evaluation.250 eval(9, 9)251 e.viewer.must("GET", "/alerts?open=true&node_id="+e.node.String(), nil, 200).json(t, &al)252 if len(al.Items) != 1 || al.Items[0].ID == id {253 t.Fatalf("%+v", al)254 }255 if n := e.count(`select count(*) from audit_log where action in ('alert.acknowledge','alert.comment','alert.resolve') and outcome='success'`); n != 3 {256 t.Fatalf("%d audit entries", n)257 }258259 // Silences.260 end := time.Now().Add(time.Hour).UTC()261 sbody := map[string]any{"reason": "disk swap", "scope": map[string]any{"node_ids": []string{e.node2.String()}}, "ends_at": end}262 if r := e.oper.do("POST", "/silences", sbody); r.Status != 403 {263 t.Fatalf("operator silence: %d", r.Status)264 }265 if r := e.admin.do("POST", "/silences", map[string]any{"reason": "x", "scope": map[string]any{"all": true}, "ends_at": time.Now().Add(-time.Hour)}); r.Status != 400 {266 t.Fatalf("silence in the past: %d", r.Status)267 }268 if r := e.admin.do("POST", "/silences", map[string]any{"reason": "x", "scope": map[string]any{}, "ends_at": end}); r.Status != 400 {269 t.Fatalf("silence without scope: %d", r.Status)270 }271 var s struct {272 ID string `json:"id"`273 Active bool `json:"active"`274 }275 e.admin.must("POST", "/silences", sbody, 201).json(t, &s)276 if !s.Active {277 t.Fatalf("%+v", s)278 }279 eval(9, 9)280 e.viewer.must("GET", "/alerts?open=true&node_id="+e.node2.String(), nil, 200).json(t, &al)281 if len(al.Items) != 1 || al.Items[0].Suppressed != "silence" || al.Items[0].State != "firing" {282 t.Fatalf("silenced alert: %+v", al)283 }284 var sl struct{ Items []struct{ ID string } }285 e.viewer.must("GET", "/silences?active=true", nil, 200).json(t, &sl)286 if len(sl.Items) != 1 {287 t.Fatalf("%+v", sl)288 }289 e.admin.must("DELETE", "/silences/"+s.ID, nil, 204)290 if r := e.admin.do("DELETE", "/silences/"+s.ID, nil); r.Status != 404 {291 t.Fatalf("delete twice: %d", r.Status)292 }293 // A silence owned by a maintenance window is not deletable here.294 win, owned := uuid.New(), uuid.New()295 e.Pool.Exec(ctx, `insert into maintenance_windows(id, title, starts_at, ends_at) values ($1,'w',now(),now()+interval '1 hour')`, win)296 e.Pool.Exec(ctx, `insert into silences(id, reason, scope, starts_at, ends_at, maintenance_window_id) values ($1,'w','{"all":true}',now(),now()+interval '1 hour',$2)`, owned, win)297 if r := e.admin.do("DELETE", "/silences/"+owned.String(), nil); r.Status != 409 {298 t.Fatalf("delete of a window silence: %d", r.Status)299 }300 e.Pool.Exec(ctx, `delete from silences`)301302 // Incidents.303 open := al.Items[0].ID304 ibody := map[string]any{"title": "Load storm on the studios", "severity": "critical", "summary": "Both nodes saturated.", "alert_ids": []string{open, id}}305 if r := e.oper.do("POST", "/incidents", ibody); r.Status != 403 {306 t.Fatalf("operator incident: %d", r.Status)307 }308 if r := e.admin.do("POST", "/incidents", map[string]any{"title": "x", "severity": "critical", "alert_ids": []string{uuid.NewString()}}); r.Status != 404 {309 t.Fatalf("incident with unknown alert: %d", r.Status)310 }311 if e.count(`select count(*) from incidents`) != 0 {312 t.Fatal("a refused incident was stored")313 }314 var inc struct {315 ID string `json:"id"`316 State string `json:"state"`317 AlertCount int `json:"alert_count"`318 ResolvedAt string `json:"resolved_at"`319 Owner string `json:"owner_display_name"`320 }321 e.admin.must("POST", "/incidents", ibody, 201).json(t, &inc)322 if inc.State != "open" || inc.AlertCount != 2 {323 t.Fatalf("%+v", inc)324 }325 e.viewer.must("GET", "/alerts?incident_id="+inc.ID, nil, 200).json(t, &al)326 if len(al.Items) != 2 || al.Items[0].IncidentID != inc.ID {327 t.Fatalf("%+v", al)328 }329 e.oper.must("POST", "/incidents/"+inc.ID+"/comments", map[string]any{"body": "indexer disabled cluster-wide"}, 201)330 if r := e.viewer.do("POST", "/incidents/"+inc.ID+"/comments", map[string]any{"body": "x"}); r.Status != 403 {331 t.Fatalf("viewer incident comment: %d", r.Status)332 }333 var me struct{ User struct{ ID string } }334 e.admin.must("GET", "/me", nil, 200).json(t, &me)335 e.admin.must("PATCH", "/incidents/"+inc.ID, map[string]any{"state": "resolved", "owner_id": me.User.ID, "remove_alert_ids": []string{id}}, 200).json(t, &inc)336 if inc.State != "resolved" || inc.ResolvedAt == "" || inc.AlertCount != 1 || inc.Owner != "User alice" {337 t.Fatalf("%+v", inc)338 }339 if r := e.admin.do("PATCH", "/incidents/"+inc.ID, map[string]any{"owner_id": uuid.NewString()}); r.Status != 400 {340 t.Fatalf("unknown owner: %d", r.Status)341 }342 var idet struct {343 Incident struct{ State string }344 Timeline []struct{ Kind string }345 Alerts []alertBody346 }347 e.viewer.must("GET", "/incidents/"+inc.ID, nil, 200).json(t, &idet)348 kinds = nil349 for _, ev := range idet.Timeline {350 kinds = append(kinds, ev.Kind)351 }352 if strings.Join(kinds, ",") != "opened,alert_linked,alert_linked,comment,state_changed,owner_changed,alert_unlinked" || len(idet.Alerts) != 1 {353 t.Fatalf("incident detail: %v %d", kinds, len(idet.Alerts))354 }355 var il struct{ Items []struct{ ID string } }356 e.viewer.must("GET", "/incidents?state=resolved&severity=critical", nil, 200).json(t, &il)357 if len(il.Items) != 1 {358 t.Fatalf("%+v", il)359 }360 if e.count(`select count(*) from events_outbox where topic='incident.changed'`) != 2 {361 t.Fatal("incident.changed events missing")362 }363 if n := e.count(`select count(*) from audit_log where action like 'incident.%' or action like 'silence.%'`); n != 5 {364 t.Fatalf("%d audit entries for incidents and silences", n)365 }366}367