Repository Public
1package api_test23import (4 "context"5 "fmt"6 "net/netip"7 "net/url"8 "strings"9 "testing"10 "time"1112 "github.com/google/uuid"1314 "clustr.spboucher.ai/mcc/internal/audit"15 "clustr.spboucher.ai/mcc/internal/ids"16)1718func TestServicesMappingHistoryAndUpdate(t *testing.T) {19 h := newHarness(t)20 root := h.owner()21 admin := h.member(root, "admin", "ada", false)22 viewer := h.member(root, "viewer", "vera", false)23 a := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second})24 b := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Hour, Kind: "linux"})25 web := h.seedService(a, "pm2", "web", "online")26 h.dbExec(`update services set app = 'shop', public_url = 'https://shop.example', health = '{"status":"ok","latency_ms":12.5,"detail":"200"}' where id = $1`, web)27 h.seedService(a, "launchd", "io.example.worker", "errored")28 h.seedService(a, "pm2", "Zeta", "weird-state")29 old := h.seedService(a, "pm2", "old", "online")30 db := h.seedService(b, "systemd", "postgresql", "online")31 h.seedService(b, "docker", "cache", "stopped")32 retired := h.seedNode(nodeSeed{Alias: "gone", Seen: true, Age: time.Second})33 h.seedService(retired, "pm2", "hidden", "online")34 h.dbExec(`update nodes set retired_at = now() where id = $1`, retired)3536 // Observed changes, as ingestion would write them: a crash, two restarts, a version change, then the service disappears.37 h.dbExec(`update services set state = 'errored', last_seen_at = now() + interval '1 second' where id = $1`, old)38 h.dbExec(`update services set state = 'online', restarts = 5, last_seen_at = now() + interval '2 seconds' where id = $1`, old)39 h.dbExec(`update services set version = '1.3.0', last_seen_at = now() + interval '3 seconds' where id = $1`, old)40 h.dbExec(`update services set last_seen_at = now() + interval '4 seconds' where id = $1`, old) // nothing changed: nothing recorded41 h.dbExec(`update services set gone_at = now() + interval '5 seconds' where id = $1`, old)4243 all := []string{"cache", "io.example.worker", "old", "postgresql", "web", "Zeta"}44 for _, limit := range []int{1, 2, 50} {45 if got := viewer.walkPages("/services", limit, "name"); strings.Join(got, ",") != strings.Join(all, ",") {46 t.Fatalf("limit %d: %v", limit, got)47 }48 }49 rows := viewer.must("GET", "/services", nil, 200).jsonItems(t)50 w := rowBy(rows, "name", "web")51 if w["node_alias"] != "alpha" || w["manager"] != "pm2" || w["state"] != "online" || w["pid"] != 4242.0 || w["restarts"] != 3.0 ||52 w["mem_bytes"] != 104857600.0 || w["cpu_pct"] != 1.5 || w["uptime_s"] != 3600.0 || w["version"] != "1.2.0" || fmt.Sprint(w["ports"]) != "[8080 8443]" ||53 w["app"] != "shop" || w["public_url"] != "https://shop.example" || w["managed"] != false || w["freshness"] != "fresh" || w["gone_at"] != nil ||54 dig(w, "health", "status") != "ok" || dig(w, "health", "latency_ms") != 12.5 || w["meta"] != nil {55 t.Fatalf("service mapping: %v", w)56 }57 if o := rowBy(rows, "name", "old"); o["gone_at"] == nil || o["version"] != "1.3.0" {58 t.Fatalf("a service that is gone is still listed, marked as gone: %v", o)59 }60 if z := rowBy(rows, "name", "Zeta"); z["state"] != "unknown" {61 t.Fatalf("an unrecognized state is unknown: %v", z)62 }63 if p := rowBy(rows, "name", "postgresql"); p["freshness"] != "offline" || p["state"] != "online" {64 t.Fatalf("last known state with the node's freshness: %v", p)65 }66 for f, want := range map[string][]string{67 "node_id=" + b: {"cache", "postgresql"},68 "manager=launchd": {"io.example.worker"},69 "state=online": {"postgresql", "web"}, // "old" is gone: it is in no current state70 "state=unknown": {"Zeta"},71 "app=SHOP": {"web"},72 "managed=true": {},73 "q=EXAMPLE": {"io.example.worker"},74 "q=shop": {"web"},75 "q=" + url.QueryEscape("%"): {},76 "state=online&node_id=" + a: {"web"},77 } {78 wantExactlyOnce(t, f, viewer.walkPages("/services?"+f, 2, "name"), want...)79 }80 for _, bad := range []string{"manager=cron", "state=dead", "limit=0", "cursor=zzz", "node_id=nope"} {81 if r := viewer.do("GET", "/services?"+bad, nil); r.Status != 400 {82 t.Fatalf("%s: %d %s", bad, r.Status, r.Raw)83 }84 }85 wantExactlyOnce(t, "node services", viewer.walkPages("/nodes/"+b+"/services", 1, "name"), "cache", "postgresql")86 if r := viewer.do("GET", "/nodes/"+ids.New().String()+"/services", nil); r.Status != 404 {87 t.Fatalf("services of an unknown node: %d", r.Status)88 }8990 // Single view: meta is included, with secret-looking members masked.91 one := viewer.must("GET", "/services/"+web, nil, 200).jsonObj(t)92 if dig(one, "meta", "raw_state") != "online" || dig(one, "meta", "api_token") != "[redacted]" || strings.Contains(fmt.Sprint(one), "s3cr3t") {93 t.Fatalf("meta: %v", one["meta"])94 }95 if r := viewer.do("GET", "/services/"+ids.New().String(), nil); r.Status != 404 {96 t.Fatalf("unknown service: %d", r.Status)97 }9899 // History: only recorded facts, most recent first.100 h.dbExec(`insert into actions(id, type, criticality, node_id, target_kind, target_id, state, requested_by, idempotency_key, request_hash, timeout_s,101 correlation_id, created_at) values ($1, 'service.restart', 'medium', $2, 'service', $3, 'succeeded', $4, 'k', '\x00', 60, $5, now() + interval '10 seconds')`,102 ids.New().String(), a, old, h.userIDOf("ada"), ids.New().String())103 hist := viewer.must("GET", "/services/"+old+"/history", nil, 200).jsonItems(t)104 var kinds []string105 for _, e := range hist {106 kinds = append(kinds, e["kind"].(string))107 }108 if strings.Join(kinds, " ") != "action disappeared deploy restart state_change state_change appeared" {109 t.Fatalf("history: %v", kinds)110 }111 if e := hist[0]; e["summary"] != "Action service.restart: succeeded." || e["actor_display_name"] != "User ada" || e["action_id"] == nil {112 t.Fatalf("action entry: %v", e)113 }114 if e := hist[2]; e["version"] != "1.3.0" || !strings.Contains(e["summary"].(string), "1.2.0 to 1.3.0") {115 t.Fatalf("deploy entry: %v", e)116 }117 if e := hist[3]; e["summary"] != "Restart counter went from 3 to 5." {118 t.Fatalf("restart entry: %v", e)119 }120 if e := hist[5]; e["from_state"] != "online" || e["to_state"] != "errored" {121 t.Fatalf("state change entry: %v", e)122 }123 if got := viewer.walkPages("/services/"+old+"/history", 2, "kind"); strings.Join(got, " ") != strings.Join(kinds, " ") {124 t.Fatalf("paged history: %v", got)125 }126 if got := viewer.walkPages("/services/"+db+"/history", 5, "kind"); strings.Join(got, " ") != "appeared" {127 t.Fatalf("history of a quiet service: %v", got)128 }129 // Without actions.read the action entries are left out.130 root.grant("user", h.userIDOf("vera"), "actions.read", "", "", "deny")131 if got := viewer.walkPages("/services/"+old+"/history", 50, "kind"); len(got) != 6 || got[0] != "disappeared" {132 t.Fatalf("history without actions.read: %v", got)133 }134135 // Update: managed flag and public URL, audited, announced.136 if r := viewer.do("PATCH", "/services/"+web, map[string]any{"managed": true}); r.Status != 403 {137 t.Fatalf("viewer patch: %d", r.Status)138 }139 for name, bad := range map[string]map[string]any{"nothing": {}, "scheme": {"public_url": "ftp://x"}, "relative": {"public_url": "/path"},140 "credentials": {"public_url": "https://user:pw@example.com"}, "long": {"public_url": "https://example.com/" + strings.Repeat("a", 300)}} {141 if r := admin.do("PATCH", "/services/"+web, bad); r.Status != 400 {142 t.Fatalf("patch %s: %d %s", name, r.Status, r.Raw)143 }144 }145 u := admin.must("PATCH", "/services/"+web, map[string]any{"managed": true, "public_url": ""}, 200).jsonObj(t)146 if u["managed"] != true || u["public_url"] != nil {147 t.Fatalf("patched service: %v", u)148 }149 admin.must("PATCH", "/services/"+web, map[string]any{"managed": true}, 200) // no change150 if h.count(`select count(*) from audit_log where action = 'service.update' and target_id = $1 and params->'changes'->'managed'->>'to' = 'true'`, web) != 1 ||151 h.count(`select count(*) from audit_log where action = 'service.update'`) != 1 {152 t.Fatal("service update must be audited once")153 }154 if h.count(`select count(*) from events_outbox where topic = 'service.changed' and payload->>'service_id' = $1 and payload->>'state' = 'online'155 and payload->>'manager' = 'pm2' and payload->>'name' = 'web' and payload->>'node_id' = $2`, web, a) != 1 {156 t.Fatal("service.changed event missing")157 }158 wantExactlyOnce(t, "managed=true", viewer.walkPages("/services?managed=true", 10, "name"), "web")159 if h.count(`select count(*) from service_events where service_id = $1`, web) != 1 {160 t.Fatal("an administrative change is not an observed change of the service")161 }162}163164func TestModelsVolumesNasAndBackups(t *testing.T) {165 h := newHarness(t)166 root := h.owner()167 admin := h.member(root, "admin", "ada", false)168 operator := h.member(root, "operator", "oscar", false)169 viewer := h.member(root, "viewer", "vera", false)170 a := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second})171 b := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Minute})172173 // Inference servers.174 mlx := h.seedModel(a, "mlx_lm", 8081, "mlx-community/Qwen2.5-7B", "ok")175 h.seedModel(a, "ollama", 11434, "llama3", "failing")176 goneModel := h.seedModel(b, "llama.cpp", 8080, "phi", "ok")177 h.dbExec(`update inference_servers set gone_at = now() where id = $1`, goneModel)178 h.seedModel(b, "mlx_lm", 9000, "", "strange")179 if got := viewer.walkPages("/models", 1, "port"); strings.Join(got, ",") != "8081,11434,8080,9000" {180 t.Fatalf("models order (node alias, port): %v", got)181 }182 ms := viewer.must("GET", "/models", nil, 200).jsonItems(t)183 m := rowBy(ms, "id", mlx)184 if m["node_alias"] != "alpha" || m["kind"] != "mlx_lm" || m["model"] != "mlx-community/Qwen2.5-7B" || m["quantization"] != "4bit" || m["health"] != "ok" ||185 m["rss_bytes"] != 5368709120.0 || m["health_latency_ms"] != 4.2 || fmt.Sprint(m["listed_models"]) != "[mlx-community/Qwen2.5-7B]" || m["freshness"] != "fresh" {186 t.Fatalf("model mapping: %v", m)187 }188 for _, family := range []string{"requests", "tokens_per_s", "ttft", "latency", "queue"} {189 if dig(m, "metrics_support", family, "availability") != "unsupported" || !strings.Contains(fmt.Sprint(dig(m, "metrics_support", family, "reason")), "mlx_lm.server") {190 t.Fatalf("metrics_support.%s must be explicitly unsupported: %v", family, m["metrics_support"])191 }192 }193 if o := rowBy(ms, "port", "11434"); !strings.Contains(fmt.Sprint(dig(o, "metrics_support", "requests", "reason")), "not collected") {194 t.Fatalf("other kinds: %v", o["metrics_support"])195 }196 if g := rowBy(ms, "id", goneModel); g["gone_at"] == nil || g["freshness"] != "stale" {197 t.Fatalf("gone model: %v", g)198 }199 if s := rowBy(ms, "port", "9000"); s["health"] != "unknown" || s["model"] != nil {200 t.Fatalf("unrecognized health is unknown, empty model is absent: %v", s)201 }202 wantExactlyOnce(t, "health=ok", viewer.walkPages("/models?health=ok", 5, "port"), "8081") // the gone one is not "ok"203 wantExactlyOnce(t, "health=unknown", viewer.walkPages("/models?health=unknown", 5, "port"), "9000")204 wantExactlyOnce(t, "node", viewer.walkPages("/models?node_id="+b, 5, "port"), "8080", "9000")205 if r := viewer.do("GET", "/models?health=great", nil); r.Status != 400 {206 t.Fatalf("bad health: %d", r.Status)207 }208 if one := viewer.must("GET", "/models/"+mlx, nil, 200).jsonObj(t); one["port"] != 8081.0 {209 t.Fatalf("get model: %v", one)210 }211 if r := viewer.do("GET", "/models/"+ids.New().String(), nil); r.Status != 404 {212 t.Fatalf("unknown model: %d", r.Status)213 }214215 // Volumes.216 h.seedVolume(a, "/", "/dev/disk3s1", "apfs", "internal", 1000, 900, false)217 h.seedVolume(a, "/Volumes/T7", "/dev/disk5s1", "exfat", "external", 2000, 100, false)218 h.seedVolume(a, "/Volumes/Old", "/dev/disk9s1", "apfs", "external", 500, 499, true)219 h.seedVolume(b, "/mnt/nas", "//192.168.2.173/share", "smbfs", "network", 4000, 2000, false)220 unk := h.seedVolume(b, "/mnt/x", "/dev/sdx", "ext4", "internal", 0, 0, false)221 h.dbExec(`update volumes set total_bytes = null, used_bytes = null, smart_status = null where id = $1`, unk)222 if got := viewer.walkPages("/storage/volumes", 2, "mount"); strings.Join(got, ",") != "/,/Volumes/Old,/Volumes/T7,/mnt/nas,/mnt/x" {223 t.Fatalf("volumes order: %v", got)224 }225 vs := viewer.must("GET", "/storage/volumes", nil, 200).jsonItems(t)226 if v := rowBy(vs, "mount", "/"); v["node_alias"] != "alpha" || v["kind"] != "internal" || v["total_bytes"] != 1000.0 || v["used_bytes"] != 900.0 ||227 v["smart_status"] != "Verified" || v["fs"] != "apfs" || v["device"] != "/dev/disk3s1" || v["freshness"] != "fresh" {228 t.Fatalf("volume mapping: %v", v)229 }230 if v := rowBy(vs, "mount", "/Volumes/Old"); v["gone_at"] == nil {231 t.Fatalf("gone volume: %v", v)232 }233 if v := rowBy(vs, "mount", "/mnt/x"); v["total_bytes"] != nil || v["used_bytes"] != nil || v["smart_status"] != nil {234 t.Fatalf("what was not observed is absent, not zero: %v", v)235 }236 wantExactlyOnce(t, "min_used_pct", viewer.walkPages("/storage/volumes?min_used_pct=50", 5, "mount"), "/", "/Volumes/Old", "/mnt/nas")237 wantExactlyOnce(t, "kind", viewer.walkPages("/storage/volumes?kind=external", 5, "mount"), "/Volumes/T7", "/Volumes/Old")238 wantExactlyOnce(t, "node", viewer.walkPages("/storage/volumes?node_id="+b, 5, "mount"), "/mnt/nas", "/mnt/x")239 for _, bad := range []string{"kind=tape", "min_used_pct=101", "min_used_pct=-1"} {240 if r := viewer.do("GET", "/storage/volumes?"+bad, nil); r.Status != 400 {241 t.Fatalf("%s: %d", bad, r.Status)242 }243 }244 if d := viewer.must("GET", "/nodes/"+a, nil, 200).jsonObj(t); len(d["volumes"].([]any)) != 2 {245 t.Fatalf("node detail lists current volumes only: %v", d["volumes"])246 }247248 // NAS targets.249 nas := map[string]any{"name": "nas-1", "host": "192.168.2.173", "share": "clustr", "protocol": "smb", "note": "rack 2"}250 if r := operator.do("POST", "/storage/nas", nas); r.Status != 403 {251 t.Fatalf("NAS registration without settings.manage: %d", r.Status)252 }253 for name, bad := range map[string]map[string]any{"host name": {"name": "x", "host": "nas.local"}, "empty name": {"name": " ", "host": "10.0.0.1"},254 "protocol": {"name": "x", "host": "10.0.0.1", "protocol": "ftp"}, "note": {"name": "x", "host": "10.0.0.1", "note": strings.Repeat("n", 501)},255 "unspecified": {"name": "x", "host": "0.0.0.0"}} {256 if r := admin.do("POST", "/storage/nas", bad); r.Status != 400 {257 t.Fatalf("NAS %s: %d %s", name, r.Status, r.Raw)258 }259 }260 created := admin.must("POST", "/storage/nas", nas, 201).jsonObj(t)261 nasID := created["id"].(string)262 if created["state"] != "unknown" || created["host"] != "192.168.2.173" || created["mounted_on"] != 0.0 || created["state_at"] != nil || created["total_bytes"] != nil {263 t.Fatalf("a new NAS is unknown until observed: %v", created)264 }265 if r := admin.do("POST", "/storage/nas", map[string]any{"name": "NAS-1", "host": "10.0.0.9"}); r.Status != 409 {266 t.Fatalf("duplicate NAS name: %d %s", r.Status, r.Raw)267 }268 // Observed by an agent (ingestion's job): online with capacity; an old observation says nothing about now.269 h.dbExec(`update nas_targets set state = 'online', state_at = now(), observed_by = $2, latency_ms = 1.5, mounted_on = 3, total_bytes = 8000, used_bytes = 100 where id = $1`, nasID, a)270 list := operator.must("GET", "/storage/nas", nil, 200).jsonItems(t)271 if n := list[0]; n["state"] != "online" || n["latency_ms"] != 1.5 || n["mounted_on"] != 3.0 || n["total_bytes"] != 8000.0 || n["observed_by"] != a || n["host"] != "192.168.2.173" {272 t.Fatalf("observed NAS: %v", n)273 }274 if n := viewer.must("GET", "/storage/nas", nil, 200).jsonItems(t)[0]; n["host"] != nil || n["name"] != "nas-1" {275 t.Fatalf("NAS host is private: %v", n)276 }277 h.dbExec(`update nas_targets set state_at = now() - interval '1 hour' where id = $1`, nasID)278 if n := operator.must("GET", "/storage/nas", nil, 200).jsonItems(t)[0]; n["state"] != "unknown" || n["state_at"] == nil || n["latency_ms"] != nil {279 t.Fatalf("a stale observation must not be reported as the current state: %v", n)280 }281 if r := operator.do("DELETE", "/storage/nas/"+nasID, nil); r.Status != 403 {282 t.Fatalf("NAS removal without settings.manage: %d", r.Status)283 }284 admin.must("DELETE", "/storage/nas/"+nasID, nil, 204)285 if r := admin.do("DELETE", "/storage/nas/"+nasID, nil); r.Status != 404 {286 t.Fatalf("second NAS removal: %d", r.Status)287 }288 if h.count(`select count(*) from audit_log where action in ('nas.create', 'nas.delete') and target_id = $1 and actor_label = 'ada' and outcome = 'success'`, nasID) != 2 {289 t.Fatal("NAS changes must be audited")290 }291292 // Backups: never "ok" without evidence.293 h.dbExec(`insert into backups(id, name, node_id, kind, last_status, last_run_at, last_success_at, size_bytes, detail, observed_at) values294 ($1, 'pg-nightly', $4, 'postgres', 'ok', now(), now(), 123456, 'dump ok', now()),295 ($2, 'app-files', null, 'app', 'success', now(), null, null, '', now()),296 ($3, 'declared', null, 'mcc', 'unknown', null, null, null, '', null)`, ids.New().String(), ids.New().String(), ids.New().String(), a)297 bs := viewer.must("GET", "/storage/backups", nil, 200).jsonItems(t)298 if len(bs) != 3 || rowBy(bs, "name", "pg-nightly")["last_status"] != "ok" || rowBy(bs, "name", "pg-nightly")["node_alias"] != "alpha" ||299 rowBy(bs, "name", "pg-nightly")["size_bytes"] != 123456.0 || rowBy(bs, "name", "app-files")["last_status"] != "unknown" ||300 rowBy(bs, "name", "declared")["last_status"] != "unknown" || rowBy(bs, "name", "declared")["last_run_at"] != nil {301 t.Fatalf("backups: %v", bs)302 }303}304305func TestDashboardsArePerUserAndValidated(t *testing.T) {306 h := newHarness(t)307 root := h.owner()308 vera := h.member(root, "viewer", "vera", false)309 walt := h.member(root, "viewer", "walt", true)310 w := func(id string, x, y, wd, ht int) map[string]any {311 return map[string]any{"id": id, "type": "metric.chart", "x": x, "y": y, "w": wd, "h": ht, "config": map[string]any{"metric": "cpu.usage"}}312 }313314 // Nothing saved: the built-in home dashboard.315 list := vera.must("GET", "/me/dashboards", nil, 200).jsonItems(t)316 if len(list) != 1 || list[0]["name"] != "default" || len(list[0]["layout"].([]any)) == 0 {317 t.Fatalf("built-in dashboard: %v", list)318 }319 saved := vera.must("PUT", "/me/dashboards/default", map[string]any{"layout": []any{w("a", 0, 0, 6, 2), w("b", 6, 0, 6, 2)}}, 200).jsonObj(t)320 if saved["name"] != "default" || len(saved["layout"].([]any)) != 2 || saved["updated_at"] == nil {321 t.Fatalf("saved dashboard: %v", saved)322 }323 vera.must("PUT", "/me/dashboards/ops_wall-2", map[string]any{"layout": []any{}}, 200)324 vera.must("PUT", "/me/dashboards/default", map[string]any{"layout": []any{w("only", 0, 0, 12, 12)}}, 200) // replace325 mine := vera.must("GET", "/me/dashboards", nil, 200).jsonItems(t)326 if len(mine) != 2 || len(rowBy(mine, "name", "default")["layout"].([]any)) != 1 || dig(rowBy(mine, "name", "default")["layout"].([]any)[0].(map[string]any), "config", "metric") != "cpu.usage" {327 t.Fatalf("vera's dashboards: %v", mine)328 }329330 // Isolation: another user sees and touches only their own.331 theirs := walt.must("GET", "/me/dashboards", nil, 200).jsonItems(t)332 if len(theirs) != 1 || len(theirs[0]["layout"].([]any)) == 1 {333 t.Fatalf("walt must not see vera's dashboards: %v", theirs)334 }335 if r := walt.do("DELETE", "/me/dashboards/ops_wall-2", nil); r.Status != 404 {336 t.Fatalf("walt deleting vera's dashboard: %d", r.Status)337 }338 walt.must("PUT", "/me/dashboards/ops_wall-2", map[string]any{"layout": []any{w("mine", 0, 0, 1, 1)}}, 200)339 if l := rowBy(vera.must("GET", "/me/dashboards", nil, 200).jsonItems(t), "name", "ops_wall-2")["layout"].([]any); len(l) != 0 {340 t.Fatalf("walt's save changed vera's dashboard: %v", l)341 }342343 // Validation: bounds, sizes, counts.344 many := make([]any, 61)345 for i := range many {346 many[i] = w(fmt.Sprint("w", i), 0, i, 1, 1)347 }348 noConfig := w("a", 0, 0, 1, 1)349 delete(noConfig, "config")350 big := w("a", 0, 0, 1, 1)351 big["config"] = map[string]any{"blob": strings.Repeat("x", 5000)}352 for name, layout := range map[string]any{353 "w too large": []any{w("a", 0, 0, 13, 1)}, "w zero": []any{w("a", 0, 0, 0, 1)}, "h too large": []any{w("a", 0, 0, 1, 13)},354 "off grid": []any{w("a", 7, 0, 6, 1)}, "negative x": []any{w("a", -1, 0, 1, 1)}, "y too far": []any{w("a", 0, 1001, 1, 1)},355 "duplicate id": []any{w("a", 0, 0, 1, 1), w("a", 1, 0, 1, 1)}, "empty id": []any{w("", 0, 0, 1, 1)}, "long id": []any{w(strings.Repeat("i", 41), 0, 0, 1, 1)},356 "too many": many, "no config": []any{noConfig}, "config too big": []any{big}, "no layout": nil,357 } {358 body := map[string]any{"layout": layout}359 if layout == nil {360 body = map[string]any{}361 }362 if r := vera.do("PUT", "/me/dashboards/test", body); r.Status != 400 || r.code() != "validation_failed" {363 t.Fatalf("%s: %d %s", name, r.Status, r.Raw)364 }365 }366 for _, name := range []string{"Bad", "has%20space", strings.Repeat("n", 41), "-lead"} {367 if r := vera.do("PUT", "/me/dashboards/"+name, map[string]any{"layout": []any{}}); r.Status != 400 {368 t.Fatalf("name %q: %d %s", name, r.Status, r.Raw)369 }370 }371 if h.count(`select count(*) from user_dashboards where name = 'test'`) != 0 {372 t.Fatal("an invalid layout was stored")373 }374 // At most 20 dashboards per user.375 for i := 0; i < 18; i++ {376 vera.must("PUT", fmt.Sprintf("/me/dashboards/d%d", i), map[string]any{"layout": []any{}}, 200)377 }378 if r := vera.do("PUT", "/me/dashboards/one-too-many", map[string]any{"layout": []any{}}); r.Status != 400 {379 t.Fatalf("21st dashboard: %d %s", r.Status, r.Raw)380 }381 vera.must("PUT", "/me/dashboards/d3", map[string]any{"layout": []any{w("x", 0, 0, 1, 1)}}, 200) // replacing an existing one stays possible382383 // Delete: `default` falls back to the built-in layout; an unknown name is 404.384 vera.must("DELETE", "/me/dashboards/default", nil, 204)385 vera.must("DELETE", "/me/dashboards/default", nil, 204)386 if d := rowBy(vera.must("GET", "/me/dashboards", nil, 200).jsonItems(t), "name", "default"); len(d["layout"].([]any)) < 2 {387 t.Fatalf("default after delete must be the built-in layout: %v", d)388 }389 if r := vera.do("DELETE", "/me/dashboards/nope", nil); r.Status != 404 {390 t.Fatalf("unknown dashboard: %d", r.Status)391 }392 if r := h.browser().do("GET", "/me/dashboards", nil); r.Status != 401 {393 t.Fatalf("anonymous: %d", r.Status)394 }395}396397func TestSearchRespectsPermissionsAndEscapesWildcards(t *testing.T) {398 h := newHarness(t)399 root := h.owner()400 viewer := h.member(root, "viewer", "vera", false)401 a := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second, IP: "192.168.2.50", Tags: []string{"render"}})402 b := h.seedNode(nodeSeed{Alias: "alphabet", Seen: true, Age: time.Hour, Site: "lab"})403 h.seedService(a, "pm2", "alpha-web", "online")404 pct := h.seedService(b, "pm2", "cpu_100%_burn", "errored")405 h.seedModel(a, "ollama", 8081, "alpha-model", "ok")406 h.seedAlert(a, "firing", "critical", "alpha disk full")407 h.dbExec(`insert into incidents(id, title, severity) values ($1, 'alpha outage', 'critical')`, ids.New().String())408 h.dbExec(`insert into jobs(id, template, state, node_id) values ($1, 'alpha-batch', 'running', $2)`, ids.New().String(), a)409 h.dbExec(`insert into actions(id, type, criticality, node_id, target_kind, target_id, state, idempotency_key, request_hash, timeout_s, correlation_id)410 values ($1, 'service.restart', 'medium', $2, 'node', $3, 'queued', 'k', '\x00', 60, $4)`, ids.New().String(), a, a, ids.New().String())411 win := ids.New().String()412 h.dbExec(`insert into maintenance_windows(id, title, starts_at, ends_at) values ($1, 'alpha firmware', now(), now() + interval '1 hour')`, win)413 h.dbExec(`insert into maintenance_notes(id, window_id, author_id, body) values ($1, $2, $3, 'replaced the alpha fan'), ($4, $2, $3, 'alpha secret note')`,414 ids.New().String(), win, h.userIDOf("root"), ids.New().String())415 h.dbExec(`update maintenance_notes set deleted_at = now() where body = 'alpha secret note'`)416417 kindsOf := func(c *client, q string) map[string]int {418 out := map[string]int{}419 for _, r := range c.must("GET", "/search?limit=50&q="+url.QueryEscape(q), nil, 200).jsonItems(t) {420 out[r["kind"].(string)]++421 }422 return out423 }424 got := kindsOf(root, "ALPHA")425 want := map[string]int{"node": 2, "service": 1, "alert": 1, "action": 1, "incident": 1, "job": 1, "inference_server": 1, "maintenance_window": 1, "maintenance_note": 1}426 if fmt.Sprint(got) != fmt.Sprint(want) {427 t.Fatalf("search kinds: %v, want %v", got, want)428 }429 res := root.must("GET", "/search?q=alpha", nil, 200).jsonObj(t)430 items := res["items"].([]any)431 first := items[0].(map[string]any)432 if res["query"] != "alpha" || first["kind"] != "node" || first["title"] != "alpha" || first["score"] != 1.0 || first["freshness"] != "fresh" || first["node_id"] != a {433 t.Fatalf("an exact match on a node alias ranks first: %v", first)434 }435 for i := 1; i < len(items); i++ {436 if items[i].(map[string]any)["score"].(float64) > items[i-1].(map[string]any)["score"].(float64) {437 t.Fatalf("results must be ordered by relevance: %v", items)438 }439 }440441 // Wildcards typed by the user are literal characters.442 for q, want := range map[string]int{"%": 1, "_": 1, "100%": 1, "100%_b": 1, "1_0": 0, "%%": 0, `\`: 0, "a%a": 0} {443 rs := root.must("GET", "/search?limit=50&q="+url.QueryEscape(q), nil, 200).jsonItems(t)444 if len(rs) != want || (want == 1 && rs[0]["id"] != pct) {445 t.Fatalf("q=%q: %d results %v, want %d", q, len(rs), rs, want)446 }447 }448 // An injection attempt is just a string that matches nothing.449 if rs := root.must("GET", "/search?q="+url.QueryEscape(`'; drop table nodes; --`), nil, 200).jsonItems(t); len(rs) != 0 || h.count(`select count(*) from nodes`) != 2 {450 t.Fatalf("injection: %v", rs)451 }452 // A private value is not searchable: an address must not be confirmable through search.453 if rs := root.must("GET", "/search?q=192.168.2.50", nil, 200).jsonItems(t); len(rs) != 0 {454 t.Fatalf("search by IP address: %v", rs)455 }456 if rs := root.must("GET", "/search?q=alpha&kinds=service,job&limit=1", nil, 200).jsonItems(t); len(rs) != 1 || rs[0]["kind"] != "service" {457 t.Fatalf("kinds + limit: %v", rs)458 }459 if rs := root.must("GET", "/search?q=render", nil, 200).jsonItems(t); len(rs) != 1 || rs[0]["title"] != "alpha" || rs[0]["score"].(float64) >= 0.5 {460 t.Fatalf("match through a tag: %v", rs)461 }462 for _, bad := range []string{"q=", "q=" + strings.Repeat("x", 121), "q=a&limit=0", "q=a&limit=51", "q=a&kinds=users", ""} {463 if r := root.do("GET", "/search?"+bad, nil); r.Status != 400 {464 t.Fatalf("%q: %d %s", bad, r.Status, r.Raw)465 }466 }467468 // Each kind needs its own permission; a scoped deny hides what belongs to that node.469 if got := kindsOf(viewer, "alpha"); fmt.Sprint(got) != fmt.Sprint(want) {470 t.Fatalf("viewer search: %v", got)471 }472 vera := h.userIDOf("vera")473 root.grant("user", vera, "services.read", "", "", "deny")474 root.grant("user", vera, "alerts.read", "", "", "deny")475 root.grant("user", vera, "maintenance.read", "", "", "deny")476 root.grant("user", vera, "nodes.read", "node", b, "deny")477 root.grant("user", vera, "models.read", "node", a, "deny")478 if got := kindsOf(viewer, "alpha"); fmt.Sprint(got) != fmt.Sprint(map[string]int{"node": 1, "action": 1, "job": 1}) {479 t.Fatalf("viewer search after denies: %v", got)480 }481 if rs := viewer.must("GET", "/search?q=alpha&kinds=service", nil, 200).jsonItems(t); len(rs) != 0 {482 t.Fatalf("asking for a kind one cannot read returns nothing: %v", rs)483 }484 // The same scoped deny applies to every other read.485 if r := viewer.do("GET", "/nodes/"+b, nil); r.Status != 404 {486 t.Fatalf("node hidden by a scoped deny: %d", r.Status)487 }488 if got := viewer.walkPages("/nodes", 1, "alias"); strings.Join(got, ",") != "alpha" {489 t.Fatalf("list after a scoped deny: %v", got)490 }491 if r := viewer.do("GET", "/nodes/"+b+"/metrics?metrics=cpu.usage", nil); r.Status != 404 {492 t.Fatalf("metrics of a hidden node: %d", r.Status)493 }494 if o := viewer.must("GET", "/cluster/overview", nil, 200).jsonObj(t); dig(o, "nodes", "total") != 1.0 || o["services"] != nil {495 t.Fatalf("overview after denies: %v", o)496 }497 if topo := viewer.must("GET", "/network/topology", nil, 200).jsonObj(t); strings.Contains(fmt.Sprint(topo), b) {498 t.Fatalf("topology shows a hidden node: %v", topo)499 }500 if r := viewer.do("GET", "/services", nil); r.Status != 403 {501 t.Fatalf("services without services.read: %d", r.Status)502 }503}504505func TestActivityIsARedactedProjectionOfTheAuditLog(t *testing.T) {506 h := newHarness(t)507 root := h.owner()508 viewer := h.member(root, "viewer", "vera", false)509 n := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second})510 other := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Second})511 if root.login().Status != 200 || viewer.login().Status != 200 { // two sign-ins for the feed512 t.Fatal("sign-in failed")513 }514 rootID, _ := uuid.Parse(h.userIDOf("root"))515 ip := netip.MustParseAddr("203.0.113.77")516 add := func(action, targetKind, targetID string, outcome audit.Outcome, params map[string]any) {517 t.Helper()518 if _, err := audit.Append(context.Background(), h.Pool, audit.Entry{ActorKind: "user", ActorID: &rootID, ActorLabel: "root", IP: &ip,519 Action: action, TargetKind: targetKind, TargetID: targetID, Outcome: outcome, Detail: "free text with internal-detail", Params: params}); err != nil {520 t.Fatal(err)521 }522 }523 add("action.create", "node", n, audit.Success, map[string]any{"type": "service.restart", "justification": "param-only-value"})524 add("action.approve", "action", ids.New().String(), audit.Success, map[string]any{"node_id": n})525 add("alert.acknowledge", "alert", ids.New().String(), audit.Success, nil)526 add("alert.resolve", "alert", ids.New().String(), audit.Success, nil)527 add("incident.create", "incident", ids.New().String(), audit.Success, nil)528 add("maintenance.window.create", "maintenance_window", ids.New().String(), audit.Success, nil)529 add("maintenance.note.create", "maintenance_note", ids.New().String(), audit.Success, nil)530 add("action.create", "node", other, audit.Denied, nil)531 add("user.update", "user", ids.New().String(), audit.Success, nil) // administration: never in the feed532 add("setting.update", "setting", "x", audit.Success, nil) // idem533 add("authz.denied", "node", n, audit.Denied, map[string]any{"a": "b"}) // idem534535 raw := string(viewer.must("GET", "/activity?limit=200", nil, 200).Raw)536 for _, leak := range []string{"203.0.113.77", "internal-detail", "param-only-value", "justification", "session", "device", "correlation"} {537 if strings.Contains(raw, leak) {538 t.Fatalf("activity leaks %q: %s", leak, raw)539 }540 }541 items := viewer.must("GET", "/activity?limit=200", nil, 200).jsonItems(t)542 count := map[string]int{}543 for _, it := range items {544 count[it["kind"].(string)]++545 for k := range it {546 if !strings.Contains("id ts kind actor_display_name summary target_kind target_id target_label outcome", k) {547 t.Fatalf("unexpected field %q in an activity item: %v", k, it)548 }549 }550 }551 if count["action"] != 3 || count["alert_ack"] != 1 || count["alert_resolve"] != 1 || count["incident"] != 1 || count["maintenance"] != 1 ||552 count["maintenance_note"] != 1 || count["login"] != 2 || len(count) != 7 {553 t.Fatalf("activity kinds: %v", count)554 }555 newest := items[0]556 if newest["kind"] != "action" || newest["outcome"] != "denied" || newest["target_label"] != "bravo" || newest["actor_display_name"] != "Root Owner" ||557 newest["summary"] != "Action: create" || newest["target_id"] != other {558 t.Fatalf("most recent item first: %v", newest)559 }560 if ack := rowBy(items, "kind", "alert_ack"); ack["summary"] != "Alert acknowledged" || ack["target_kind"] != "alert" {561 t.Fatalf("ack item: %v", ack)562 }563 if note := rowBy(items, "kind", "maintenance_note"); note["summary"] != "Maintenance note: create" {564 t.Fatalf("note item: %v", note)565 }566 if login := rowBy(items, "kind", "login"); login["summary"] != "Signed in" || login["target_id"] != nil || login["actor_display_name"] == nil {567 t.Fatalf("login item: %v", login)568 }569570 // Pagination walks every item once; filters narrow the feed.571 var allIDs []string572 for _, it := range items {573 allIDs = append(allIDs, it["id"].(string))574 }575 wantExactlyOnce(t, "paged activity", viewer.walkPages("/activity", 3, "id"), allIDs...)576 wantExactlyOnce(t, "kind filter", viewer.walkPages("/activity?kind=incident", 3, "kind"), "incident")577 if got := viewer.walkPages("/activity?node_id="+n, 10, "summary"); strings.Join(got, "|") != "Action: approve|Action: create" {578 t.Fatalf("node filter (target or node parameter): %v", got)579 }580 for _, bad := range []string{"kind=secret", "cursor=xyz", "limit=500", "node_id=x"} {581 if r := viewer.do("GET", "/activity?"+bad, nil); r.Status != 400 {582 t.Fatalf("%s: %d", bad, r.Status)583 }584 }585586 // Items are only returned when the caller can read the underlying resource.587 vera := h.userIDOf("vera")588 root.grant("user", vera, "actions.read", "", "", "deny")589 root.grant("user", vera, "alerts.read", "", "", "deny")590 after := map[string]int{}591 for _, it := range viewer.must("GET", "/activity?limit=200", nil, 200).jsonItems(t) {592 after[it["kind"].(string)]++593 }594 if after["action"] != 0 || after["alert_ack"] != 0 || after["alert_resolve"] != 0 || after["incident"] != 0 || after["maintenance"] != 1 || after["login"] != 2 {595 t.Fatalf("activity after denies: %v", after)596 }597 if got := viewer.walkPages("/activity?kind=action", 10, "id"); len(got) != 0 {598 t.Fatalf("asking for a kind one cannot read: %v", got)599 }600}601