SPB Git

Repository Public

cluster_services_test.go 32.5 KB 601 lines go

1package api_test23import (4	"context"5	"fmt"6	"net/netip"7	"net/url"8	"strings"9	"testing"10	"time"1112	"github.com/google/uuid"1314	"clustr.spboucher.ai/mcc/internal/audit"15	"clustr.spboucher.ai/mcc/internal/ids"16)1718func TestServicesMappingHistoryAndUpdate(t *testing.T) {19	h := newHarness(t)20	root := h.owner()21	admin := h.member(root, "admin", "ada", false)22	viewer := h.member(root, "viewer", "vera", false)23	a := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second})24	b := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Hour, Kind: "linux"})25	web := h.seedService(a, "pm2", "web", "online")26	h.dbExec(`update services set app = 'shop', public_url = 'https://shop.example', health = '{"status":"ok","latency_ms":12.5,"detail":"200"}' where id = $1`, web)27	h.seedService(a, "launchd", "io.example.worker", "errored")28	h.seedService(a, "pm2", "Zeta", "weird-state")29	old := h.seedService(a, "pm2", "old", "online")30	db := h.seedService(b, "systemd", "postgresql", "online")31	h.seedService(b, "docker", "cache", "stopped")32	retired := h.seedNode(nodeSeed{Alias: "gone", Seen: true, Age: time.Second})33	h.seedService(retired, "pm2", "hidden", "online")34	h.dbExec(`update nodes set retired_at = now() where id = $1`, retired)3536	// Observed changes, as ingestion would write them: a crash, two restarts, a version change, then the service disappears.37	h.dbExec(`update services set state = 'errored', last_seen_at = now() + interval '1 second' where id = $1`, old)38	h.dbExec(`update services set state = 'online', restarts = 5, last_seen_at = now() + interval '2 seconds' where id = $1`, old)39	h.dbExec(`update services set version = '1.3.0', last_seen_at = now() + interval '3 seconds' where id = $1`, old)40	h.dbExec(`update services set last_seen_at = now() + interval '4 seconds' where id = $1`, old) // nothing changed: nothing recorded41	h.dbExec(`update services set gone_at = now() + interval '5 seconds' where id = $1`, old)4243	all := []string{"cache", "io.example.worker", "old", "postgresql", "web", "Zeta"}44	for _, limit := range []int{1, 2, 50} {45		if got := viewer.walkPages("/services", limit, "name"); strings.Join(got, ",") != strings.Join(all, ",") {46			t.Fatalf("limit %d: %v", limit, got)47		}48	}49	rows := viewer.must("GET", "/services", nil, 200).jsonItems(t)50	w := rowBy(rows, "name", "web")51	if w["node_alias"] != "alpha" || w["manager"] != "pm2" || w["state"] != "online" || w["pid"] != 4242.0 || w["restarts"] != 3.0 ||52		w["mem_bytes"] != 104857600.0 || w["cpu_pct"] != 1.5 || w["uptime_s"] != 3600.0 || w["version"] != "1.2.0" || fmt.Sprint(w["ports"]) != "[8080 8443]" ||53		w["app"] != "shop" || w["public_url"] != "https://shop.example" || w["managed"] != false || w["freshness"] != "fresh" || w["gone_at"] != nil ||54		dig(w, "health", "status") != "ok" || dig(w, "health", "latency_ms") != 12.5 || w["meta"] != nil {55		t.Fatalf("service mapping: %v", w)56	}57	if o := rowBy(rows, "name", "old"); o["gone_at"] == nil || o["version"] != "1.3.0" {58		t.Fatalf("a service that is gone is still listed, marked as gone: %v", o)59	}60	if z := rowBy(rows, "name", "Zeta"); z["state"] != "unknown" {61		t.Fatalf("an unrecognized state is unknown: %v", z)62	}63	if p := rowBy(rows, "name", "postgresql"); p["freshness"] != "offline" || p["state"] != "online" {64		t.Fatalf("last known state with the node's freshness: %v", p)65	}66	for f, want := range map[string][]string{67		"node_id=" + b:              {"cache", "postgresql"},68		"manager=launchd":           {"io.example.worker"},69		"state=online":              {"postgresql", "web"}, // "old" is gone: it is in no current state70		"state=unknown":             {"Zeta"},71		"app=SHOP":                  {"web"},72		"managed=true":              {},73		"q=EXAMPLE":                 {"io.example.worker"},74		"q=shop":                    {"web"},75		"q=" + url.QueryEscape("%"): {},76		"state=online&node_id=" + a: {"web"},77	} {78		wantExactlyOnce(t, f, viewer.walkPages("/services?"+f, 2, "name"), want...)79	}80	for _, bad := range []string{"manager=cron", "state=dead", "limit=0", "cursor=zzz", "node_id=nope"} {81		if r := viewer.do("GET", "/services?"+bad, nil); r.Status != 400 {82			t.Fatalf("%s: %d %s", bad, r.Status, r.Raw)83		}84	}85	wantExactlyOnce(t, "node services", viewer.walkPages("/nodes/"+b+"/services", 1, "name"), "cache", "postgresql")86	if r := viewer.do("GET", "/nodes/"+ids.New().String()+"/services", nil); r.Status != 404 {87		t.Fatalf("services of an unknown node: %d", r.Status)88	}8990	// Single view: meta is included, with secret-looking members masked.91	one := viewer.must("GET", "/services/"+web, nil, 200).jsonObj(t)92	if dig(one, "meta", "raw_state") != "online" || dig(one, "meta", "api_token") != "[redacted]" || strings.Contains(fmt.Sprint(one), "s3cr3t") {93		t.Fatalf("meta: %v", one["meta"])94	}95	if r := viewer.do("GET", "/services/"+ids.New().String(), nil); r.Status != 404 {96		t.Fatalf("unknown service: %d", r.Status)97	}9899	// History: only recorded facts, most recent first.100	h.dbExec(`insert into actions(id, type, criticality, node_id, target_kind, target_id, state, requested_by, idempotency_key, request_hash, timeout_s,101		correlation_id, created_at) values ($1, 'service.restart', 'medium', $2, 'service', $3, 'succeeded', $4, 'k', '\x00', 60, $5, now() + interval '10 seconds')`,102		ids.New().String(), a, old, h.userIDOf("ada"), ids.New().String())103	hist := viewer.must("GET", "/services/"+old+"/history", nil, 200).jsonItems(t)104	var kinds []string105	for _, e := range hist {106		kinds = append(kinds, e["kind"].(string))107	}108	if strings.Join(kinds, " ") != "action disappeared deploy restart state_change state_change appeared" {109		t.Fatalf("history: %v", kinds)110	}111	if e := hist[0]; e["summary"] != "Action service.restart: succeeded." || e["actor_display_name"] != "User ada" || e["action_id"] == nil {112		t.Fatalf("action entry: %v", e)113	}114	if e := hist[2]; e["version"] != "1.3.0" || !strings.Contains(e["summary"].(string), "1.2.0 to 1.3.0") {115		t.Fatalf("deploy entry: %v", e)116	}117	if e := hist[3]; e["summary"] != "Restart counter went from 3 to 5." {118		t.Fatalf("restart entry: %v", e)119	}120	if e := hist[5]; e["from_state"] != "online" || e["to_state"] != "errored" {121		t.Fatalf("state change entry: %v", e)122	}123	if got := viewer.walkPages("/services/"+old+"/history", 2, "kind"); strings.Join(got, " ") != strings.Join(kinds, " ") {124		t.Fatalf("paged history: %v", got)125	}126	if got := viewer.walkPages("/services/"+db+"/history", 5, "kind"); strings.Join(got, " ") != "appeared" {127		t.Fatalf("history of a quiet service: %v", got)128	}129	// Without actions.read the action entries are left out.130	root.grant("user", h.userIDOf("vera"), "actions.read", "", "", "deny")131	if got := viewer.walkPages("/services/"+old+"/history", 50, "kind"); len(got) != 6 || got[0] != "disappeared" {132		t.Fatalf("history without actions.read: %v", got)133	}134135	// Update: managed flag and public URL, audited, announced.136	if r := viewer.do("PATCH", "/services/"+web, map[string]any{"managed": true}); r.Status != 403 {137		t.Fatalf("viewer patch: %d", r.Status)138	}139	for name, bad := range map[string]map[string]any{"nothing": {}, "scheme": {"public_url": "ftp://x"}, "relative": {"public_url": "/path"},140		"credentials": {"public_url": "https://user:pw@example.com"}, "long": {"public_url": "https://example.com/" + strings.Repeat("a", 300)}} {141		if r := admin.do("PATCH", "/services/"+web, bad); r.Status != 400 {142			t.Fatalf("patch %s: %d %s", name, r.Status, r.Raw)143		}144	}145	u := admin.must("PATCH", "/services/"+web, map[string]any{"managed": true, "public_url": ""}, 200).jsonObj(t)146	if u["managed"] != true || u["public_url"] != nil {147		t.Fatalf("patched service: %v", u)148	}149	admin.must("PATCH", "/services/"+web, map[string]any{"managed": true}, 200) // no change150	if h.count(`select count(*) from audit_log where action = 'service.update' and target_id = $1 and params->'changes'->'managed'->>'to' = 'true'`, web) != 1 ||151		h.count(`select count(*) from audit_log where action = 'service.update'`) != 1 {152		t.Fatal("service update must be audited once")153	}154	if h.count(`select count(*) from events_outbox where topic = 'service.changed' and payload->>'service_id' = $1 and payload->>'state' = 'online'155		and payload->>'manager' = 'pm2' and payload->>'name' = 'web' and payload->>'node_id' = $2`, web, a) != 1 {156		t.Fatal("service.changed event missing")157	}158	wantExactlyOnce(t, "managed=true", viewer.walkPages("/services?managed=true", 10, "name"), "web")159	if h.count(`select count(*) from service_events where service_id = $1`, web) != 1 {160		t.Fatal("an administrative change is not an observed change of the service")161	}162}163164func TestModelsVolumesNasAndBackups(t *testing.T) {165	h := newHarness(t)166	root := h.owner()167	admin := h.member(root, "admin", "ada", false)168	operator := h.member(root, "operator", "oscar", false)169	viewer := h.member(root, "viewer", "vera", false)170	a := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second})171	b := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Minute})172173	// Inference servers.174	mlx := h.seedModel(a, "mlx_lm", 8081, "mlx-community/Qwen2.5-7B", "ok")175	h.seedModel(a, "ollama", 11434, "llama3", "failing")176	goneModel := h.seedModel(b, "llama.cpp", 8080, "phi", "ok")177	h.dbExec(`update inference_servers set gone_at = now() where id = $1`, goneModel)178	h.seedModel(b, "mlx_lm", 9000, "", "strange")179	if got := viewer.walkPages("/models", 1, "port"); strings.Join(got, ",") != "8081,11434,8080,9000" {180		t.Fatalf("models order (node alias, port): %v", got)181	}182	ms := viewer.must("GET", "/models", nil, 200).jsonItems(t)183	m := rowBy(ms, "id", mlx)184	if m["node_alias"] != "alpha" || m["kind"] != "mlx_lm" || m["model"] != "mlx-community/Qwen2.5-7B" || m["quantization"] != "4bit" || m["health"] != "ok" ||185		m["rss_bytes"] != 5368709120.0 || m["health_latency_ms"] != 4.2 || fmt.Sprint(m["listed_models"]) != "[mlx-community/Qwen2.5-7B]" || m["freshness"] != "fresh" {186		t.Fatalf("model mapping: %v", m)187	}188	for _, family := range []string{"requests", "tokens_per_s", "ttft", "latency", "queue"} {189		if dig(m, "metrics_support", family, "availability") != "unsupported" || !strings.Contains(fmt.Sprint(dig(m, "metrics_support", family, "reason")), "mlx_lm.server") {190			t.Fatalf("metrics_support.%s must be explicitly unsupported: %v", family, m["metrics_support"])191		}192	}193	if o := rowBy(ms, "port", "11434"); !strings.Contains(fmt.Sprint(dig(o, "metrics_support", "requests", "reason")), "not collected") {194		t.Fatalf("other kinds: %v", o["metrics_support"])195	}196	if g := rowBy(ms, "id", goneModel); g["gone_at"] == nil || g["freshness"] != "stale" {197		t.Fatalf("gone model: %v", g)198	}199	if s := rowBy(ms, "port", "9000"); s["health"] != "unknown" || s["model"] != nil {200		t.Fatalf("unrecognized health is unknown, empty model is absent: %v", s)201	}202	wantExactlyOnce(t, "health=ok", viewer.walkPages("/models?health=ok", 5, "port"), "8081") // the gone one is not "ok"203	wantExactlyOnce(t, "health=unknown", viewer.walkPages("/models?health=unknown", 5, "port"), "9000")204	wantExactlyOnce(t, "node", viewer.walkPages("/models?node_id="+b, 5, "port"), "8080", "9000")205	if r := viewer.do("GET", "/models?health=great", nil); r.Status != 400 {206		t.Fatalf("bad health: %d", r.Status)207	}208	if one := viewer.must("GET", "/models/"+mlx, nil, 200).jsonObj(t); one["port"] != 8081.0 {209		t.Fatalf("get model: %v", one)210	}211	if r := viewer.do("GET", "/models/"+ids.New().String(), nil); r.Status != 404 {212		t.Fatalf("unknown model: %d", r.Status)213	}214215	// Volumes.216	h.seedVolume(a, "/", "/dev/disk3s1", "apfs", "internal", 1000, 900, false)217	h.seedVolume(a, "/Volumes/T7", "/dev/disk5s1", "exfat", "external", 2000, 100, false)218	h.seedVolume(a, "/Volumes/Old", "/dev/disk9s1", "apfs", "external", 500, 499, true)219	h.seedVolume(b, "/mnt/nas", "//192.168.2.173/share", "smbfs", "network", 4000, 2000, false)220	unk := h.seedVolume(b, "/mnt/x", "/dev/sdx", "ext4", "internal", 0, 0, false)221	h.dbExec(`update volumes set total_bytes = null, used_bytes = null, smart_status = null where id = $1`, unk)222	if got := viewer.walkPages("/storage/volumes", 2, "mount"); strings.Join(got, ",") != "/,/Volumes/Old,/Volumes/T7,/mnt/nas,/mnt/x" {223		t.Fatalf("volumes order: %v", got)224	}225	vs := viewer.must("GET", "/storage/volumes", nil, 200).jsonItems(t)226	if v := rowBy(vs, "mount", "/"); v["node_alias"] != "alpha" || v["kind"] != "internal" || v["total_bytes"] != 1000.0 || v["used_bytes"] != 900.0 ||227		v["smart_status"] != "Verified" || v["fs"] != "apfs" || v["device"] != "/dev/disk3s1" || v["freshness"] != "fresh" {228		t.Fatalf("volume mapping: %v", v)229	}230	if v := rowBy(vs, "mount", "/Volumes/Old"); v["gone_at"] == nil {231		t.Fatalf("gone volume: %v", v)232	}233	if v := rowBy(vs, "mount", "/mnt/x"); v["total_bytes"] != nil || v["used_bytes"] != nil || v["smart_status"] != nil {234		t.Fatalf("what was not observed is absent, not zero: %v", v)235	}236	wantExactlyOnce(t, "min_used_pct", viewer.walkPages("/storage/volumes?min_used_pct=50", 5, "mount"), "/", "/Volumes/Old", "/mnt/nas")237	wantExactlyOnce(t, "kind", viewer.walkPages("/storage/volumes?kind=external", 5, "mount"), "/Volumes/T7", "/Volumes/Old")238	wantExactlyOnce(t, "node", viewer.walkPages("/storage/volumes?node_id="+b, 5, "mount"), "/mnt/nas", "/mnt/x")239	for _, bad := range []string{"kind=tape", "min_used_pct=101", "min_used_pct=-1"} {240		if r := viewer.do("GET", "/storage/volumes?"+bad, nil); r.Status != 400 {241			t.Fatalf("%s: %d", bad, r.Status)242		}243	}244	if d := viewer.must("GET", "/nodes/"+a, nil, 200).jsonObj(t); len(d["volumes"].([]any)) != 2 {245		t.Fatalf("node detail lists current volumes only: %v", d["volumes"])246	}247248	// NAS targets.249	nas := map[string]any{"name": "nas-1", "host": "192.168.2.173", "share": "clustr", "protocol": "smb", "note": "rack 2"}250	if r := operator.do("POST", "/storage/nas", nas); r.Status != 403 {251		t.Fatalf("NAS registration without settings.manage: %d", r.Status)252	}253	for name, bad := range map[string]map[string]any{"host name": {"name": "x", "host": "nas.local"}, "empty name": {"name": " ", "host": "10.0.0.1"},254		"protocol": {"name": "x", "host": "10.0.0.1", "protocol": "ftp"}, "note": {"name": "x", "host": "10.0.0.1", "note": strings.Repeat("n", 501)},255		"unspecified": {"name": "x", "host": "0.0.0.0"}} {256		if r := admin.do("POST", "/storage/nas", bad); r.Status != 400 {257			t.Fatalf("NAS %s: %d %s", name, r.Status, r.Raw)258		}259	}260	created := admin.must("POST", "/storage/nas", nas, 201).jsonObj(t)261	nasID := created["id"].(string)262	if created["state"] != "unknown" || created["host"] != "192.168.2.173" || created["mounted_on"] != 0.0 || created["state_at"] != nil || created["total_bytes"] != nil {263		t.Fatalf("a new NAS is unknown until observed: %v", created)264	}265	if r := admin.do("POST", "/storage/nas", map[string]any{"name": "NAS-1", "host": "10.0.0.9"}); r.Status != 409 {266		t.Fatalf("duplicate NAS name: %d %s", r.Status, r.Raw)267	}268	// Observed by an agent (ingestion's job): online with capacity; an old observation says nothing about now.269	h.dbExec(`update nas_targets set state = 'online', state_at = now(), observed_by = $2, latency_ms = 1.5, mounted_on = 3, total_bytes = 8000, used_bytes = 100 where id = $1`, nasID, a)270	list := operator.must("GET", "/storage/nas", nil, 200).jsonItems(t)271	if n := list[0]; n["state"] != "online" || n["latency_ms"] != 1.5 || n["mounted_on"] != 3.0 || n["total_bytes"] != 8000.0 || n["observed_by"] != a || n["host"] != "192.168.2.173" {272		t.Fatalf("observed NAS: %v", n)273	}274	if n := viewer.must("GET", "/storage/nas", nil, 200).jsonItems(t)[0]; n["host"] != nil || n["name"] != "nas-1" {275		t.Fatalf("NAS host is private: %v", n)276	}277	h.dbExec(`update nas_targets set state_at = now() - interval '1 hour' where id = $1`, nasID)278	if n := operator.must("GET", "/storage/nas", nil, 200).jsonItems(t)[0]; n["state"] != "unknown" || n["state_at"] == nil || n["latency_ms"] != nil {279		t.Fatalf("a stale observation must not be reported as the current state: %v", n)280	}281	if r := operator.do("DELETE", "/storage/nas/"+nasID, nil); r.Status != 403 {282		t.Fatalf("NAS removal without settings.manage: %d", r.Status)283	}284	admin.must("DELETE", "/storage/nas/"+nasID, nil, 204)285	if r := admin.do("DELETE", "/storage/nas/"+nasID, nil); r.Status != 404 {286		t.Fatalf("second NAS removal: %d", r.Status)287	}288	if h.count(`select count(*) from audit_log where action in ('nas.create', 'nas.delete') and target_id = $1 and actor_label = 'ada' and outcome = 'success'`, nasID) != 2 {289		t.Fatal("NAS changes must be audited")290	}291292	// Backups: never "ok" without evidence.293	h.dbExec(`insert into backups(id, name, node_id, kind, last_status, last_run_at, last_success_at, size_bytes, detail, observed_at) values294		($1, 'pg-nightly', $4, 'postgres', 'ok', now(), now(), 123456, 'dump ok', now()),295		($2, 'app-files', null, 'app', 'success', now(), null, null, '', now()),296		($3, 'declared', null, 'mcc', 'unknown', null, null, null, '', null)`, ids.New().String(), ids.New().String(), ids.New().String(), a)297	bs := viewer.must("GET", "/storage/backups", nil, 200).jsonItems(t)298	if len(bs) != 3 || rowBy(bs, "name", "pg-nightly")["last_status"] != "ok" || rowBy(bs, "name", "pg-nightly")["node_alias"] != "alpha" ||299		rowBy(bs, "name", "pg-nightly")["size_bytes"] != 123456.0 || rowBy(bs, "name", "app-files")["last_status"] != "unknown" ||300		rowBy(bs, "name", "declared")["last_status"] != "unknown" || rowBy(bs, "name", "declared")["last_run_at"] != nil {301		t.Fatalf("backups: %v", bs)302	}303}304305func TestDashboardsArePerUserAndValidated(t *testing.T) {306	h := newHarness(t)307	root := h.owner()308	vera := h.member(root, "viewer", "vera", false)309	walt := h.member(root, "viewer", "walt", true)310	w := func(id string, x, y, wd, ht int) map[string]any {311		return map[string]any{"id": id, "type": "metric.chart", "x": x, "y": y, "w": wd, "h": ht, "config": map[string]any{"metric": "cpu.usage"}}312	}313314	// Nothing saved: the built-in home dashboard.315	list := vera.must("GET", "/me/dashboards", nil, 200).jsonItems(t)316	if len(list) != 1 || list[0]["name"] != "default" || len(list[0]["layout"].([]any)) == 0 {317		t.Fatalf("built-in dashboard: %v", list)318	}319	saved := vera.must("PUT", "/me/dashboards/default", map[string]any{"layout": []any{w("a", 0, 0, 6, 2), w("b", 6, 0, 6, 2)}}, 200).jsonObj(t)320	if saved["name"] != "default" || len(saved["layout"].([]any)) != 2 || saved["updated_at"] == nil {321		t.Fatalf("saved dashboard: %v", saved)322	}323	vera.must("PUT", "/me/dashboards/ops_wall-2", map[string]any{"layout": []any{}}, 200)324	vera.must("PUT", "/me/dashboards/default", map[string]any{"layout": []any{w("only", 0, 0, 12, 12)}}, 200) // replace325	mine := vera.must("GET", "/me/dashboards", nil, 200).jsonItems(t)326	if len(mine) != 2 || len(rowBy(mine, "name", "default")["layout"].([]any)) != 1 || dig(rowBy(mine, "name", "default")["layout"].([]any)[0].(map[string]any), "config", "metric") != "cpu.usage" {327		t.Fatalf("vera's dashboards: %v", mine)328	}329330	// Isolation: another user sees and touches only their own.331	theirs := walt.must("GET", "/me/dashboards", nil, 200).jsonItems(t)332	if len(theirs) != 1 || len(theirs[0]["layout"].([]any)) == 1 {333		t.Fatalf("walt must not see vera's dashboards: %v", theirs)334	}335	if r := walt.do("DELETE", "/me/dashboards/ops_wall-2", nil); r.Status != 404 {336		t.Fatalf("walt deleting vera's dashboard: %d", r.Status)337	}338	walt.must("PUT", "/me/dashboards/ops_wall-2", map[string]any{"layout": []any{w("mine", 0, 0, 1, 1)}}, 200)339	if l := rowBy(vera.must("GET", "/me/dashboards", nil, 200).jsonItems(t), "name", "ops_wall-2")["layout"].([]any); len(l) != 0 {340		t.Fatalf("walt's save changed vera's dashboard: %v", l)341	}342343	// Validation: bounds, sizes, counts.344	many := make([]any, 61)345	for i := range many {346		many[i] = w(fmt.Sprint("w", i), 0, i, 1, 1)347	}348	noConfig := w("a", 0, 0, 1, 1)349	delete(noConfig, "config")350	big := w("a", 0, 0, 1, 1)351	big["config"] = map[string]any{"blob": strings.Repeat("x", 5000)}352	for name, layout := range map[string]any{353		"w too large": []any{w("a", 0, 0, 13, 1)}, "w zero": []any{w("a", 0, 0, 0, 1)}, "h too large": []any{w("a", 0, 0, 1, 13)},354		"off grid": []any{w("a", 7, 0, 6, 1)}, "negative x": []any{w("a", -1, 0, 1, 1)}, "y too far": []any{w("a", 0, 1001, 1, 1)},355		"duplicate id": []any{w("a", 0, 0, 1, 1), w("a", 1, 0, 1, 1)}, "empty id": []any{w("", 0, 0, 1, 1)}, "long id": []any{w(strings.Repeat("i", 41), 0, 0, 1, 1)},356		"too many": many, "no config": []any{noConfig}, "config too big": []any{big}, "no layout": nil,357	} {358		body := map[string]any{"layout": layout}359		if layout == nil {360			body = map[string]any{}361		}362		if r := vera.do("PUT", "/me/dashboards/test", body); r.Status != 400 || r.code() != "validation_failed" {363			t.Fatalf("%s: %d %s", name, r.Status, r.Raw)364		}365	}366	for _, name := range []string{"Bad", "has%20space", strings.Repeat("n", 41), "-lead"} {367		if r := vera.do("PUT", "/me/dashboards/"+name, map[string]any{"layout": []any{}}); r.Status != 400 {368			t.Fatalf("name %q: %d %s", name, r.Status, r.Raw)369		}370	}371	if h.count(`select count(*) from user_dashboards where name = 'test'`) != 0 {372		t.Fatal("an invalid layout was stored")373	}374	// At most 20 dashboards per user.375	for i := 0; i < 18; i++ {376		vera.must("PUT", fmt.Sprintf("/me/dashboards/d%d", i), map[string]any{"layout": []any{}}, 200)377	}378	if r := vera.do("PUT", "/me/dashboards/one-too-many", map[string]any{"layout": []any{}}); r.Status != 400 {379		t.Fatalf("21st dashboard: %d %s", r.Status, r.Raw)380	}381	vera.must("PUT", "/me/dashboards/d3", map[string]any{"layout": []any{w("x", 0, 0, 1, 1)}}, 200) // replacing an existing one stays possible382383	// Delete: `default` falls back to the built-in layout; an unknown name is 404.384	vera.must("DELETE", "/me/dashboards/default", nil, 204)385	vera.must("DELETE", "/me/dashboards/default", nil, 204)386	if d := rowBy(vera.must("GET", "/me/dashboards", nil, 200).jsonItems(t), "name", "default"); len(d["layout"].([]any)) < 2 {387		t.Fatalf("default after delete must be the built-in layout: %v", d)388	}389	if r := vera.do("DELETE", "/me/dashboards/nope", nil); r.Status != 404 {390		t.Fatalf("unknown dashboard: %d", r.Status)391	}392	if r := h.browser().do("GET", "/me/dashboards", nil); r.Status != 401 {393		t.Fatalf("anonymous: %d", r.Status)394	}395}396397func TestSearchRespectsPermissionsAndEscapesWildcards(t *testing.T) {398	h := newHarness(t)399	root := h.owner()400	viewer := h.member(root, "viewer", "vera", false)401	a := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second, IP: "192.168.2.50", Tags: []string{"render"}})402	b := h.seedNode(nodeSeed{Alias: "alphabet", Seen: true, Age: time.Hour, Site: "lab"})403	h.seedService(a, "pm2", "alpha-web", "online")404	pct := h.seedService(b, "pm2", "cpu_100%_burn", "errored")405	h.seedModel(a, "ollama", 8081, "alpha-model", "ok")406	h.seedAlert(a, "firing", "critical", "alpha disk full")407	h.dbExec(`insert into incidents(id, title, severity) values ($1, 'alpha outage', 'critical')`, ids.New().String())408	h.dbExec(`insert into jobs(id, template, state, node_id) values ($1, 'alpha-batch', 'running', $2)`, ids.New().String(), a)409	h.dbExec(`insert into actions(id, type, criticality, node_id, target_kind, target_id, state, idempotency_key, request_hash, timeout_s, correlation_id)410		values ($1, 'service.restart', 'medium', $2, 'node', $3, 'queued', 'k', '\x00', 60, $4)`, ids.New().String(), a, a, ids.New().String())411	win := ids.New().String()412	h.dbExec(`insert into maintenance_windows(id, title, starts_at, ends_at) values ($1, 'alpha firmware', now(), now() + interval '1 hour')`, win)413	h.dbExec(`insert into maintenance_notes(id, window_id, author_id, body) values ($1, $2, $3, 'replaced the alpha fan'), ($4, $2, $3, 'alpha secret note')`,414		ids.New().String(), win, h.userIDOf("root"), ids.New().String())415	h.dbExec(`update maintenance_notes set deleted_at = now() where body = 'alpha secret note'`)416417	kindsOf := func(c *client, q string) map[string]int {418		out := map[string]int{}419		for _, r := range c.must("GET", "/search?limit=50&q="+url.QueryEscape(q), nil, 200).jsonItems(t) {420			out[r["kind"].(string)]++421		}422		return out423	}424	got := kindsOf(root, "ALPHA")425	want := map[string]int{"node": 2, "service": 1, "alert": 1, "action": 1, "incident": 1, "job": 1, "inference_server": 1, "maintenance_window": 1, "maintenance_note": 1}426	if fmt.Sprint(got) != fmt.Sprint(want) {427		t.Fatalf("search kinds: %v, want %v", got, want)428	}429	res := root.must("GET", "/search?q=alpha", nil, 200).jsonObj(t)430	items := res["items"].([]any)431	first := items[0].(map[string]any)432	if res["query"] != "alpha" || first["kind"] != "node" || first["title"] != "alpha" || first["score"] != 1.0 || first["freshness"] != "fresh" || first["node_id"] != a {433		t.Fatalf("an exact match on a node alias ranks first: %v", first)434	}435	for i := 1; i < len(items); i++ {436		if items[i].(map[string]any)["score"].(float64) > items[i-1].(map[string]any)["score"].(float64) {437			t.Fatalf("results must be ordered by relevance: %v", items)438		}439	}440441	// Wildcards typed by the user are literal characters.442	for q, want := range map[string]int{"%": 1, "_": 1, "100%": 1, "100%_b": 1, "1_0": 0, "%%": 0, `\`: 0, "a%a": 0} {443		rs := root.must("GET", "/search?limit=50&q="+url.QueryEscape(q), nil, 200).jsonItems(t)444		if len(rs) != want || (want == 1 && rs[0]["id"] != pct) {445			t.Fatalf("q=%q: %d results %v, want %d", q, len(rs), rs, want)446		}447	}448	// An injection attempt is just a string that matches nothing.449	if rs := root.must("GET", "/search?q="+url.QueryEscape(`'; drop table nodes; --`), nil, 200).jsonItems(t); len(rs) != 0 || h.count(`select count(*) from nodes`) != 2 {450		t.Fatalf("injection: %v", rs)451	}452	// A private value is not searchable: an address must not be confirmable through search.453	if rs := root.must("GET", "/search?q=192.168.2.50", nil, 200).jsonItems(t); len(rs) != 0 {454		t.Fatalf("search by IP address: %v", rs)455	}456	if rs := root.must("GET", "/search?q=alpha&kinds=service,job&limit=1", nil, 200).jsonItems(t); len(rs) != 1 || rs[0]["kind"] != "service" {457		t.Fatalf("kinds + limit: %v", rs)458	}459	if rs := root.must("GET", "/search?q=render", nil, 200).jsonItems(t); len(rs) != 1 || rs[0]["title"] != "alpha" || rs[0]["score"].(float64) >= 0.5 {460		t.Fatalf("match through a tag: %v", rs)461	}462	for _, bad := range []string{"q=", "q=" + strings.Repeat("x", 121), "q=a&limit=0", "q=a&limit=51", "q=a&kinds=users", ""} {463		if r := root.do("GET", "/search?"+bad, nil); r.Status != 400 {464			t.Fatalf("%q: %d %s", bad, r.Status, r.Raw)465		}466	}467468	// Each kind needs its own permission; a scoped deny hides what belongs to that node.469	if got := kindsOf(viewer, "alpha"); fmt.Sprint(got) != fmt.Sprint(want) {470		t.Fatalf("viewer search: %v", got)471	}472	vera := h.userIDOf("vera")473	root.grant("user", vera, "services.read", "", "", "deny")474	root.grant("user", vera, "alerts.read", "", "", "deny")475	root.grant("user", vera, "maintenance.read", "", "", "deny")476	root.grant("user", vera, "nodes.read", "node", b, "deny")477	root.grant("user", vera, "models.read", "node", a, "deny")478	if got := kindsOf(viewer, "alpha"); fmt.Sprint(got) != fmt.Sprint(map[string]int{"node": 1, "action": 1, "job": 1}) {479		t.Fatalf("viewer search after denies: %v", got)480	}481	if rs := viewer.must("GET", "/search?q=alpha&kinds=service", nil, 200).jsonItems(t); len(rs) != 0 {482		t.Fatalf("asking for a kind one cannot read returns nothing: %v", rs)483	}484	// The same scoped deny applies to every other read.485	if r := viewer.do("GET", "/nodes/"+b, nil); r.Status != 404 {486		t.Fatalf("node hidden by a scoped deny: %d", r.Status)487	}488	if got := viewer.walkPages("/nodes", 1, "alias"); strings.Join(got, ",") != "alpha" {489		t.Fatalf("list after a scoped deny: %v", got)490	}491	if r := viewer.do("GET", "/nodes/"+b+"/metrics?metrics=cpu.usage", nil); r.Status != 404 {492		t.Fatalf("metrics of a hidden node: %d", r.Status)493	}494	if o := viewer.must("GET", "/cluster/overview", nil, 200).jsonObj(t); dig(o, "nodes", "total") != 1.0 || o["services"] != nil {495		t.Fatalf("overview after denies: %v", o)496	}497	if topo := viewer.must("GET", "/network/topology", nil, 200).jsonObj(t); strings.Contains(fmt.Sprint(topo), b) {498		t.Fatalf("topology shows a hidden node: %v", topo)499	}500	if r := viewer.do("GET", "/services", nil); r.Status != 403 {501		t.Fatalf("services without services.read: %d", r.Status)502	}503}504505func TestActivityIsARedactedProjectionOfTheAuditLog(t *testing.T) {506	h := newHarness(t)507	root := h.owner()508	viewer := h.member(root, "viewer", "vera", false)509	n := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second})510	other := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Second})511	if root.login().Status != 200 || viewer.login().Status != 200 { // two sign-ins for the feed512		t.Fatal("sign-in failed")513	}514	rootID, _ := uuid.Parse(h.userIDOf("root"))515	ip := netip.MustParseAddr("203.0.113.77")516	add := func(action, targetKind, targetID string, outcome audit.Outcome, params map[string]any) {517		t.Helper()518		if _, err := audit.Append(context.Background(), h.Pool, audit.Entry{ActorKind: "user", ActorID: &rootID, ActorLabel: "root", IP: &ip,519			Action: action, TargetKind: targetKind, TargetID: targetID, Outcome: outcome, Detail: "free text with internal-detail", Params: params}); err != nil {520			t.Fatal(err)521		}522	}523	add("action.create", "node", n, audit.Success, map[string]any{"type": "service.restart", "justification": "param-only-value"})524	add("action.approve", "action", ids.New().String(), audit.Success, map[string]any{"node_id": n})525	add("alert.acknowledge", "alert", ids.New().String(), audit.Success, nil)526	add("alert.resolve", "alert", ids.New().String(), audit.Success, nil)527	add("incident.create", "incident", ids.New().String(), audit.Success, nil)528	add("maintenance.window.create", "maintenance_window", ids.New().String(), audit.Success, nil)529	add("maintenance.note.create", "maintenance_note", ids.New().String(), audit.Success, nil)530	add("action.create", "node", other, audit.Denied, nil)531	add("user.update", "user", ids.New().String(), audit.Success, nil)     // administration: never in the feed532	add("setting.update", "setting", "x", audit.Success, nil)              // idem533	add("authz.denied", "node", n, audit.Denied, map[string]any{"a": "b"}) // idem534535	raw := string(viewer.must("GET", "/activity?limit=200", nil, 200).Raw)536	for _, leak := range []string{"203.0.113.77", "internal-detail", "param-only-value", "justification", "session", "device", "correlation"} {537		if strings.Contains(raw, leak) {538			t.Fatalf("activity leaks %q: %s", leak, raw)539		}540	}541	items := viewer.must("GET", "/activity?limit=200", nil, 200).jsonItems(t)542	count := map[string]int{}543	for _, it := range items {544		count[it["kind"].(string)]++545		for k := range it {546			if !strings.Contains("id ts kind actor_display_name summary target_kind target_id target_label outcome", k) {547				t.Fatalf("unexpected field %q in an activity item: %v", k, it)548			}549		}550	}551	if count["action"] != 3 || count["alert_ack"] != 1 || count["alert_resolve"] != 1 || count["incident"] != 1 || count["maintenance"] != 1 ||552		count["maintenance_note"] != 1 || count["login"] != 2 || len(count) != 7 {553		t.Fatalf("activity kinds: %v", count)554	}555	newest := items[0]556	if newest["kind"] != "action" || newest["outcome"] != "denied" || newest["target_label"] != "bravo" || newest["actor_display_name"] != "Root Owner" ||557		newest["summary"] != "Action: create" || newest["target_id"] != other {558		t.Fatalf("most recent item first: %v", newest)559	}560	if ack := rowBy(items, "kind", "alert_ack"); ack["summary"] != "Alert acknowledged" || ack["target_kind"] != "alert" {561		t.Fatalf("ack item: %v", ack)562	}563	if note := rowBy(items, "kind", "maintenance_note"); note["summary"] != "Maintenance note: create" {564		t.Fatalf("note item: %v", note)565	}566	if login := rowBy(items, "kind", "login"); login["summary"] != "Signed in" || login["target_id"] != nil || login["actor_display_name"] == nil {567		t.Fatalf("login item: %v", login)568	}569570	// Pagination walks every item once; filters narrow the feed.571	var allIDs []string572	for _, it := range items {573		allIDs = append(allIDs, it["id"].(string))574	}575	wantExactlyOnce(t, "paged activity", viewer.walkPages("/activity", 3, "id"), allIDs...)576	wantExactlyOnce(t, "kind filter", viewer.walkPages("/activity?kind=incident", 3, "kind"), "incident")577	if got := viewer.walkPages("/activity?node_id="+n, 10, "summary"); strings.Join(got, "|") != "Action: approve|Action: create" {578		t.Fatalf("node filter (target or node parameter): %v", got)579	}580	for _, bad := range []string{"kind=secret", "cursor=xyz", "limit=500", "node_id=x"} {581		if r := viewer.do("GET", "/activity?"+bad, nil); r.Status != 400 {582			t.Fatalf("%s: %d", bad, r.Status)583		}584	}585586	// Items are only returned when the caller can read the underlying resource.587	vera := h.userIDOf("vera")588	root.grant("user", vera, "actions.read", "", "", "deny")589	root.grant("user", vera, "alerts.read", "", "", "deny")590	after := map[string]int{}591	for _, it := range viewer.must("GET", "/activity?limit=200", nil, 200).jsonItems(t) {592		after[it["kind"].(string)]++593	}594	if after["action"] != 0 || after["alert_ack"] != 0 || after["alert_resolve"] != 0 || after["incident"] != 0 || after["maintenance"] != 1 || after["login"] != 2 {595		t.Fatalf("activity after denies: %v", after)596	}597	if got := viewer.walkPages("/activity?kind=action", 10, "id"); len(got) != 0 {598		t.Fatalf("asking for a kind one cannot read: %v", got)599	}600}601
Files main
Menu

Theme