SPB Git

Repository Public

cluster_nodes_test.go 30.5 KB 567 lines go

1package api_test23import (4	"context"5	"fmt"6	"net/url"7	"strings"8	"testing"9	"time"1011	"clustr.spboucher.ai/mcc/internal/ids"12	"clustr.spboucher.ai/mcc/internal/protocol"13)1415const gib = int64(1) << 301617// dig walks nested JSON objects; a missing member is nil.18func dig(m map[string]any, path ...string) any {19	var cur any = m20	for _, p := range path {21		o, ok := cur.(map[string]any)22		if !ok {23			return nil24		}25		cur = o[p]26	}27	return cur28}2930func wantNum(t *testing.T, m map[string]any, want float64, path ...string) {31	t.Helper()32	if got, ok := dig(m, path...).(float64); !ok || got != want {33		t.Fatalf("%s = %v, want %v", strings.Join(path, "."), dig(m, path...), want)34	}35}3637func (h *harness) seedVolume(nodeID, mount, device, fs, kind string, total, used int64, gone bool) string {38	h.t.Helper()39	id := ids.New().String()40	var remote any41	if kind == "network" {42		remote = "//192.168.2.173/share"43	}44	h.dbExec(`insert into volumes(id, node_id, mount, device, fs, kind, total_bytes, used_bytes, smart_status, remote, last_seen_at, gone_at)45		values ($1,$2,$3,$4,$5,$6,$7,$8,'Verified',$9, now(), case when $10 then now() end)`, id, nodeID, mount, device, fs, kind, total, used, remote, gone)46	return id47}4849func (h *harness) seedModel(nodeID, kind string, port int, model, health string) string {50	h.t.Helper()51	id := ids.New().String()52	h.dbExec(`insert into inference_servers(id, node_id, kind, port, model, quantization, pid, rss_bytes, uptime_s, health, health_at, health_latency_ms,53		listed_models, last_seen_at) values ($1,$2,$3,$4,$5,'4bit',777,5368709120,86400,$6, now(), 4.2, $7, now())`,54		id, nodeID, kind, port, model, health, []string{model})55	return id56}5758func (h *harness) seedAlert(nodeID, state, severity, summary string) string {59	h.t.Helper()60	var rule string61	err := h.Pool.QueryRow(context.Background(), `select id::text from alert_rules where name = 'seed rule'`).Scan(&rule)62	if err != nil {63		rule = ids.New().String()64		h.dbExec(`insert into alert_rules(id, name, kind) values ($1, 'seed rule', 'threshold')`, rule)65	}66	id := ids.New().String()67	h.dbExec(`insert into alerts(id, rule_id, node_id, fingerprint, state, severity, summary, started_at, fired_at, resolved_at, last_eval_at)68		values ($1,$2,$3,$4,$5::alert_state,$6::severity,$7, now() - interval '10 minutes', now() - interval '9 minutes',69			case when $5 = 'resolved' then now() end, now())`, id, rule, nodeID, "fp-"+id, state, severity, summary)70	return id71}7273func TestClusterOverviewCountsOnlyFreshNodes(t *testing.T) {74	h := newHarness(t)75	root := h.owner()76	viewer := h.member(root, "viewer", "vera", false)7778	a := h.seedNode(nodeSeed{Alias: "alpha", Cores: 16, Mem: 64 * gib, Seen: true, Age: 5 * time.Second,79		Values: map[string]any{"cpu.usage": 10.0, "mem.used": 16 * gib, "mem.total": 64 * gib}})80	b := h.seedNode(nodeSeed{Alias: "bravo", Kind: "linux", Cores: 8, Mem: 32 * gib, Seen: true, Age: 10 * time.Second,81		Values: map[string]any{"mem.used": 4 * gib}}) // mem.total comes from the inventory82	c := h.seedNode(nodeSeed{Alias: "charlie", Cores: 12, Mem: 32 * gib, Seen: true, Age: 60 * time.Second, Maintenance: true,83		Values: map[string]any{"mem.used": 30 * gib, "mem.total": 32 * gib}}) // stale84	d := h.seedNode(nodeSeed{Alias: "delta", Cores: 10, Mem: 16 * gib, Seen: true, Age: 10 * time.Minute,85		Values: map[string]any{"mem.used": 15 * gib, "mem.total": 16 * gib}}) // offline86	h.seedNode(nodeSeed{Alias: "echo", Manual: true}) // registered, never seen87	f := h.seedNode(nodeSeed{Alias: "foxtrot", Cores: 99, Mem: 999 * gib, Seen: true, Age: time.Second, Values: map[string]any{"mem.used": gib}})88	h.dbExec(`update nodes set retired_at = now() where id = $1`, f)8990	// alpha: two APFS volumes of one container (size counted once), an external disk and a gone volume (both ignored).91	h.seedVolume(a, "/", "/dev/disk3s1s1", "apfs", "internal", 1000, 300, false)92	h.seedVolume(a, "/System/Volumes/Data", "/dev/disk3s5", "apfs", "internal", 1000, 100, false)93	h.seedVolume(a, "/Volumes/Ext", "/dev/disk7s1", "apfs", "external", 5000, 5000, false)94	h.seedVolume(a, "/Volumes/Old", "/dev/disk9s1", "apfs", "internal", 7000, 7000, true)95	h.seedVolume(b, "/", "/dev/sda1", "ext4", "internal", 500, 100, false)96	h.seedVolume(b, "/data", "/dev/sdb1", "ext4", "internal", 200, 50, false)97	h.seedVolume(c, "/", "/dev/disk3s1", "apfs", "internal", 999, 999, false) // stale node: not current9899	h.seedService(a, "pm2", "web", "online")100	h.seedService(a, "pm2", "api", "online")101	h.seedService(a, "launchd", "worker", "errored")102	gone := h.seedService(a, "pm2", "old", "online")103	h.dbExec(`update services set gone_at = now() where id = $1`, gone)104	h.seedService(c, "pm2", "batch", "stopped")105	h.seedService(d, "pm2", "ghost", "online") // node offline: state not known now106	h.seedModel(a, "mlx_lm", 8081, "mlx-community/Qwen", "ok")107	h.seedModel(c, "ollama", 11434, "llama3", "ok") // healthy when last seen, but the node is stale108109	h.dbExec(`insert into nas_targets(id, name, host, state, state_at) values110		($1, 'nas-live', '192.168.2.10', 'online', now()), ($2, 'nas-old', '192.168.2.11', 'online', now() - interval '2 hours'),111		($3, 'nas-new', '192.168.2.12', 'unknown', null)`, ids.New().String(), ids.New().String(), ids.New().String())112	h.dbExec(`insert into backups(id, name, kind, last_status, last_run_at, last_success_at) values113		($1, 'pg', 'postgres', 'ok', now(), now() - interval '1 hour'), ($2, 'app', 'app', 'failed', now(), now() - interval '3 days'),114		($3, 'new', 'app', 'unknown', null, null)`, ids.New().String(), ids.New().String(), ids.New().String())115	h.seedAlert(a, "firing", "warning", "disk almost full")116	h.seedAlert(a, "acknowledged", "critical", "service down")117	h.seedAlert(b, "resolved", "critical", "old trouble")118	h.seedAlert(b, "pending", "info", "not firing yet")119	h.dbExec(`insert into incidents(id, title, severity, state) values ($1, 'Outage', 'critical', 'open'), ($2, 'Past', 'warning', 'resolved')`,120		ids.New().String(), ids.New().String())121	h.dbExec(`insert into jobs(id, template, state) values ($1,'t','queued'), ($2,'t','running'), ($3,'t','running'), ($4,'t','succeeded')`,122		ids.New().String(), ids.New().String(), ids.New().String(), ids.New().String())123	h.dbExec(`insert into actions(id, type, criticality, node_id, target_kind, target_id, state, idempotency_key, request_hash, timeout_s, correlation_id)124		values ($1, 'inventory.refresh', 'low', $2, 'node', $3, 'succeeded', 'k1', '\x00', 60, $4)`, ids.New().String(), a, a, ids.New().String())125126	o := viewer.must("GET", "/cluster/overview", nil, 200).jsonObj(t)127	for path, want := range map[string]float64{128		"nodes.total": 5, "nodes.fresh": 2, "nodes.stale": 1, "nodes.offline": 2, "nodes.maintenance": 1,129		"capacity.cpu_cores.total": 24, "capacity.cpu_cores.coverage": 2,130		"capacity.memory.total": float64(96 * gib), "capacity.memory.used": float64(20 * gib), "capacity.memory.coverage": 2,131		"capacity.storage.total": 1700, "capacity.storage.used": 550, "capacity.storage.coverage": 2,132		"alerts.warning": 1, "alerts.critical": 1, "alerts.info": 0, "open_incidents": 1, "jobs.queued": 1, "jobs.running": 2,133		"services.online": 2, "services.errored": 1, "services.stopped": 1, "services.unknown": 1,134		"models.total": 2, "models.online": 1, "nas.total": 3, "nas.online": 1, "nas.offline": 0, "nas.unknown": 2,135		"backups.total": 3, "backups.ok": 1, "backups.failed": 1, "backups.unknown": 1,136	} {137		wantNum(t, o, want, strings.Split(path, ".")...)138	}139	if _, has := dig(o, "capacity", "cpu_cores").(map[string]any)["used"]; has {140		t.Fatal("cpu_cores must not report a usage")141	}142	if dig(o, "backups", "last_success_at") == nil {143		t.Fatal("last successful backup missing")144	}145	acts, _ := o["recent_actions"].([]any)146	if len(acts) != 1 || dig(acts[0].(map[string]any), "target", "id") != a || dig(acts[0].(map[string]any), "outcome_unknown") != false {147		t.Fatalf("recent_actions: %v", o["recent_actions"])148	}149150	// With no fresh node at all nothing is summed, and usage is absent rather than zero.151	h.dbExec(`update node_state set received_at = now() - interval '1 hour'`)152	h.dbExec(`update nodes set last_seen_at = now() - interval '1 hour'`)153	o = viewer.must("GET", "/cluster/overview", nil, 200).jsonObj(t)154	wantNum(t, o, 0, "nodes", "fresh")155	wantNum(t, o, 5, "nodes", "offline")156	wantNum(t, o, 0, "capacity", "memory", "coverage")157	wantNum(t, o, 0, "capacity", "memory", "total")158	if _, has := dig(o, "capacity", "memory").(map[string]any)["used"]; has {159		t.Fatal("memory usage reported although no node is fresh")160	}161	wantNum(t, o, 5, "services", "unknown")162	wantNum(t, o, 0, "models", "online")163164	// A section the caller may not read is omitted, not emptied.165	root.grant("user", h.userIDOf("vera"), "actions.read", "", "", "deny")166	root.grant("user", h.userIDOf("vera"), "alerts.read", "", "", "deny")167	o = viewer.must("GET", "/cluster/overview", nil, 200).jsonObj(t)168	for _, k := range []string{"recent_actions", "alerts", "open_incidents"} {169		if _, has := o[k]; has {170			t.Fatalf("overview exposes %s without the permission", k)171		}172	}173	if o["services"] == nil || o["nas"] == nil {174		t.Fatal("readable sections must stay")175	}176}177178func TestNodeListFiltersSortAndPagination(t *testing.T) {179	h := newHarness(t)180	root := h.owner()181	cpu := func(v float64) map[string]any {182		return map[string]any{"cpu.usage": v, "mem.used": gib, "mem.total": 4 * gib}183	}184	h.seedNode(nodeSeed{Alias: "Alpha", Seen: true, Age: 2 * time.Second, Values: cpu(50), Tags: []string{"gpu", "studio"}})185	h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: 3 * time.Second, Values: cpu(10), Site: "lab", Kind: "linux"})186	h.seedNode(nodeSeed{Alias: "Charlie", Seen: true, Age: 50 * time.Second, Values: cpu(90), Site: "lab", Tags: []string{"gpu"}})187	h.seedNode(nodeSeed{Alias: "delta", Seen: true, Age: time.Hour, Values: cpu(99), Role: "gateway"}) // offline: its 99 % is not current188	h.seedNode(nodeSeed{Alias: "Echo", Manual: true})189	h.seedNode(nodeSeed{Alias: "foxtrot", Seen: true, Age: 4 * time.Second, Values: map[string]any{}, Maintenance: true})190	gone := h.seedNode(nodeSeed{Alias: "golf", Seen: true, Age: time.Second, Values: cpu(1)})191	h.dbExec(`update nodes set retired_at = now() where id = $1`, gone)192	all := []string{"Alpha", "bravo", "Charlie", "delta", "Echo", "foxtrot"}193194	// Default order: alias, case-insensitive; every row exactly once whatever the page size.195	for _, limit := range []int{1, 2, 4, 50} {196		got := root.walkPages("/nodes", limit, "alias")197		if strings.Join(got, ",") != strings.Join(all, ",") {198			t.Fatalf("limit %d: order %v", limit, got)199		}200	}201	desc := root.walkPages("/nodes?sort=alias&order=desc", 2, "alias")202	if strings.Join(desc, ",") != "foxtrot,Echo,delta,Charlie,bravo,Alpha" {203		t.Fatalf("descending: %v", desc)204	}205	// Sorting on a live value: nodes without a current value come last, each row still exactly once.206	byCPU := root.walkPages("/nodes?sort=cpu&order=desc", 2, "alias")207	wantExactlyOnce(t, "sort=cpu", byCPU, all...)208	if strings.Join(byCPU[:3], ",") != "Charlie,Alpha,bravo" {209		t.Fatalf("cpu desc: %v", byCPU)210	}211	if asc := root.walkPages("/nodes?sort=cpu&order=asc", 2, "alias"); strings.Join(asc[:3], ",") != "bravo,Alpha,Charlie" || len(asc) != 6 {212		t.Fatalf("cpu asc (nulls last): %v", asc)213	}214	for _, s := range []string{"site", "role", "last_seen", "memory", "disk"} {215		for _, o := range []string{"asc", "desc"} {216			wantExactlyOnce(t, s+" "+o, root.walkPages("/nodes?sort="+s+"&order="+o, 2, "alias"), all...)217		}218	}219	if seen := root.walkPages("/nodes?sort=last_seen&order=desc", 3, "alias"); strings.Join(seen[:5], ",") != "Alpha,bravo,foxtrot,Charlie,delta" || seen[5] != "Echo" {220		t.Fatalf("last_seen desc: %v", seen)221	}222223	filters := map[string][]string{224		"site=lab":                  {"bravo", "Charlie"},225		"role=gateway":              {"delta"},226		"tag=gpu":                   {"Alpha", "Charlie"},227		"kind=linux":                {"bravo"},228		"freshness=fresh":           {"Alpha", "bravo", "foxtrot"},229		"freshness=stale":           {"Charlie"},230		"freshness=offline":         {"delta", "Echo"},231		"maintenance=true":          {"foxtrot"},232		"q=CHAR":                    {"Charlie"},233		"q=" + url.QueryEscape("%"): {},234		"q=_":                       {},235		"q=host-al":                 {"Alpha"},236		"retired=true":              append([]string{"golf"}, all...),237		"site=lab&tag=gpu":          {"Charlie"},238		"freshness=fresh&q=rav":     {"bravo"},239	}240	for f, want := range filters {241		wantExactlyOnce(t, f, root.walkPages("/nodes?"+f, 2, "alias"), want...)242	}243244	// Headline: present for a reporting node, absent for an offline or never-seen one.245	rows := root.must("GET", "/nodes", nil, 200).jsonItems(t)246	if hl, _ := rowBy(rows, "alias", "Alpha")["headline"].(map[string]any); hl == nil || hl["cpu_usage_pct"] != 50.0 || hl["mem_used_pct"] != 25.0 {247		t.Fatalf("headline of a fresh node: %v", rowBy(rows, "alias", "Alpha"))248	}249	for _, alias := range []string{"delta", "Echo"} {250		if r := rowBy(rows, "alias", alias); r["headline"] != nil || r["freshness"] != "offline" {251			t.Fatalf("%s must be offline without headline: %v", alias, r)252		}253	}254	if r := rowBy(rows, "alias", "Echo"); r["observed_at"] != nil || r["last_seen_at"] != nil || r["cpu_cores"] != nil || r["model"] != nil || r["registered_via"] != "manual" {255		t.Fatalf("never-seen node must have no observation and no hardware: %v", r)256	}257258	// Invalid input is refused with a structured error.259	first := root.must("GET", "/nodes?limit=2", nil, 200).nextCursor(t)260	for _, bad := range []string{"limit=0", "limit=201", "sort=bogus", "order=up", "freshness=weird", "kind=windows", "cursor=garbage",261		"cursor=" + url.QueryEscape(first) + "&sort=cpu", "cursor=" + url.QueryEscape(first) + "&order=desc", "q=" + strings.Repeat("x", 121)} {262		if r := root.do("GET", "/nodes?"+bad, nil); r.Status != 400 || r.code() != "validation_failed" {263			t.Fatalf("%s: %d %s", bad, r.Status, r.Raw)264		}265	}266}267268func TestNodePrivateFieldsNeedPermission(t *testing.T) {269	h := newHarness(t)270	root := h.owner()271	viewer := h.member(root, "viewer", "vera", false)272	operator := h.member(root, "operator", "oscar", true)273	inv := map[string]any{"hardware_uuid": "HW-alpha", "serial_number": "SN-C02XYZ", "cpu_levels": []any{map[string]any{"name": "Performance", "cores": 12}},274		"interfaces": []any{275			map[string]any{"name": "en0", "kind": "ethernet", "mac": "aa:bb:cc:dd:ee:01", "addrs": []string{"192.168.2.50/24", "fe80::1/64"}, "media": "1000baseT <full-duplex>", "up": true, "default": true},276			map[string]any{"name": "utun4", "kind": "tunnel", "addrs": []string{"10.67.0.9"}, "up": true}}}277	x := h.seedNode(nodeSeed{Alias: "alpha", IP: "192.168.2.50", Seen: true, Age: time.Second, Inventory: inv,278		Values: map[string]any{"net.rx{en0}": 1200.0, "net.tx{en0}": 300.0, "net.latency{gateway}": 0.4, "net.loss{gateway}": 0.0}})279	y := h.seedNode(nodeSeed{Alias: "bravo", IP: "192.168.2.51", Seen: true, Age: time.Second, Inventory: inv})280	h.seedVolume(x, "/Volumes/nas", "//192.168.2.173/share", "smbfs", "network", 100, 10, false)281	h.dbExec(`insert into nas_targets(id, name, host) values ($1, 'nas-1', '192.168.2.173')`, ids.New().String())282	secrets := []string{"192.168.2.50", "192.168.2.51", "aa:bb:cc:dd:ee:01", "SN-C02XYZ", "HW-alpha", "192.168.2.173", "fe80::1", "10.67.0.9"}283	paths := []string{"/nodes", "/nodes/" + x, "/network/interfaces", "/network/topology", "/storage/volumes", "/storage/nas", "/search?q=alpha"}284285	for _, p := range paths {286		raw := string(viewer.must("GET", p, nil, 200).Raw)287		for _, s := range secrets {288			if strings.Contains(raw, s) {289				t.Fatalf("viewer sees private value %q in %s: %s", s, p, raw)290			}291		}292	}293	// The resource itself stays readable; only the private fields are left out.294	d := viewer.must("GET", "/nodes/"+x, nil, 200).jsonObj(t)295	ifs := d["interfaces"].([]any)296	en0 := ifs[0].(map[string]any)297	if len(ifs) != 2 || en0["name"] != "en0" || en0["kind"] != "ethernet" || en0["link_speed_mbps"] != 1000.0 || en0["rx_bytes_per_s"] != 1200.0 ||298		len(en0["ipv4"].([]any)) != 0 || en0["mac"] != nil || d["hardware_uuid"] != nil || dig(d, "node", "primary_ip") != nil {299		t.Fatalf("viewer detail: %v", d)300	}301	if ifs[1].(map[string]any)["kind"] != "other" { // a utun interface is not guessed to be WireGuard302		t.Fatalf("utun must stay 'other': %v", ifs[1])303	}304	if lv := d["cpu_levels"].([]any); len(lv) != 1 || lv[0].(map[string]any)["name"] != "Performance" {305		t.Fatalf("cpu levels: %v", d["cpu_levels"])306	}307308	od := operator.must("GET", "/nodes/"+x, nil, 200).jsonObj(t)309	oen0 := od["interfaces"].([]any)[0].(map[string]any)310	if od["hardware_uuid"] != "HW-alpha" || od["serial_number"] != "SN-C02XYZ" || dig(od, "node", "primary_ip") != "192.168.2.50" ||311		oen0["mac"] != "aa:bb:cc:dd:ee:01" || fmt.Sprint(oen0["ipv4"]) != "[192.168.2.50/24]" || fmt.Sprint(oen0["ipv6"]) != "[fe80::1/64]" {312		t.Fatalf("operator detail: %v", od)313	}314	for p, want := range map[string]string{"/nodes": "192.168.2.51", "/network/interfaces": "aa:bb:cc:dd:ee:01", "/network/topology": "192.168.2.50",315		"/storage/volumes": "//192.168.2.173/share", "/storage/nas": "192.168.2.173"} {316		if raw := string(operator.must("GET", p, nil, 200).Raw); !strings.Contains(raw, want) {317			t.Fatalf("operator must see %q in %s: %s", want, p, raw)318		}319	}320321	// A grant scoped to one node extends the private view to that node only.322	root.grant("user", h.userIDOf("vera"), "nodes.private.read", "node", x, "allow")323	rows := viewer.must("GET", "/nodes", nil, 200).jsonItems(t)324	if rowBy(rows, "alias", "alpha")["primary_ip"] != "192.168.2.50" || rowBy(rows, "alias", "bravo")["primary_ip"] != nil {325		t.Fatalf("scoped allow: %v", rows)326	}327	if raw := string(viewer.must("GET", "/nodes/"+y, nil, 200).Raw); strings.Contains(raw, "192.168.2.51") || strings.Contains(raw, "aa:bb") {328		t.Fatalf("the grant on alpha leaked bravo's private fields: %s", raw)329	}330	if raw := string(viewer.must("GET", "/storage/nas", nil, 200).Raw); strings.Contains(raw, "192.168.2.173") {331		t.Fatal("a node-scoped grant must not reveal NAS hosts")332	}333334	// Topology: the LAN link carries the measurements; no hub link without a WireGuard interface or hub probe.335	topo := operator.must("GET", "/network/topology", nil, 200).jsonObj(t)336	links := topo["links"].([]any)337	if len(links) != 2 {338		t.Fatalf("want one LAN link per reporting node and no hub link: %v", links)339	}340	var lx map[string]any341	for _, l := range links {342		if l.(map[string]any)["source"] == x {343			lx = l.(map[string]any)344		}345	}346	if lx["target"] != "lan-gateway" || lx["kind"] != "lan" || lx["state"] != "up" || lx["latency_ms"] != 0.4000000059604645 && lx["latency_ms"] != 0.4 ||347		lx["latency_availability"] != "available" || lx["interface"] != "en0" {348		t.Fatalf("lan link of alpha: %v", lx)349	}350	for _, l := range links {351		if m := l.(map[string]any); m["source"] == y && (m["state"] != "unknown" || m["latency_availability"] != "unavailable" || m["latency_ms"] != nil) {352			t.Fatalf("a link that was not measured must say so: %v", m)353		}354	}355}356357func TestManualNodeRegistrationUpdateAndRetire(t *testing.T) {358	h := newHarness(t)359	root := h.owner()360	admin := h.member(root, "admin", "ada", false)361	operator := h.member(root, "operator", "oscar", false)362	viewer := h.member(root, "viewer", "vera", false)363	h.seedPartitions()364	body := map[string]any{"alias": "M4BP36", "kind": "mac", "tags": []string{"laptop", "laptop"},365		"expected": map[string]any{"model": "MacBook Pro", "cpu_cores": 14, "mem_bytes": 38654705664}}366367	for _, c := range []*client{viewer, operator} {368		if r := c.do("POST", "/nodes", body); r.Status != 403 || r.code() != "forbidden" {369			t.Fatalf("node registration without nodes.write: %d %s", r.Status, r.Raw)370		}371	}372	for name, bad := range map[string]map[string]any{373		"alias pattern": {"alias": "bad alias!", "kind": "mac"}, "alias empty": {"alias": "", "kind": "mac"},374		"kind": {"alias": "ok", "kind": "windows"}, "long site": {"alias": "ok", "kind": "mac", "site": strings.Repeat("s", 65)},375		"empty tag": {"alias": "ok", "kind": "mac", "tags": []string{""}}, "zero cores": {"alias": "ok", "kind": "mac", "expected": map[string]any{"cpu_cores": 0}},376	} {377		if r := admin.do("POST", "/nodes", bad); r.Status != 400 || r.code() != "validation_failed" {378			t.Fatalf("%s: %d %s", name, r.Status, r.Raw)379		}380	}381	n := admin.must("POST", "/nodes", body, 201).jsonObj(t)382	id := n["id"].(string)383	if n["registered_via"] != "manual" || n["freshness"] != "offline" || n["observed_at"] != nil || n["last_seen_at"] != nil ||384		n["cpu_cores"] != nil || n["model"] != nil || n["mem_bytes"] != nil || fmt.Sprint(n["tags"]) != "[laptop]" || n["site"] != "default" {385		t.Fatalf("manual node must be offline, unobserved, with declared hardware kept apart: %v", n)386	}387	var hw *string388	var expected string389	h.dbScan([]any{&hw, &expected}, `select hardware_uuid, inventory->'expected'->>'model' from nodes where id = $1`, id)390	if hw != nil || expected != "MacBook Pro" {391		t.Fatalf("stored node: hardware_uuid=%v expected=%q", hw, expected)392	}393	if r := admin.do("POST", "/nodes", map[string]any{"alias": "m4bp36", "kind": "mac"}); r.Status != 409 || r.code() != "conflict" {394		t.Fatalf("duplicate alias (case-insensitive): %d %s", r.Status, r.Raw)395	}396397	// A node without hardware_uuid and without any report is handled by every read.398	d := viewer.must("GET", "/nodes/"+id, nil, 200).jsonObj(t)399	if len(d["interfaces"].([]any)) != 0 || len(d["volumes"].([]any)) != 0 || len(d["cpu_levels"].([]any)) != 0 || d["agent"] != nil {400		t.Fatalf("detail of a never-seen node: %v", d)401	}402	latest := d["latest"].(map[string]any)403	if len(latest) < 20 {404		t.Fatalf("latest must list every unlabeled catalog metric, got %d", len(latest))405	}406	for name, v := range latest {407		if mv := v.(map[string]any); mv["availability"] != "offline" || mv["value"] != nil {408			t.Fatalf("latest[%s] of a never-seen node: %v", name, mv)409		}410	}411	if p := viewer.must("GET", "/nodes/"+id+"/processes", nil, 200).jsonObj(t); p["availability"] != "offline" || len(p["items"].([]any)) != 0 || p["observed_at"] != nil {412		t.Fatalf("processes: %v", p)413	}414	m := viewer.must("GET", "/nodes/"+id+"/metrics?metrics=cpu.usage,net.rx", nil, 200).jsonObj(t)415	if len(m["series"].([]any)) != 0 || dig(m, "availability", "cpu.usage") != "offline" || dig(m, "availability", "net.rx") != "offline" {416		t.Fatalf("metrics: %v", m)417	}418	hh := viewer.must("GET", "/nodes/"+id+"/health-history?range=1h", nil, 200).jsonObj(t)419	if hh["uptime_ratio"] != nil || len(hh["items"].([]any)) != 12 {420		t.Fatalf("health history: %v", hh)421	}422	for _, b := range hh["items"].([]any) {423		if b.(map[string]any)["state"] != "unknown" {424			t.Fatalf("a node that never reported has no known health: %v", b)425		}426	}427	if len(viewer.must("GET", "/nodes/"+id+"/services", nil, 200).jsonItems(t)) != 0 {428		t.Fatal("services of a never-seen node")429	}430	o := viewer.must("GET", "/cluster/overview", nil, 200).jsonObj(t)431	wantNum(t, o, 1, "nodes", "total")432	wantNum(t, o, 1, "nodes", "offline")433	wantNum(t, o, 0, "capacity", "cpu_cores", "coverage")434	topo := viewer.must("GET", "/network/topology", nil, 200).jsonObj(t)435	if len(topo["nodes"].([]any)) != 1 || len(topo["links"].([]any)) != 0 {436		t.Fatalf("topology: one vertex, no observed link: %v", topo)437	}438	if len(viewer.must("GET", "/network/interfaces", nil, 200).jsonItems(t)) != 0 || len(viewer.must("GET", "/storage/volumes", nil, 200).jsonItems(t)) != 0 {439		t.Fatal("no interface and no volume was observed")440	}441	if s := viewer.must("GET", "/search?q=m4bp", nil, 200).jsonItems(t); len(s) != 1 || s[0]["kind"] != "node" || s[0]["freshness"] != "offline" {442		t.Fatalf("search: %v", s)443	}444	if tr := viewer.must("GET", "/cluster/trends?metric=cpu.usage&range=1h", nil, 200).jsonObj(t); dig(tr, "series").([]any)[0].(map[string]any)["coverage"] != 0.0 {445		t.Fatalf("trends: %v", tr)446	}447448	// Update.449	if r := viewer.do("PATCH", "/nodes/"+id, map[string]any{"site": "lab"}); r.Status != 403 {450		t.Fatalf("viewer patch: %d", r.Status)451	}452	for name, bad := range map[string]map[string]any{"nothing": {}, "alias": {"alias": "no spaces"}, "tags": {"tags": []string{strings.Repeat("t", 49)}}} {453		if r := admin.do("PATCH", "/nodes/"+id, bad); r.Status != 400 {454			t.Fatalf("patch %s: %d %s", name, r.Status, r.Raw)455		}456	}457	events0 := h.count(`select count(*) from events_outbox where topic = 'node.updated'`)458	u := admin.must("PATCH", "/nodes/"+id, map[string]any{"alias": "M4BP36b", "site": "lab", "role": "gateway", "tags": []string{"a", "b"}}, 200).jsonObj(t)459	if u["alias"] != "M4BP36b" || u["site"] != "lab" || u["role"] != "gateway" || fmt.Sprint(u["tags"]) != "[a b]" || u["environment"] != "production" {460		t.Fatalf("patched node: %v", u)461	}462	admin.must("PATCH", "/nodes/"+id, map[string]any{"site": "lab"}, 200) // no change: nothing recorded463	if n := h.count(`select count(*) from audit_log where action = 'node.update' and target_id = $1 and outcome = 'success'464		and params->'alias'->>'to' = 'M4BP36b' and params->'site'->>'from' = 'default'`, id); n != 1 {465		t.Fatalf("%d node.update audit entries, want 1", n)466	}467	if n := h.count(`select count(*) from audit_log where action = 'node.create' and target_id = $1 and actor_label = 'ada'`, id); n != 1 {468		t.Fatalf("%d node.create audit entries", n)469	}470	if n := h.count(`select count(*) from events_outbox where topic = 'node.updated' and payload->>'node_id' = $1 and payload->>'change' = 'updated'`, id); n != 1 ||471		h.count(`select count(*) from events_outbox where topic = 'node.updated'`) != events0+1 {472		t.Fatalf("node.updated events after patch: %d", n)473	}474	other := admin.must("POST", "/nodes", map[string]any{"alias": "other", "kind": "linux"}, 201).jsonObj(t)["id"].(string)475	if r := admin.do("PATCH", "/nodes/"+other, map[string]any{"alias": "m4bp36B"}); r.Status != 409 {476		t.Fatalf("alias collision on rename: %d %s", r.Status, r.Raw)477	}478	if r := admin.do("PATCH", "/nodes/"+ids.New().String(), map[string]any{"site": "x"}); r.Status != 404 {479		t.Fatalf("unknown node: %d", r.Status)480	}481482	// "Adoption" is ingest's job: once an agent has spoken the same row simply becomes a live node.483	h.dbExec(`update nodes set hardware_uuid = 'HW-1', cpu_cores = 14, inventory_at = now(), last_seen_at = now() where id = $1`, id)484	h.dbExec(`insert into node_state(node_id, ts, received_at, values) values ($1, now(), now(), '{"cpu.usage": 3.5}')`, id)485	agent := h.seedAgent(id, time.Second)486	alert := h.seedAlert(id, "firing", "warning", "hot")487	if n := admin.must("GET", "/nodes/"+id, nil, 200).jsonObj(t); dig(n, "node", "freshness") != "fresh" || dig(n, "node", "cpu_cores") != 14.0 ||488		dig(n, "agent", "id") != agent || dig(n, "latest", "cpu.usage", "value") != 3.5 || dig(n, "node", "open_alerts") != 1.0 {489		t.Fatalf("adopted node: %v", n)490	}491492	// Retirement: step-up, then agent revoked, alerts resolved, history kept.493	h.expireStepUp("ada")494	if r := admin.do("DELETE", "/nodes/"+id, nil); r.Status != 403 || r.code() != "step_up_required" {495		t.Fatalf("retire without step-up: %d %s", r.Status, r.Raw)496	}497	if r := operator.do("DELETE", "/nodes/"+id, nil); r.Status != 403 || r.code() != "forbidden" {498		t.Fatalf("retire by operator: %d %s", r.Status, r.Raw)499	}500	admin.stepUp()501	admin.must("DELETE", "/nodes/"+id, nil, 204)502	if h.count(`select count(*) from nodes where id = $1 and retired_at is not null`, id) != 1 ||503		h.count(`select count(*) from agents where id = $1 and revoked_at is not null and revoked_reason = 'node retired'`, agent) != 1 ||504		h.count(`select count(*) from alerts where id = $1 and state = 'resolved' and resolved_at is not null`, alert) != 1 ||505		h.count(`select count(*) from timeline_events where alert_id = $1 and kind = 'resolved'`, alert) != 1 {506		t.Fatal("retirement did not revoke the agent and resolve the alerts")507	}508	if h.count(`select count(*) from audit_log where action = 'node.retire' and target_id = $1 and (params->>'agents_revoked')::int = 1509		and (params->>'alerts_resolved')::int = 1`, id) != 1 {510		t.Fatal("retirement not audited")511	}512	if h.count(`select count(*) from events_outbox where topic = 'node.updated' and payload->>'node_id' = $1 and payload->>'change' = 'retired'`, id) != 1 ||513		h.count(`select count(*) from events_outbox where topic = 'alert.changed' and payload->>'alert_id' = $1 and payload->>'state' = 'resolved'`, alert) != 1 {514		t.Fatal("retirement events missing")515	}516	admin.must("DELETE", "/nodes/"+id, nil, 204) // idempotent517	if h.count(`select count(*) from audit_log where action = 'node.retire' and target_id = $1`, id) != 1 {518		t.Fatal("a repeated retirement must not be recorded twice")519	}520	if got := admin.walkPages("/nodes", 50, "alias"); strings.Join(got, ",") != "other" {521		t.Fatalf("retired node still listed: %v", got)522	}523	if got := admin.walkPages("/nodes?retired=true", 50, "alias"); len(got) != 2 {524		t.Fatalf("retired=true: %v", got)525	}526	if r := admin.do("PATCH", "/nodes/"+id, map[string]any{"site": "x"}); r.Status != 409 {527		t.Fatalf("patch of a retired node: %d %s", r.Status, r.Raw)528	}529	if r := admin.do("POST", "/nodes", map[string]any{"alias": "M4BP36b", "kind": "mac"}); r.Status != 409 || !strings.Contains(string(r.Raw), "retired") {530		t.Fatalf("alias of a retired node: %d %s", r.Status, r.Raw)531	}532	if n := admin.must("GET", "/nodes/"+id, nil, 200).jsonObj(t); dig(n, "node", "retired_at") == nil || n["agent"] != nil {533		t.Fatalf("a retired node stays readable, without active agent: %v", n)534	}535}536537func TestNodeProcesses(t *testing.T) {538	h := newHarness(t)539	root := h.owner()540	procs := []protocol.Process{{PID: 1, Name: "launchd", User: "root", CPU: 0.1, RSS: 1 << 20}, {PID: 4242, Name: "node", User: "sp", CPU: 180.5, RSS: 1 << 30}}541	fresh := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second, Processes: procs})542	stale := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Minute, Processes: procs})543	off := h.seedNode(nodeSeed{Alias: "charlie", Seen: true, Age: time.Hour, Processes: procs})544	none := h.seedNode(nodeSeed{Alias: "delta", Seen: true, Age: time.Second})545546	p := root.must("GET", "/nodes/"+fresh+"/processes", nil, 200).jsonObj(t)547	items := p["items"].([]any)548	if p["availability"] != "available" || p["freshness"] != "fresh" || len(items) != 2 || p["observed_at"] == nil {549		t.Fatalf("fresh: %v", p)550	}551	if it := items[1].(map[string]any); it["pid"] != 4242.0 || it["name"] != "node" || it["user"] != "sp" || it["cpu_pct"] != 180.5 || it["mem_bytes"] != float64(1<<30) || it["command"] != nil {552		t.Fatalf("process mapping: %v", it)553	}554	if p := root.must("GET", "/nodes/"+stale+"/processes", nil, 200).jsonObj(t); p["availability"] != "stale" || len(p["items"].([]any)) != 2 {555		t.Fatalf("stale: %v", p)556	}557	if p := root.must("GET", "/nodes/"+off+"/processes", nil, 200).jsonObj(t); p["availability"] != "offline" || len(p["items"].([]any)) != 0 {558		t.Fatalf("an old snapshot of an offline node must not be shown as its processes: %v", p)559	}560	if p := root.must("GET", "/nodes/"+none+"/processes", nil, 200).jsonObj(t); p["availability"] != "unavailable" || len(p["items"].([]any)) != 0 {561		t.Fatalf("no snapshot: %v", p)562	}563	if r := root.do("GET", "/nodes/"+ids.New().String()+"/processes", nil); r.Status != 404 {564		t.Fatalf("unknown node: %d", r.Status)565	}566}567
Files main
Menu

Theme