Repository Public
1package api_test23import (4 "context"5 "fmt"6 "net/url"7 "strings"8 "testing"9 "time"1011 "clustr.spboucher.ai/mcc/internal/ids"12 "clustr.spboucher.ai/mcc/internal/protocol"13)1415const gib = int64(1) << 301617// dig walks nested JSON objects; a missing member is nil.18func dig(m map[string]any, path ...string) any {19 var cur any = m20 for _, p := range path {21 o, ok := cur.(map[string]any)22 if !ok {23 return nil24 }25 cur = o[p]26 }27 return cur28}2930func wantNum(t *testing.T, m map[string]any, want float64, path ...string) {31 t.Helper()32 if got, ok := dig(m, path...).(float64); !ok || got != want {33 t.Fatalf("%s = %v, want %v", strings.Join(path, "."), dig(m, path...), want)34 }35}3637func (h *harness) seedVolume(nodeID, mount, device, fs, kind string, total, used int64, gone bool) string {38 h.t.Helper()39 id := ids.New().String()40 var remote any41 if kind == "network" {42 remote = "//192.168.2.173/share"43 }44 h.dbExec(`insert into volumes(id, node_id, mount, device, fs, kind, total_bytes, used_bytes, smart_status, remote, last_seen_at, gone_at)45 values ($1,$2,$3,$4,$5,$6,$7,$8,'Verified',$9, now(), case when $10 then now() end)`, id, nodeID, mount, device, fs, kind, total, used, remote, gone)46 return id47}4849func (h *harness) seedModel(nodeID, kind string, port int, model, health string) string {50 h.t.Helper()51 id := ids.New().String()52 h.dbExec(`insert into inference_servers(id, node_id, kind, port, model, quantization, pid, rss_bytes, uptime_s, health, health_at, health_latency_ms,53 listed_models, last_seen_at) values ($1,$2,$3,$4,$5,'4bit',777,5368709120,86400,$6, now(), 4.2, $7, now())`,54 id, nodeID, kind, port, model, health, []string{model})55 return id56}5758func (h *harness) seedAlert(nodeID, state, severity, summary string) string {59 h.t.Helper()60 var rule string61 err := h.Pool.QueryRow(context.Background(), `select id::text from alert_rules where name = 'seed rule'`).Scan(&rule)62 if err != nil {63 rule = ids.New().String()64 h.dbExec(`insert into alert_rules(id, name, kind) values ($1, 'seed rule', 'threshold')`, rule)65 }66 id := ids.New().String()67 h.dbExec(`insert into alerts(id, rule_id, node_id, fingerprint, state, severity, summary, started_at, fired_at, resolved_at, last_eval_at)68 values ($1,$2,$3,$4,$5::alert_state,$6::severity,$7, now() - interval '10 minutes', now() - interval '9 minutes',69 case when $5 = 'resolved' then now() end, now())`, id, rule, nodeID, "fp-"+id, state, severity, summary)70 return id71}7273func TestClusterOverviewCountsOnlyFreshNodes(t *testing.T) {74 h := newHarness(t)75 root := h.owner()76 viewer := h.member(root, "viewer", "vera", false)7778 a := h.seedNode(nodeSeed{Alias: "alpha", Cores: 16, Mem: 64 * gib, Seen: true, Age: 5 * time.Second,79 Values: map[string]any{"cpu.usage": 10.0, "mem.used": 16 * gib, "mem.total": 64 * gib}})80 b := h.seedNode(nodeSeed{Alias: "bravo", Kind: "linux", Cores: 8, Mem: 32 * gib, Seen: true, Age: 10 * time.Second,81 Values: map[string]any{"mem.used": 4 * gib}}) // mem.total comes from the inventory82 c := h.seedNode(nodeSeed{Alias: "charlie", Cores: 12, Mem: 32 * gib, Seen: true, Age: 60 * time.Second, Maintenance: true,83 Values: map[string]any{"mem.used": 30 * gib, "mem.total": 32 * gib}}) // stale84 d := h.seedNode(nodeSeed{Alias: "delta", Cores: 10, Mem: 16 * gib, Seen: true, Age: 10 * time.Minute,85 Values: map[string]any{"mem.used": 15 * gib, "mem.total": 16 * gib}}) // offline86 h.seedNode(nodeSeed{Alias: "echo", Manual: true}) // registered, never seen87 f := h.seedNode(nodeSeed{Alias: "foxtrot", Cores: 99, Mem: 999 * gib, Seen: true, Age: time.Second, Values: map[string]any{"mem.used": gib}})88 h.dbExec(`update nodes set retired_at = now() where id = $1`, f)8990 // alpha: two APFS volumes of one container (size counted once), an external disk and a gone volume (both ignored).91 h.seedVolume(a, "/", "/dev/disk3s1s1", "apfs", "internal", 1000, 300, false)92 h.seedVolume(a, "/System/Volumes/Data", "/dev/disk3s5", "apfs", "internal", 1000, 100, false)93 h.seedVolume(a, "/Volumes/Ext", "/dev/disk7s1", "apfs", "external", 5000, 5000, false)94 h.seedVolume(a, "/Volumes/Old", "/dev/disk9s1", "apfs", "internal", 7000, 7000, true)95 h.seedVolume(b, "/", "/dev/sda1", "ext4", "internal", 500, 100, false)96 h.seedVolume(b, "/data", "/dev/sdb1", "ext4", "internal", 200, 50, false)97 h.seedVolume(c, "/", "/dev/disk3s1", "apfs", "internal", 999, 999, false) // stale node: not current9899 h.seedService(a, "pm2", "web", "online")100 h.seedService(a, "pm2", "api", "online")101 h.seedService(a, "launchd", "worker", "errored")102 gone := h.seedService(a, "pm2", "old", "online")103 h.dbExec(`update services set gone_at = now() where id = $1`, gone)104 h.seedService(c, "pm2", "batch", "stopped")105 h.seedService(d, "pm2", "ghost", "online") // node offline: state not known now106 h.seedModel(a, "mlx_lm", 8081, "mlx-community/Qwen", "ok")107 h.seedModel(c, "ollama", 11434, "llama3", "ok") // healthy when last seen, but the node is stale108109 h.dbExec(`insert into nas_targets(id, name, host, state, state_at) values110 ($1, 'nas-live', '192.168.2.10', 'online', now()), ($2, 'nas-old', '192.168.2.11', 'online', now() - interval '2 hours'),111 ($3, 'nas-new', '192.168.2.12', 'unknown', null)`, ids.New().String(), ids.New().String(), ids.New().String())112 h.dbExec(`insert into backups(id, name, kind, last_status, last_run_at, last_success_at) values113 ($1, 'pg', 'postgres', 'ok', now(), now() - interval '1 hour'), ($2, 'app', 'app', 'failed', now(), now() - interval '3 days'),114 ($3, 'new', 'app', 'unknown', null, null)`, ids.New().String(), ids.New().String(), ids.New().String())115 h.seedAlert(a, "firing", "warning", "disk almost full")116 h.seedAlert(a, "acknowledged", "critical", "service down")117 h.seedAlert(b, "resolved", "critical", "old trouble")118 h.seedAlert(b, "pending", "info", "not firing yet")119 h.dbExec(`insert into incidents(id, title, severity, state) values ($1, 'Outage', 'critical', 'open'), ($2, 'Past', 'warning', 'resolved')`,120 ids.New().String(), ids.New().String())121 h.dbExec(`insert into jobs(id, template, state) values ($1,'t','queued'), ($2,'t','running'), ($3,'t','running'), ($4,'t','succeeded')`,122 ids.New().String(), ids.New().String(), ids.New().String(), ids.New().String())123 h.dbExec(`insert into actions(id, type, criticality, node_id, target_kind, target_id, state, idempotency_key, request_hash, timeout_s, correlation_id)124 values ($1, 'inventory.refresh', 'low', $2, 'node', $3, 'succeeded', 'k1', '\x00', 60, $4)`, ids.New().String(), a, a, ids.New().String())125126 o := viewer.must("GET", "/cluster/overview", nil, 200).jsonObj(t)127 for path, want := range map[string]float64{128 "nodes.total": 5, "nodes.fresh": 2, "nodes.stale": 1, "nodes.offline": 2, "nodes.maintenance": 1,129 "capacity.cpu_cores.total": 24, "capacity.cpu_cores.coverage": 2,130 "capacity.memory.total": float64(96 * gib), "capacity.memory.used": float64(20 * gib), "capacity.memory.coverage": 2,131 "capacity.storage.total": 1700, "capacity.storage.used": 550, "capacity.storage.coverage": 2,132 "alerts.warning": 1, "alerts.critical": 1, "alerts.info": 0, "open_incidents": 1, "jobs.queued": 1, "jobs.running": 2,133 "services.online": 2, "services.errored": 1, "services.stopped": 1, "services.unknown": 1,134 "models.total": 2, "models.online": 1, "nas.total": 3, "nas.online": 1, "nas.offline": 0, "nas.unknown": 2,135 "backups.total": 3, "backups.ok": 1, "backups.failed": 1, "backups.unknown": 1,136 } {137 wantNum(t, o, want, strings.Split(path, ".")...)138 }139 if _, has := dig(o, "capacity", "cpu_cores").(map[string]any)["used"]; has {140 t.Fatal("cpu_cores must not report a usage")141 }142 if dig(o, "backups", "last_success_at") == nil {143 t.Fatal("last successful backup missing")144 }145 acts, _ := o["recent_actions"].([]any)146 if len(acts) != 1 || dig(acts[0].(map[string]any), "target", "id") != a || dig(acts[0].(map[string]any), "outcome_unknown") != false {147 t.Fatalf("recent_actions: %v", o["recent_actions"])148 }149150 // With no fresh node at all nothing is summed, and usage is absent rather than zero.151 h.dbExec(`update node_state set received_at = now() - interval '1 hour'`)152 h.dbExec(`update nodes set last_seen_at = now() - interval '1 hour'`)153 o = viewer.must("GET", "/cluster/overview", nil, 200).jsonObj(t)154 wantNum(t, o, 0, "nodes", "fresh")155 wantNum(t, o, 5, "nodes", "offline")156 wantNum(t, o, 0, "capacity", "memory", "coverage")157 wantNum(t, o, 0, "capacity", "memory", "total")158 if _, has := dig(o, "capacity", "memory").(map[string]any)["used"]; has {159 t.Fatal("memory usage reported although no node is fresh")160 }161 wantNum(t, o, 5, "services", "unknown")162 wantNum(t, o, 0, "models", "online")163164 // A section the caller may not read is omitted, not emptied.165 root.grant("user", h.userIDOf("vera"), "actions.read", "", "", "deny")166 root.grant("user", h.userIDOf("vera"), "alerts.read", "", "", "deny")167 o = viewer.must("GET", "/cluster/overview", nil, 200).jsonObj(t)168 for _, k := range []string{"recent_actions", "alerts", "open_incidents"} {169 if _, has := o[k]; has {170 t.Fatalf("overview exposes %s without the permission", k)171 }172 }173 if o["services"] == nil || o["nas"] == nil {174 t.Fatal("readable sections must stay")175 }176}177178func TestNodeListFiltersSortAndPagination(t *testing.T) {179 h := newHarness(t)180 root := h.owner()181 cpu := func(v float64) map[string]any {182 return map[string]any{"cpu.usage": v, "mem.used": gib, "mem.total": 4 * gib}183 }184 h.seedNode(nodeSeed{Alias: "Alpha", Seen: true, Age: 2 * time.Second, Values: cpu(50), Tags: []string{"gpu", "studio"}})185 h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: 3 * time.Second, Values: cpu(10), Site: "lab", Kind: "linux"})186 h.seedNode(nodeSeed{Alias: "Charlie", Seen: true, Age: 50 * time.Second, Values: cpu(90), Site: "lab", Tags: []string{"gpu"}})187 h.seedNode(nodeSeed{Alias: "delta", Seen: true, Age: time.Hour, Values: cpu(99), Role: "gateway"}) // offline: its 99 % is not current188 h.seedNode(nodeSeed{Alias: "Echo", Manual: true})189 h.seedNode(nodeSeed{Alias: "foxtrot", Seen: true, Age: 4 * time.Second, Values: map[string]any{}, Maintenance: true})190 gone := h.seedNode(nodeSeed{Alias: "golf", Seen: true, Age: time.Second, Values: cpu(1)})191 h.dbExec(`update nodes set retired_at = now() where id = $1`, gone)192 all := []string{"Alpha", "bravo", "Charlie", "delta", "Echo", "foxtrot"}193194 // Default order: alias, case-insensitive; every row exactly once whatever the page size.195 for _, limit := range []int{1, 2, 4, 50} {196 got := root.walkPages("/nodes", limit, "alias")197 if strings.Join(got, ",") != strings.Join(all, ",") {198 t.Fatalf("limit %d: order %v", limit, got)199 }200 }201 desc := root.walkPages("/nodes?sort=alias&order=desc", 2, "alias")202 if strings.Join(desc, ",") != "foxtrot,Echo,delta,Charlie,bravo,Alpha" {203 t.Fatalf("descending: %v", desc)204 }205 // Sorting on a live value: nodes without a current value come last, each row still exactly once.206 byCPU := root.walkPages("/nodes?sort=cpu&order=desc", 2, "alias")207 wantExactlyOnce(t, "sort=cpu", byCPU, all...)208 if strings.Join(byCPU[:3], ",") != "Charlie,Alpha,bravo" {209 t.Fatalf("cpu desc: %v", byCPU)210 }211 if asc := root.walkPages("/nodes?sort=cpu&order=asc", 2, "alias"); strings.Join(asc[:3], ",") != "bravo,Alpha,Charlie" || len(asc) != 6 {212 t.Fatalf("cpu asc (nulls last): %v", asc)213 }214 for _, s := range []string{"site", "role", "last_seen", "memory", "disk"} {215 for _, o := range []string{"asc", "desc"} {216 wantExactlyOnce(t, s+" "+o, root.walkPages("/nodes?sort="+s+"&order="+o, 2, "alias"), all...)217 }218 }219 if seen := root.walkPages("/nodes?sort=last_seen&order=desc", 3, "alias"); strings.Join(seen[:5], ",") != "Alpha,bravo,foxtrot,Charlie,delta" || seen[5] != "Echo" {220 t.Fatalf("last_seen desc: %v", seen)221 }222223 filters := map[string][]string{224 "site=lab": {"bravo", "Charlie"},225 "role=gateway": {"delta"},226 "tag=gpu": {"Alpha", "Charlie"},227 "kind=linux": {"bravo"},228 "freshness=fresh": {"Alpha", "bravo", "foxtrot"},229 "freshness=stale": {"Charlie"},230 "freshness=offline": {"delta", "Echo"},231 "maintenance=true": {"foxtrot"},232 "q=CHAR": {"Charlie"},233 "q=" + url.QueryEscape("%"): {},234 "q=_": {},235 "q=host-al": {"Alpha"},236 "retired=true": append([]string{"golf"}, all...),237 "site=lab&tag=gpu": {"Charlie"},238 "freshness=fresh&q=rav": {"bravo"},239 }240 for f, want := range filters {241 wantExactlyOnce(t, f, root.walkPages("/nodes?"+f, 2, "alias"), want...)242 }243244 // Headline: present for a reporting node, absent for an offline or never-seen one.245 rows := root.must("GET", "/nodes", nil, 200).jsonItems(t)246 if hl, _ := rowBy(rows, "alias", "Alpha")["headline"].(map[string]any); hl == nil || hl["cpu_usage_pct"] != 50.0 || hl["mem_used_pct"] != 25.0 {247 t.Fatalf("headline of a fresh node: %v", rowBy(rows, "alias", "Alpha"))248 }249 for _, alias := range []string{"delta", "Echo"} {250 if r := rowBy(rows, "alias", alias); r["headline"] != nil || r["freshness"] != "offline" {251 t.Fatalf("%s must be offline without headline: %v", alias, r)252 }253 }254 if r := rowBy(rows, "alias", "Echo"); r["observed_at"] != nil || r["last_seen_at"] != nil || r["cpu_cores"] != nil || r["model"] != nil || r["registered_via"] != "manual" {255 t.Fatalf("never-seen node must have no observation and no hardware: %v", r)256 }257258 // Invalid input is refused with a structured error.259 first := root.must("GET", "/nodes?limit=2", nil, 200).nextCursor(t)260 for _, bad := range []string{"limit=0", "limit=201", "sort=bogus", "order=up", "freshness=weird", "kind=windows", "cursor=garbage",261 "cursor=" + url.QueryEscape(first) + "&sort=cpu", "cursor=" + url.QueryEscape(first) + "&order=desc", "q=" + strings.Repeat("x", 121)} {262 if r := root.do("GET", "/nodes?"+bad, nil); r.Status != 400 || r.code() != "validation_failed" {263 t.Fatalf("%s: %d %s", bad, r.Status, r.Raw)264 }265 }266}267268func TestNodePrivateFieldsNeedPermission(t *testing.T) {269 h := newHarness(t)270 root := h.owner()271 viewer := h.member(root, "viewer", "vera", false)272 operator := h.member(root, "operator", "oscar", true)273 inv := map[string]any{"hardware_uuid": "HW-alpha", "serial_number": "SN-C02XYZ", "cpu_levels": []any{map[string]any{"name": "Performance", "cores": 12}},274 "interfaces": []any{275 map[string]any{"name": "en0", "kind": "ethernet", "mac": "aa:bb:cc:dd:ee:01", "addrs": []string{"192.168.2.50/24", "fe80::1/64"}, "media": "1000baseT <full-duplex>", "up": true, "default": true},276 map[string]any{"name": "utun4", "kind": "tunnel", "addrs": []string{"10.67.0.9"}, "up": true}}}277 x := h.seedNode(nodeSeed{Alias: "alpha", IP: "192.168.2.50", Seen: true, Age: time.Second, Inventory: inv,278 Values: map[string]any{"net.rx{en0}": 1200.0, "net.tx{en0}": 300.0, "net.latency{gateway}": 0.4, "net.loss{gateway}": 0.0}})279 y := h.seedNode(nodeSeed{Alias: "bravo", IP: "192.168.2.51", Seen: true, Age: time.Second, Inventory: inv})280 h.seedVolume(x, "/Volumes/nas", "//192.168.2.173/share", "smbfs", "network", 100, 10, false)281 h.dbExec(`insert into nas_targets(id, name, host) values ($1, 'nas-1', '192.168.2.173')`, ids.New().String())282 secrets := []string{"192.168.2.50", "192.168.2.51", "aa:bb:cc:dd:ee:01", "SN-C02XYZ", "HW-alpha", "192.168.2.173", "fe80::1", "10.67.0.9"}283 paths := []string{"/nodes", "/nodes/" + x, "/network/interfaces", "/network/topology", "/storage/volumes", "/storage/nas", "/search?q=alpha"}284285 for _, p := range paths {286 raw := string(viewer.must("GET", p, nil, 200).Raw)287 for _, s := range secrets {288 if strings.Contains(raw, s) {289 t.Fatalf("viewer sees private value %q in %s: %s", s, p, raw)290 }291 }292 }293 // The resource itself stays readable; only the private fields are left out.294 d := viewer.must("GET", "/nodes/"+x, nil, 200).jsonObj(t)295 ifs := d["interfaces"].([]any)296 en0 := ifs[0].(map[string]any)297 if len(ifs) != 2 || en0["name"] != "en0" || en0["kind"] != "ethernet" || en0["link_speed_mbps"] != 1000.0 || en0["rx_bytes_per_s"] != 1200.0 ||298 len(en0["ipv4"].([]any)) != 0 || en0["mac"] != nil || d["hardware_uuid"] != nil || dig(d, "node", "primary_ip") != nil {299 t.Fatalf("viewer detail: %v", d)300 }301 if ifs[1].(map[string]any)["kind"] != "other" { // a utun interface is not guessed to be WireGuard302 t.Fatalf("utun must stay 'other': %v", ifs[1])303 }304 if lv := d["cpu_levels"].([]any); len(lv) != 1 || lv[0].(map[string]any)["name"] != "Performance" {305 t.Fatalf("cpu levels: %v", d["cpu_levels"])306 }307308 od := operator.must("GET", "/nodes/"+x, nil, 200).jsonObj(t)309 oen0 := od["interfaces"].([]any)[0].(map[string]any)310 if od["hardware_uuid"] != "HW-alpha" || od["serial_number"] != "SN-C02XYZ" || dig(od, "node", "primary_ip") != "192.168.2.50" ||311 oen0["mac"] != "aa:bb:cc:dd:ee:01" || fmt.Sprint(oen0["ipv4"]) != "[192.168.2.50/24]" || fmt.Sprint(oen0["ipv6"]) != "[fe80::1/64]" {312 t.Fatalf("operator detail: %v", od)313 }314 for p, want := range map[string]string{"/nodes": "192.168.2.51", "/network/interfaces": "aa:bb:cc:dd:ee:01", "/network/topology": "192.168.2.50",315 "/storage/volumes": "//192.168.2.173/share", "/storage/nas": "192.168.2.173"} {316 if raw := string(operator.must("GET", p, nil, 200).Raw); !strings.Contains(raw, want) {317 t.Fatalf("operator must see %q in %s: %s", want, p, raw)318 }319 }320321 // A grant scoped to one node extends the private view to that node only.322 root.grant("user", h.userIDOf("vera"), "nodes.private.read", "node", x, "allow")323 rows := viewer.must("GET", "/nodes", nil, 200).jsonItems(t)324 if rowBy(rows, "alias", "alpha")["primary_ip"] != "192.168.2.50" || rowBy(rows, "alias", "bravo")["primary_ip"] != nil {325 t.Fatalf("scoped allow: %v", rows)326 }327 if raw := string(viewer.must("GET", "/nodes/"+y, nil, 200).Raw); strings.Contains(raw, "192.168.2.51") || strings.Contains(raw, "aa:bb") {328 t.Fatalf("the grant on alpha leaked bravo's private fields: %s", raw)329 }330 if raw := string(viewer.must("GET", "/storage/nas", nil, 200).Raw); strings.Contains(raw, "192.168.2.173") {331 t.Fatal("a node-scoped grant must not reveal NAS hosts")332 }333334 // Topology: the LAN link carries the measurements; no hub link without a WireGuard interface or hub probe.335 topo := operator.must("GET", "/network/topology", nil, 200).jsonObj(t)336 links := topo["links"].([]any)337 if len(links) != 2 {338 t.Fatalf("want one LAN link per reporting node and no hub link: %v", links)339 }340 var lx map[string]any341 for _, l := range links {342 if l.(map[string]any)["source"] == x {343 lx = l.(map[string]any)344 }345 }346 if lx["target"] != "lan-gateway" || lx["kind"] != "lan" || lx["state"] != "up" || lx["latency_ms"] != 0.4000000059604645 && lx["latency_ms"] != 0.4 ||347 lx["latency_availability"] != "available" || lx["interface"] != "en0" {348 t.Fatalf("lan link of alpha: %v", lx)349 }350 for _, l := range links {351 if m := l.(map[string]any); m["source"] == y && (m["state"] != "unknown" || m["latency_availability"] != "unavailable" || m["latency_ms"] != nil) {352 t.Fatalf("a link that was not measured must say so: %v", m)353 }354 }355}356357func TestManualNodeRegistrationUpdateAndRetire(t *testing.T) {358 h := newHarness(t)359 root := h.owner()360 admin := h.member(root, "admin", "ada", false)361 operator := h.member(root, "operator", "oscar", false)362 viewer := h.member(root, "viewer", "vera", false)363 h.seedPartitions()364 body := map[string]any{"alias": "M4BP36", "kind": "mac", "tags": []string{"laptop", "laptop"},365 "expected": map[string]any{"model": "MacBook Pro", "cpu_cores": 14, "mem_bytes": 38654705664}}366367 for _, c := range []*client{viewer, operator} {368 if r := c.do("POST", "/nodes", body); r.Status != 403 || r.code() != "forbidden" {369 t.Fatalf("node registration without nodes.write: %d %s", r.Status, r.Raw)370 }371 }372 for name, bad := range map[string]map[string]any{373 "alias pattern": {"alias": "bad alias!", "kind": "mac"}, "alias empty": {"alias": "", "kind": "mac"},374 "kind": {"alias": "ok", "kind": "windows"}, "long site": {"alias": "ok", "kind": "mac", "site": strings.Repeat("s", 65)},375 "empty tag": {"alias": "ok", "kind": "mac", "tags": []string{""}}, "zero cores": {"alias": "ok", "kind": "mac", "expected": map[string]any{"cpu_cores": 0}},376 } {377 if r := admin.do("POST", "/nodes", bad); r.Status != 400 || r.code() != "validation_failed" {378 t.Fatalf("%s: %d %s", name, r.Status, r.Raw)379 }380 }381 n := admin.must("POST", "/nodes", body, 201).jsonObj(t)382 id := n["id"].(string)383 if n["registered_via"] != "manual" || n["freshness"] != "offline" || n["observed_at"] != nil || n["last_seen_at"] != nil ||384 n["cpu_cores"] != nil || n["model"] != nil || n["mem_bytes"] != nil || fmt.Sprint(n["tags"]) != "[laptop]" || n["site"] != "default" {385 t.Fatalf("manual node must be offline, unobserved, with declared hardware kept apart: %v", n)386 }387 var hw *string388 var expected string389 h.dbScan([]any{&hw, &expected}, `select hardware_uuid, inventory->'expected'->>'model' from nodes where id = $1`, id)390 if hw != nil || expected != "MacBook Pro" {391 t.Fatalf("stored node: hardware_uuid=%v expected=%q", hw, expected)392 }393 if r := admin.do("POST", "/nodes", map[string]any{"alias": "m4bp36", "kind": "mac"}); r.Status != 409 || r.code() != "conflict" {394 t.Fatalf("duplicate alias (case-insensitive): %d %s", r.Status, r.Raw)395 }396397 // A node without hardware_uuid and without any report is handled by every read.398 d := viewer.must("GET", "/nodes/"+id, nil, 200).jsonObj(t)399 if len(d["interfaces"].([]any)) != 0 || len(d["volumes"].([]any)) != 0 || len(d["cpu_levels"].([]any)) != 0 || d["agent"] != nil {400 t.Fatalf("detail of a never-seen node: %v", d)401 }402 latest := d["latest"].(map[string]any)403 if len(latest) < 20 {404 t.Fatalf("latest must list every unlabeled catalog metric, got %d", len(latest))405 }406 for name, v := range latest {407 if mv := v.(map[string]any); mv["availability"] != "offline" || mv["value"] != nil {408 t.Fatalf("latest[%s] of a never-seen node: %v", name, mv)409 }410 }411 if p := viewer.must("GET", "/nodes/"+id+"/processes", nil, 200).jsonObj(t); p["availability"] != "offline" || len(p["items"].([]any)) != 0 || p["observed_at"] != nil {412 t.Fatalf("processes: %v", p)413 }414 m := viewer.must("GET", "/nodes/"+id+"/metrics?metrics=cpu.usage,net.rx", nil, 200).jsonObj(t)415 if len(m["series"].([]any)) != 0 || dig(m, "availability", "cpu.usage") != "offline" || dig(m, "availability", "net.rx") != "offline" {416 t.Fatalf("metrics: %v", m)417 }418 hh := viewer.must("GET", "/nodes/"+id+"/health-history?range=1h", nil, 200).jsonObj(t)419 if hh["uptime_ratio"] != nil || len(hh["items"].([]any)) != 12 {420 t.Fatalf("health history: %v", hh)421 }422 for _, b := range hh["items"].([]any) {423 if b.(map[string]any)["state"] != "unknown" {424 t.Fatalf("a node that never reported has no known health: %v", b)425 }426 }427 if len(viewer.must("GET", "/nodes/"+id+"/services", nil, 200).jsonItems(t)) != 0 {428 t.Fatal("services of a never-seen node")429 }430 o := viewer.must("GET", "/cluster/overview", nil, 200).jsonObj(t)431 wantNum(t, o, 1, "nodes", "total")432 wantNum(t, o, 1, "nodes", "offline")433 wantNum(t, o, 0, "capacity", "cpu_cores", "coverage")434 topo := viewer.must("GET", "/network/topology", nil, 200).jsonObj(t)435 if len(topo["nodes"].([]any)) != 1 || len(topo["links"].([]any)) != 0 {436 t.Fatalf("topology: one vertex, no observed link: %v", topo)437 }438 if len(viewer.must("GET", "/network/interfaces", nil, 200).jsonItems(t)) != 0 || len(viewer.must("GET", "/storage/volumes", nil, 200).jsonItems(t)) != 0 {439 t.Fatal("no interface and no volume was observed")440 }441 if s := viewer.must("GET", "/search?q=m4bp", nil, 200).jsonItems(t); len(s) != 1 || s[0]["kind"] != "node" || s[0]["freshness"] != "offline" {442 t.Fatalf("search: %v", s)443 }444 if tr := viewer.must("GET", "/cluster/trends?metric=cpu.usage&range=1h", nil, 200).jsonObj(t); dig(tr, "series").([]any)[0].(map[string]any)["coverage"] != 0.0 {445 t.Fatalf("trends: %v", tr)446 }447448 // Update.449 if r := viewer.do("PATCH", "/nodes/"+id, map[string]any{"site": "lab"}); r.Status != 403 {450 t.Fatalf("viewer patch: %d", r.Status)451 }452 for name, bad := range map[string]map[string]any{"nothing": {}, "alias": {"alias": "no spaces"}, "tags": {"tags": []string{strings.Repeat("t", 49)}}} {453 if r := admin.do("PATCH", "/nodes/"+id, bad); r.Status != 400 {454 t.Fatalf("patch %s: %d %s", name, r.Status, r.Raw)455 }456 }457 events0 := h.count(`select count(*) from events_outbox where topic = 'node.updated'`)458 u := admin.must("PATCH", "/nodes/"+id, map[string]any{"alias": "M4BP36b", "site": "lab", "role": "gateway", "tags": []string{"a", "b"}}, 200).jsonObj(t)459 if u["alias"] != "M4BP36b" || u["site"] != "lab" || u["role"] != "gateway" || fmt.Sprint(u["tags"]) != "[a b]" || u["environment"] != "production" {460 t.Fatalf("patched node: %v", u)461 }462 admin.must("PATCH", "/nodes/"+id, map[string]any{"site": "lab"}, 200) // no change: nothing recorded463 if n := h.count(`select count(*) from audit_log where action = 'node.update' and target_id = $1 and outcome = 'success'464 and params->'alias'->>'to' = 'M4BP36b' and params->'site'->>'from' = 'default'`, id); n != 1 {465 t.Fatalf("%d node.update audit entries, want 1", n)466 }467 if n := h.count(`select count(*) from audit_log where action = 'node.create' and target_id = $1 and actor_label = 'ada'`, id); n != 1 {468 t.Fatalf("%d node.create audit entries", n)469 }470 if n := h.count(`select count(*) from events_outbox where topic = 'node.updated' and payload->>'node_id' = $1 and payload->>'change' = 'updated'`, id); n != 1 ||471 h.count(`select count(*) from events_outbox where topic = 'node.updated'`) != events0+1 {472 t.Fatalf("node.updated events after patch: %d", n)473 }474 other := admin.must("POST", "/nodes", map[string]any{"alias": "other", "kind": "linux"}, 201).jsonObj(t)["id"].(string)475 if r := admin.do("PATCH", "/nodes/"+other, map[string]any{"alias": "m4bp36B"}); r.Status != 409 {476 t.Fatalf("alias collision on rename: %d %s", r.Status, r.Raw)477 }478 if r := admin.do("PATCH", "/nodes/"+ids.New().String(), map[string]any{"site": "x"}); r.Status != 404 {479 t.Fatalf("unknown node: %d", r.Status)480 }481482 // "Adoption" is ingest's job: once an agent has spoken the same row simply becomes a live node.483 h.dbExec(`update nodes set hardware_uuid = 'HW-1', cpu_cores = 14, inventory_at = now(), last_seen_at = now() where id = $1`, id)484 h.dbExec(`insert into node_state(node_id, ts, received_at, values) values ($1, now(), now(), '{"cpu.usage": 3.5}')`, id)485 agent := h.seedAgent(id, time.Second)486 alert := h.seedAlert(id, "firing", "warning", "hot")487 if n := admin.must("GET", "/nodes/"+id, nil, 200).jsonObj(t); dig(n, "node", "freshness") != "fresh" || dig(n, "node", "cpu_cores") != 14.0 ||488 dig(n, "agent", "id") != agent || dig(n, "latest", "cpu.usage", "value") != 3.5 || dig(n, "node", "open_alerts") != 1.0 {489 t.Fatalf("adopted node: %v", n)490 }491492 // Retirement: step-up, then agent revoked, alerts resolved, history kept.493 h.expireStepUp("ada")494 if r := admin.do("DELETE", "/nodes/"+id, nil); r.Status != 403 || r.code() != "step_up_required" {495 t.Fatalf("retire without step-up: %d %s", r.Status, r.Raw)496 }497 if r := operator.do("DELETE", "/nodes/"+id, nil); r.Status != 403 || r.code() != "forbidden" {498 t.Fatalf("retire by operator: %d %s", r.Status, r.Raw)499 }500 admin.stepUp()501 admin.must("DELETE", "/nodes/"+id, nil, 204)502 if h.count(`select count(*) from nodes where id = $1 and retired_at is not null`, id) != 1 ||503 h.count(`select count(*) from agents where id = $1 and revoked_at is not null and revoked_reason = 'node retired'`, agent) != 1 ||504 h.count(`select count(*) from alerts where id = $1 and state = 'resolved' and resolved_at is not null`, alert) != 1 ||505 h.count(`select count(*) from timeline_events where alert_id = $1 and kind = 'resolved'`, alert) != 1 {506 t.Fatal("retirement did not revoke the agent and resolve the alerts")507 }508 if h.count(`select count(*) from audit_log where action = 'node.retire' and target_id = $1 and (params->>'agents_revoked')::int = 1509 and (params->>'alerts_resolved')::int = 1`, id) != 1 {510 t.Fatal("retirement not audited")511 }512 if h.count(`select count(*) from events_outbox where topic = 'node.updated' and payload->>'node_id' = $1 and payload->>'change' = 'retired'`, id) != 1 ||513 h.count(`select count(*) from events_outbox where topic = 'alert.changed' and payload->>'alert_id' = $1 and payload->>'state' = 'resolved'`, alert) != 1 {514 t.Fatal("retirement events missing")515 }516 admin.must("DELETE", "/nodes/"+id, nil, 204) // idempotent517 if h.count(`select count(*) from audit_log where action = 'node.retire' and target_id = $1`, id) != 1 {518 t.Fatal("a repeated retirement must not be recorded twice")519 }520 if got := admin.walkPages("/nodes", 50, "alias"); strings.Join(got, ",") != "other" {521 t.Fatalf("retired node still listed: %v", got)522 }523 if got := admin.walkPages("/nodes?retired=true", 50, "alias"); len(got) != 2 {524 t.Fatalf("retired=true: %v", got)525 }526 if r := admin.do("PATCH", "/nodes/"+id, map[string]any{"site": "x"}); r.Status != 409 {527 t.Fatalf("patch of a retired node: %d %s", r.Status, r.Raw)528 }529 if r := admin.do("POST", "/nodes", map[string]any{"alias": "M4BP36b", "kind": "mac"}); r.Status != 409 || !strings.Contains(string(r.Raw), "retired") {530 t.Fatalf("alias of a retired node: %d %s", r.Status, r.Raw)531 }532 if n := admin.must("GET", "/nodes/"+id, nil, 200).jsonObj(t); dig(n, "node", "retired_at") == nil || n["agent"] != nil {533 t.Fatalf("a retired node stays readable, without active agent: %v", n)534 }535}536537func TestNodeProcesses(t *testing.T) {538 h := newHarness(t)539 root := h.owner()540 procs := []protocol.Process{{PID: 1, Name: "launchd", User: "root", CPU: 0.1, RSS: 1 << 20}, {PID: 4242, Name: "node", User: "sp", CPU: 180.5, RSS: 1 << 30}}541 fresh := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second, Processes: procs})542 stale := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Minute, Processes: procs})543 off := h.seedNode(nodeSeed{Alias: "charlie", Seen: true, Age: time.Hour, Processes: procs})544 none := h.seedNode(nodeSeed{Alias: "delta", Seen: true, Age: time.Second})545546 p := root.must("GET", "/nodes/"+fresh+"/processes", nil, 200).jsonObj(t)547 items := p["items"].([]any)548 if p["availability"] != "available" || p["freshness"] != "fresh" || len(items) != 2 || p["observed_at"] == nil {549 t.Fatalf("fresh: %v", p)550 }551 if it := items[1].(map[string]any); it["pid"] != 4242.0 || it["name"] != "node" || it["user"] != "sp" || it["cpu_pct"] != 180.5 || it["mem_bytes"] != float64(1<<30) || it["command"] != nil {552 t.Fatalf("process mapping: %v", it)553 }554 if p := root.must("GET", "/nodes/"+stale+"/processes", nil, 200).jsonObj(t); p["availability"] != "stale" || len(p["items"].([]any)) != 2 {555 t.Fatalf("stale: %v", p)556 }557 if p := root.must("GET", "/nodes/"+off+"/processes", nil, 200).jsonObj(t); p["availability"] != "offline" || len(p["items"].([]any)) != 0 {558 t.Fatalf("an old snapshot of an offline node must not be shown as its processes: %v", p)559 }560 if p := root.must("GET", "/nodes/"+none+"/processes", nil, 200).jsonObj(t); p["availability"] != "unavailable" || len(p["items"].([]any)) != 0 {561 t.Fatalf("no snapshot: %v", p)562 }563 if r := root.do("GET", "/nodes/"+ids.New().String()+"/processes", nil); r.Status != 404 {564 t.Fatalf("unknown node: %d", r.Status)565 }566}567