Repository Public
1package api_test23import (4 "crypto/sha256"5 "encoding/json"6 "fmt"7 "sort"8 "strings"9 "testing"10 "time"1112 "clustr.spboucher.ai/mcc/internal/api"13 "clustr.spboucher.ai/mcc/internal/identity/vauth"14 "clustr.spboucher.ai/mcc/internal/ids"15)1617// opPath fills the path parameters of a contract path with plausible values.18func opPath(p string) string {19 r := strings.NewReplacer("{id}", ids.New().String(), "{passkeyId}", ids.New().String(), "{key}", "retention.raw_days", "{name}", "default")20 return r.Replace(p)21}2223// Operations of the read side and of administration (this file's scope),24// with the query parameters the contract makes mandatory.25var clusterAndAdminOps = map[string]string{26 "GetClusterOverview": "", "GetClusterTrends": "?metric=cpu.usage", "ListNodes": "", "CreateNode": "", "GetNode": "", "UpdateNode": "", "RetireNode": "",27 "GetNodeMetrics": "?metrics=cpu.usage", "CompareMetrics": "?metric=cpu.usage&node_ids=" + ids.New().String(), "GetMetricCatalog": "",28 "ListNodeProcesses": "", "ListNodeServices": "", "GetNodeHealthHistory": "", "ListServices": "", "GetService": "", "UpdateService": "",29 "GetServiceHistory": "", "ListInferenceServers": "", "GetInferenceServer": "", "GetNetworkTopology": "", "ListNetworkInterfaces": "",30 "ListVolumes": "", "ListNasTargets": "", "CreateNasTarget": "", "DeleteNasTarget": "", "ListBackups": "", "ListMyDashboards": "",31 "PutMyDashboard": "", "DeleteMyDashboard": "", "ListActivity": "", "Search": "?q=x",32 "ListUsers": "", "GetUser": "", "UpdateUser": "", "ListUserPasskeys": "", "RevokeUserPasskey": "", "ListUserSessions": "", "RevokeSession": "",33 "ListUserDevices": "", "RevokeDevice": "", "ListPolicyGrants": "", "CreatePolicyGrant": "", "DeletePolicyGrant": "", "ListAgents": "",34 "ListEnrollmentTokens": "", "CreateEnrollmentToken": "", "RevokeEnrollmentToken": "", "RevokeAgent": "", "ListSettings": "", "PutSetting": "",35 "GetAdminStatus": "",36}3738func TestEveryClusterAndAdminOperationIsImplemented(t *testing.T) {39 h := newHarness(t)40 root := h.owner()41 h.seedPartitions()42 if len(clusterAndAdminOps) != 51 {43 t.Fatalf("%d operations listed", len(clusterAndAdminOps))44 }45 for name, query := range clusterAndAdminOps {46 op, ok := api.Operations[name]47 if !ok {48 t.Fatalf("operation %s is not in the contract", name)49 }50 var body any51 if op.Method != "GET" && op.Method != "DELETE" {52 body = map[string]any{}53 }54 r := root.do(op.Method, opPath(op.Path)+query, body)55 if r.Status == 501 || r.Status >= 500 || r.Status == 401 || r.code() == "forbidden" || r.code() == "step_up_required" {56 t.Errorf("%s %s as owner: %d %s", op.Method, op.Path, r.Status, r.Raw)57 }58 }59}6061func TestAdministrationIsRefusedToViewersAndOperators(t *testing.T) {62 h := newHarness(t)63 root := h.owner()64 viewer := h.member(root, "viewer", "vera", false)65 operator := h.member(root, "operator", "oscar", true)66 checked := 067 var names []string68 for name, op := range api.Operations {69 switch op.Permission {70 case "users.manage", "agents.manage", "settings.manage":71 names = append(names, name)72 }73 }74 sort.Strings(names)75 for _, name := range names {76 op := api.Operations[name]77 var body any78 if op.Method != "GET" && op.Method != "DELETE" {79 body = map[string]any{}80 }81 for who, c := range map[string]*client{"viewer": viewer, "operator": operator} {82 r := c.do(op.Method, opPath(op.Path), body)83 if r.Status != 403 || r.code() != "forbidden" || !strings.Contains(string(r.Raw), op.Permission) {84 t.Errorf("%s %s (%s) as %s: %d %s", op.Method, op.Path, name, who, r.Status, r.Raw)85 }86 }87 checked++88 }89 // users (9), policy grants (3), agents and enrollment tokens (5), settings and status (3), plus the settings.manage operations90 // outside the Admin tag (service update, NAS create / delete).91 if checked != 23 {92 t.Fatalf("%d administration operations checked: %v", checked, names)93 }94 if n := h.count(`select count(*) from audit_log where action = 'authz.denied' and outcome = 'denied'`); n != 2*checked {95 t.Fatalf("%d refusals audited, want %d", n, 2*checked)96 }97 if h.count(`select count(*) from settings where key not in ('events.pruned')`)+h.count(`select count(*) from policy_grants`)+98 h.count(`select count(*) from agent_enrollment_tokens`)+h.count(`select count(*) from nas_targets`) != 0 {99 t.Fatal("a refused request changed state")100 }101}102103func TestUserAdministrationRules(t *testing.T) {104 h := newHarness(t)105 root := h.owner()106 admin := h.member(root, "admin", "ada", false)107 viewer := h.member(root, "viewer", "vera", false)108 h.member(root, "operator", "oscar", true)109 id := h.userIDOf110111 // Listing, filtering, paging.112 if got := admin.walkPages("/users", 1, "handle"); strings.Join(got, ",") != "ada,oscar,root,vera" {113 t.Fatalf("users: %v", got)114 }115 wantExactlyOnce(t, "role filter", admin.walkPages("/users?role=viewer", 5, "handle"), "vera")116 wantExactlyOnce(t, "q", admin.walkPages("/users?q=USER%20o", 5, "handle"), "oscar")117 wantExactlyOnce(t, "disabled", admin.walkPages("/users?disabled=true", 5, "handle"))118 u := admin.must("GET", "/users/"+id("vera"), nil, 200).jsonObj(t)119 if u["handle"] != "vera" || u["role"] != "viewer" || u["passkey_count"] != 1.0 || u["last_seen_at"] == nil || u["disabled"] != false {120 t.Fatalf("user: %v", u)121 }122 for _, bad := range []string{"role=king", "limit=0", "cursor=x"} {123 if r := admin.do("GET", "/users?"+bad, nil); r.Status != 400 {124 t.Fatalf("%s: %d", bad, r.Status)125 }126 }127 if r := admin.do("GET", "/users/"+ids.New().String(), nil); r.Status != 404 {128 t.Fatalf("unknown user: %d", r.Status)129 }130131 // Role changes.132 patch := func(c *client, handle string, body map[string]any) resp {133 return c.do("PATCH", "/users/"+id(handle), body)134 }135 if r := patch(admin, "root", map[string]any{"role": "viewer"}); r.Status != 403 {136 t.Fatalf("admin demoting an owner: %d %s", r.Status, r.Raw)137 }138 if r := patch(admin, "root", map[string]any{"disabled": true}); r.Status != 403 {139 t.Fatalf("admin disabling an owner: %d %s", r.Status, r.Raw)140 }141 if r := patch(admin, "vera", map[string]any{"role": "owner"}); r.Status != 403 {142 t.Fatalf("admin granting owner: %d %s", r.Status, r.Raw)143 }144 if r := patch(admin, "ada", map[string]any{"role": "viewer"}); r.Status != 409 || r.code() != "self_modification" {145 t.Fatalf("self-modification: %d %s", r.Status, r.Raw)146 }147 if r := patch(root, "root", map[string]any{"disabled": true}); r.Status != 409 || r.code() != "self_modification" {148 t.Fatalf("owner disabling itself: %d %s", r.Status, r.Raw)149 }150 for name, body := range map[string]map[string]any{"empty": {}, "bad role": {"role": "king"}} {151 if r := patch(admin, "vera", body); r.Status != 400 {152 t.Fatalf("%s: %d %s", name, r.Status, r.Raw)153 }154 }155 if r := admin.do("PATCH", "/users/"+ids.New().String(), map[string]any{"role": "viewer"}); r.Status != 404 {156 t.Fatalf("unknown user: %d", r.Status)157 }158 if got := patch(admin, "vera", map[string]any{"role": "operator"}); got.Status != 200 || got.jsonObj(t)["role"] != "operator" {159 t.Fatalf("admin promoting a viewer: %d %s", got.Status, got.Raw)160 }161 h.expireStepUp("ada")162 if r := patch(admin, "vera", map[string]any{"role": "viewer"}); r.Status != 403 || r.code() != "step_up_required" {163 t.Fatalf("role change without step-up: %d %s", r.Status, r.Raw)164 }165 admin.stepUp()166 patch(admin, "vera", map[string]any{"role": "viewer"})167 if h.count(`select count(*) from audit_log where action = 'user.update' and outcome = 'success' and target_id = $1`, id("vera")) != 2 ||168 h.count(`select count(*) from audit_log where action = 'user.update' and outcome = 'denied'`) != 3 {169 t.Fatal("role changes and refusals must be audited")170 }171172 // Disabling revokes the sessions; enabling does not restore them.173 viewer.must("GET", "/me", nil, 200)174 if d := patch(admin, "vera", map[string]any{"disabled": true}).jsonObj(t); d["disabled"] != true || d["disabled_at"] == nil {175 t.Fatalf("disabled user: %v", d)176 }177 if r := viewer.do("GET", "/me", nil); r.Status != 401 {178 t.Fatalf("disabled user still signed in: %d", r.Status)179 }180 if r := viewer.login(); r.Status == 200 {181 t.Fatal("a disabled user signed in")182 }183 wantExactlyOnce(t, "disabled", admin.walkPages("/users?disabled=true", 5, "handle"), "vera")184 patch(admin, "vera", map[string]any{"disabled": false})185 if r := viewer.login(); r.Status != 200 {186 t.Fatalf("enabled user cannot sign in: %d %s", r.Status, r.Raw)187 }188189 // The last owner able to sign in cannot be demoted or disabled, even by another owner session.190 second := h.member(root, "owner", "olga", false)191 _ = second192 h.dbExec(`update passkeys set revoked_at = now() where user_id = $1`, id("root")) // root's session survives, but root can no longer sign in193 for _, body := range []map[string]any{{"role": "admin"}, {"disabled": true}} {194 if r := patch(root, "olga", body); r.Status != 409 || r.code() != "last_owner" {195 t.Fatalf("demoting the last active owner (%v): %d %s", body, r.Status, r.Raw)196 }197 }198 if h.count(`select count(*) from users where handle = 'olga' and role = 'owner' and disabled_at is null`) != 1 {199 t.Fatal("the last owner was modified")200 }201}202203func TestAdminRevokesSessionsDevicesAndPasskeys(t *testing.T) {204 h := newHarness(t)205 root := h.owner()206 admin := h.member(root, "admin", "ada", false)207 viewer := h.member(root, "viewer", "vera", false)208 vera, rootID := h.userIDOf("vera"), h.userIDOf("root")209210 // Sessions.211 ss := admin.must("GET", "/users/"+vera+"/sessions", nil, 200).jsonItems(t)212 if len(ss) != 1 || ss[0]["user_id"] != vera || ss[0]["current"] != false || ss[0]["ip"] == nil || ss[0]["kind"] != "cookie" {213 t.Fatalf("sessions of another user: %v", ss)214 }215 sid := ss[0]["id"].(string)216 h.expireStepUp("ada")217 if r := admin.do("DELETE", "/sessions/"+sid, nil); r.Status != 403 || r.code() != "step_up_required" {218 t.Fatalf("revoking a session without step-up: %d %s", r.Status, r.Raw)219 }220 viewer.must("GET", "/me", nil, 200)221 admin.stepUp()222 admin.must("DELETE", "/sessions/"+sid, nil, 204)223 if r := viewer.do("GET", "/me", nil); r.Status != 401 {224 t.Fatalf("revoked session still valid: %d", r.Status)225 }226 if r := admin.do("DELETE", "/sessions/"+sid, nil); r.Status != 404 {227 t.Fatalf("revoking twice: %d", r.Status)228 }229 if h.count(`select count(*) from audit_log where action = 'session.revoke' and target_id = $1 and actor_label = 'ada' and detail = 'revoked_by_admin'`, sid) != 1 {230 t.Fatal("session revocation not audited")231 }232233 // Devices: revoking one ends its sessions.234 if viewer.login().Status != 200 {235 t.Fatal("sign-in failed")236 }237 ds := admin.must("GET", "/users/"+vera+"/devices", nil, 200).jsonItems(t)238 if len(ds) != 1 || ds[0]["kind"] != "web" || ds[0]["user_id"] != vera || ds[0]["current"] != false || strings.Contains(fmt.Sprint(ds[0]), "push_token") {239 t.Fatalf("devices: %v", ds)240 }241 admin.must("DELETE", "/devices/"+ds[0]["id"].(string), nil, 204)242 if r := viewer.do("GET", "/me", nil); r.Status != 401 {243 t.Fatalf("session of a revoked device still valid: %d", r.Status)244 }245 if d := admin.must("GET", "/users/"+vera+"/devices", nil, 200).jsonItems(t); d[0]["revoked_at"] == nil {246 t.Fatalf("revoked device: %v", d)247 }248249 // Passkeys: the last one cannot be revoked; a second one can, and stays listed as revoked.250 if viewer.login().Status != 200 {251 t.Fatal("sign-in failed")252 }253 keys := admin.must("GET", "/users/"+vera+"/passkeys", nil, 200).jsonItems(t)254 if len(keys) != 1 {255 t.Fatalf("passkeys: %v", keys)256 }257 if r := admin.do("DELETE", "/users/"+vera+"/passkeys/"+keys[0]["id"].(string), nil); r.Status != 409 || r.code() != "last_passkey" {258 t.Fatalf("revoking the last passkey: %d %s", r.Status, r.Raw)259 }260 other := vauth.New(h.URL)261 var ch struct {262 ChallengeID string `json:"challenge_id"`263 PublicKey json.RawMessage `json:"public_key"`264 }265 viewer.must("POST", "/auth/webauthn/registration/begin", map[string]any{}, 200).json(t, &ch)266 cred, err := other.Create([]byte(`{"publicKey":` + string(ch.PublicKey) + `}`))267 if err != nil {268 t.Fatal(err)269 }270 viewer.must("POST", "/auth/webauthn/registration/finish", map[string]any{"challenge_id": ch.ChallengeID, "credential": json.RawMessage(cred), "client": viewer.clientInfo()}, 200)271 keys = admin.must("GET", "/users/"+vera+"/passkeys", nil, 200).jsonItems(t)272 if len(keys) != 2 {273 t.Fatalf("passkeys after adding one: %v", keys)274 }275 first := keys[0]["id"].(string)276 if r := admin.do("DELETE", "/users/"+rootID+"/passkeys/"+first, nil); r.Status != 403 {277 t.Fatalf("admin on an owner's passkeys: %d", r.Status)278 }279 if r := admin.do("DELETE", "/users/"+h.userIDOf("ada")+"/passkeys/"+first, nil); r.Status != 404 {280 t.Fatalf("a passkey is only revocable under its own user: %d", r.Status)281 }282 admin.must("DELETE", "/users/"+vera+"/passkeys/"+first, nil, 204)283 keys = admin.must("GET", "/users/"+vera+"/passkeys", nil, 200).jsonItems(t)284 if len(keys) != 2 || rowBy(keys, "id", first)["revoked_at"] == nil {285 t.Fatalf("revoked passkeys stay in the administration view: %v", keys)286 }287 if r := viewer.do("GET", "/me", nil); r.Status != 401 {288 t.Fatalf("session opened with a revoked passkey still valid: %d", r.Status)289 }290 if r := viewer.login(); r.Status == 200 {291 t.Fatal("signed in with a revoked passkey")292 }293 if h.count(`select count(*) from audit_log where action = 'passkey.revoke' and outcome = 'success' and actor_label = 'ada'`) != 1 {294 t.Fatal("passkey revocation not audited")295 }296297 // An admin does not touch an owner's credentials; an owner does.298 rs := root.must("GET", "/users/"+rootID+"/sessions", nil, 200).jsonItems(t)299 if rs[0]["current"] != true {300 t.Fatalf("own session must be flagged current: %v", rs)301 }302 if r := admin.do("DELETE", "/sessions/"+rs[0]["id"].(string), nil); r.Status != 403 {303 t.Fatalf("admin revoking an owner's session: %d", r.Status)304 }305 rd := root.must("GET", "/users/"+rootID+"/devices", nil, 200).jsonItems(t)306 if r := admin.do("DELETE", "/devices/"+rd[0]["id"].(string), nil); r.Status != 403 {307 t.Fatalf("admin revoking an owner's device: %d", r.Status)308 }309 root.must("GET", "/me", nil, 200)310 if h.count(`select count(*) from audit_log where outcome = 'denied' and action in ('session.revoke', 'device.revoke', 'passkey.revoke')`) != 3 {311 t.Fatal("refused attempts on an owner must be audited")312 }313 as := root.must("GET", "/users/"+h.userIDOf("ada")+"/sessions", nil, 200).jsonItems(t)314 root.must("DELETE", "/sessions/"+as[0]["id"].(string), nil, 204)315 if r := admin.do("GET", "/users", nil); r.Status != 401 {316 t.Fatalf("admin session revoked by the owner: %d", r.Status)317 }318 for _, p := range []string{"/users/" + ids.New().String() + "/sessions", "/users/" + ids.New().String() + "/devices", "/users/" + ids.New().String() + "/passkeys"} {319 if r := root.do("GET", p, nil); r.Status != 404 {320 t.Fatalf("%s: %d", p, r.Status)321 }322 }323}324325func TestPolicyGrantsTakeEffectAndAreValidated(t *testing.T) {326 h := newHarness(t)327 root := h.owner()328 admin := h.member(root, "admin", "ada", false)329 operator := h.member(root, "operator", "oscar", false)330 viewer := h.member(root, "viewer", "vera", false)331 x := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second, IP: "192.168.2.50", Site: "lab"})332 y := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Second, IP: "192.168.2.51"})333 oscar, vera := h.userIDOf("oscar"), h.userIDOf("vera")334 g := func(kind, subject, perm, rk, rid, effect string) map[string]any {335 b := map[string]any{"subject_kind": kind, "subject": subject, "permission": perm, "effect": effect}336 if rk != "" {337 b["resource_kind"] = rk338 }339 if rid != "" {340 b["resource_id"] = rid341 }342 return b343 }344345 // A scoped deny blocks, at once, and only that resource.346 operator.must("GET", "/nodes/"+x, nil, 200)347 deny := admin.must("POST", "/policy-grants", g("user", oscar, "nodes.read", "node", x, "deny"), 201).jsonObj(t)348 if deny["effect"] != "deny" || deny["resource_kind"] != "node" || deny["resource_id"] != x || deny["created_by"] != h.userIDOf("ada") {349 t.Fatalf("created grant: %v", deny)350 }351 if r := operator.do("GET", "/nodes/"+x, nil); r.Status != 404 {352 t.Fatalf("scoped deny not applied: %d", r.Status)353 }354 operator.must("GET", "/nodes/"+y, nil, 200)355 if got := operator.walkPages("/nodes", 5, "alias"); strings.Join(got, ",") != "bravo" {356 t.Fatalf("list under a scoped deny: %v", got)357 }358 admin.must("DELETE", "/policy-grants/"+deny["id"].(string), nil, 204)359 operator.must("GET", "/nodes/"+x, nil, 200)360 if r := admin.do("DELETE", "/policy-grants/"+deny["id"].(string), nil); r.Status != 404 {361 t.Fatalf("deleting a grant twice: %d", r.Status)362 }363 // A deny on a site covers the nodes of that site.364 site := admin.must("POST", "/policy-grants", g("role", "operator", "nodes.read", "site", "lab", "deny"), 201).jsonObj(t)365 if r := operator.do("GET", "/nodes/"+x, nil); r.Status != 404 {366 t.Fatalf("site-scoped deny not applied: %d", r.Status)367 }368 admin.must("DELETE", "/policy-grants/"+site["id"].(string), nil, 204)369370 // An allow extends: private fields for viewers, visible in /me too.371 if n := viewer.must("GET", "/nodes/"+x, nil, 200).jsonObj(t); dig(n, "node", "primary_ip") != nil {372 t.Fatal("viewer sees a private field by default")373 }374 allow := admin.must("POST", "/policy-grants", g("role", "viewer", "nodes.private.read", "", "", "allow"), 201).jsonObj(t)375 if n := viewer.must("GET", "/nodes/"+x, nil, 200).jsonObj(t); dig(n, "node", "primary_ip") != "192.168.2.50" {376 t.Fatal("allow grant not applied")377 }378 if me := string(viewer.must("GET", "/me", nil, 200).Raw); !strings.Contains(me, "nodes.private.read") {379 t.Fatalf("/me must reflect the grant: %s", me)380 }381 // Deny wins over allow.382 admin.must("POST", "/policy-grants", g("user", vera, "nodes.private.read", "node", x, "deny"), 201)383 if n := viewer.must("GET", "/nodes/"+x, nil, 200).jsonObj(t); dig(n, "node", "primary_ip") != nil {384 t.Fatal("deny must win over allow")385 }386 if n := viewer.must("GET", "/nodes/"+y, nil, 200).jsonObj(t); dig(n, "node", "primary_ip") != "192.168.2.51" {387 t.Fatal("the deny is scoped to one node")388 }389390 // Invalid or pointless grants are refused with a precise code.391 cases := []struct {392 name string393 body map[string]any394 status int395 code string396 reason string397 }{398 {"unknown permission", g("role", "viewer", "nodes.fly", "", "", "allow"), 400, "validation_failed", ""},399 {"unknown role", g("role", "king", "nodes.read", "", "", "deny"), 400, "validation_failed", ""},400 {"bad subject kind", g("group", "x", "nodes.read", "", "", "deny"), 400, "validation_failed", ""},401 {"bad effect", g("role", "viewer", "nodes.read", "", "", "maybe"), 400, "validation_failed", ""},402 {"user id shape", g("user", "vera", "nodes.read", "", "", "deny"), 400, "validation_failed", ""},403 {"resource id without kind", g("role", "viewer", "nodes.read", "", x, "deny"), 400, "validation_failed", ""},404 {"bad resource kind", g("role", "viewer", "nodes.read", "rack", "r1", "deny"), 400, "validation_failed", ""},405 {"node id shape", g("role", "viewer", "nodes.read", "node", "alpha", "deny"), 400, "validation_failed", ""},406 {"unknown user", g("user", ids.New().String(), "nodes.read", "", "", "deny"), 422, "unprocessable", "unknown_subject"},407 {"unknown node", g("role", "viewer", "nodes.read", "node", ids.New().String(), "deny"), 422, "unprocessable", "unknown_resource"},408 {"critical to admins", g("role", "admin", "actions.run.critical", "", "", "allow"), 422, "unprocessable", "grant_ineffective"},409 {"critical to an operator", g("user", oscar, "actions.run.critical", "", "", "allow"), 422, "unprocessable", "grant_ineffective"},410 {"user management below admin", g("role", "operator", "users.manage", "", "", "allow"), 422, "unprocessable", "grant_ineffective"},411 {"settings below admin", g("user", vera, "settings.manage", "", "", "allow"), 422, "unprocessable", "grant_ineffective"},412 {"scoped cluster-wide permission", g("role", "admin", "users.manage", "node", x, "deny"), 422, "unprocessable", "permission_not_scopable"},413 {"duplicate", g("role", "viewer", "nodes.private.read", "", "", "allow"), 409, "conflict", ""},414 {"opposite", g("role", "viewer", "nodes.private.read", "", "", "deny"), 409, "conflict", ""},415 }416 before := h.count(`select count(*) from policy_grants`)417 for _, c := range cases {418 r := admin.do("POST", "/policy-grants", c.body)419 if r.Status != c.status || r.code() != c.code || (c.reason != "" && !strings.Contains(string(r.Raw), `"reason":"`+c.reason+`"`)) {420 t.Errorf("%s: %d %s, want %d %s %s", c.name, r.Status, r.Raw, c.status, c.code, c.reason)421 }422 }423 if r := root.do("POST", "/policy-grants", g("role", "owner", "users.manage", "", "", "deny")); r.Status != 422 || !strings.Contains(string(r.Raw), "lockout") {424 t.Errorf("locking owners out: %d %s", r.Status, r.Raw)425 }426 if h.count(`select count(*) from policy_grants`) != before {427 t.Fatal("a refused grant was stored")428 }429430 // An admin neither targets owners nor hands out what it does not hold.431 for name, body := range map[string]map[string]any{432 "owner role": g("role", "owner", "nodes.read", "", "", "deny"), "owner user": g("user", h.userIDOf("root"), "logs.read", "", "", "deny")} {433 if r := admin.do("POST", "/policy-grants", body); r.Status != 403 {434 t.Errorf("admin targeting an %s: %d %s", name, r.Status, r.Raw)435 }436 }437 ownerGrant := root.grant("role", "owner", "nodes.read", "node", y, "deny")438 if r := admin.do("DELETE", "/policy-grants/"+ownerGrant, nil); r.Status != 403 {439 t.Errorf("admin deleting a grant on owners: %d", r.Status)440 }441 root.must("DELETE", "/policy-grants/"+ownerGrant, nil, 204)442 root.grant("user", h.userIDOf("ada"), "logs.read", "", "", "deny")443 if r := admin.do("POST", "/policy-grants", g("user", vera, "logs.read", "", "", "allow")); r.Status != 403 {444 t.Errorf("admin granting a permission it does not hold: %d %s", r.Status, r.Raw)445 }446 root.must("POST", "/policy-grants", g("user", vera, "logs.read", "", "", "allow"), 201)447448 // Step-up and roles.449 if r := operator.do("POST", "/policy-grants", g("role", "viewer", "nodes.read", "", "", "deny")); r.Status != 403 || r.code() != "forbidden" {450 t.Fatalf("operator creating a grant: %d", r.Status)451 }452 h.expireStepUp("ada")453 if r := admin.do("POST", "/policy-grants", g("role", "viewer", "jobs.read", "", "", "deny")); r.code() != "step_up_required" {454 t.Fatalf("grant without step-up: %d %s", r.Status, r.Raw)455 }456 if r := admin.do("DELETE", "/policy-grants/"+allow["id"].(string), nil); r.code() != "step_up_required" {457 t.Fatalf("grant deletion without step-up: %d %s", r.Status, r.Raw)458 }459460 // Listing: most recent first, filters, paging.461 all := admin.must("GET", "/policy-grants", nil, 200).jsonItems(t)462 if len(all) != 4 || all[0]["permission"] != "logs.read" || all[0]["effect"] != "allow" || all[3]["id"] != allow["id"] {463 t.Fatalf("grants: %v", all)464 }465 if got := admin.walkPages("/policy-grants", 1, "id"); len(got) != 4 || got[3] != allow["id"] {466 t.Fatalf("paged grants: %v", got)467 }468 wantExactlyOnce(t, "by subject", admin.walkPages("/policy-grants?subject_kind=user&subject="+vera, 5, "permission"), "nodes.private.read", "logs.read")469 wantExactlyOnce(t, "by permission", admin.walkPages("/policy-grants?permission=nodes.private.read", 5, "effect"), "allow", "deny")470 for _, bad := range []string{"permission=nope", "subject_kind=group", "cursor=x"} {471 if r := admin.do("GET", "/policy-grants?"+bad, nil); r.Status != 400 {472 t.Fatalf("%s: %d", bad, r.Status)473 }474 }475 if h.count(`select count(*) from audit_log where action = 'policy.grant.create' and outcome = 'success'`) != 7 ||476 h.count(`select count(*) from audit_log where action = 'policy.grant.delete' and outcome = 'success'`) != 3 ||477 h.count(`select count(*) from audit_log where action like 'policy.grant.%' and outcome = 'denied'`) != 4 {478 t.Fatalf("grant audit trail: %d created, %d deleted, %d denied",479 h.count(`select count(*) from audit_log where action = 'policy.grant.create' and outcome = 'success'`),480 h.count(`select count(*) from audit_log where action = 'policy.grant.delete' and outcome = 'success'`),481 h.count(`select count(*) from audit_log where action like 'policy.grant.%' and outcome = 'denied'`))482 }483}484485func TestEnrollmentTokensAndAgents(t *testing.T) {486 h := newHarness(t)487 root := h.owner()488 admin := h.member(root, "admin", "ada", false)489 body := map[string]any{"node_alias": "m4mn", "node_kind": "mac"}490491 h.expireStepUp("ada")492 if r := admin.do("POST", "/agents/enrollment-tokens", body); r.Status != 403 || r.code() != "step_up_required" {493 t.Fatalf("token without step-up: %d %s", r.Status, r.Raw)494 }495 admin.stepUp()496 for name, bad := range map[string]map[string]any{"alias": {"node_alias": "bad alias", "node_kind": "mac"}, "kind": {"node_alias": "ok", "node_kind": "bsd"},497 "ttl low": {"node_alias": "ok", "node_kind": "mac", "ttl_minutes": 0}, "ttl high": {"node_alias": "ok", "node_kind": "mac", "ttl_minutes": 1441}} {498 if r := admin.do("POST", "/agents/enrollment-tokens", bad); r.Status != 400 {499 t.Fatalf("%s: %d %s", name, r.Status, r.Raw)500 }501 }502503 // Ingest never started: no CA exists, and the response says so instead of inventing a fingerprint.504 c1 := admin.must("POST", "/agents/enrollment-tokens", body, 201).jsonObj(t)505 token := c1["token"].(string)506 hint := c1["install_hint"].(string)507 if len(token) != 43 || c1["ca_fingerprint_sha256"] != "" || !strings.Contains(hint, "not available") || !strings.Contains(hint, "<CA_FINGERPRINT_UNAVAILABLE>") ||508 !strings.Contains(hint, "MCC_INGEST_ADVERTISE") || !strings.Contains(hint, "--token "+token) {509 t.Fatalf("token without CA: %v", c1)510 }511 et := c1["enrollment_token"].(map[string]any)512 id1 := et["id"].(string)513 exp, _ := time.Parse(time.RFC3339Nano, et["expires_at"].(string))514 if et["status"] != "pending" || et["node_alias"] != "m4mn" || et["created_by"] != h.userIDOf("ada") || time.Until(exp) < 14*time.Minute || time.Until(exp) > 15*time.Minute {515 t.Fatalf("enrollment token: %v (expires in %v)", et, time.Until(exp))516 }517 // Only the hash is stored; the clear token appears nowhere else, ever.518 sum := sha256.Sum256([]byte(token))519 if h.count(`select count(*) from agent_enrollment_tokens where id = $1 and token_hash = $2`, id1, sum[:]) != 1 {520 t.Fatal("the stored value must be the SHA-256 of the token")521 }522 if h.count(`select count(*) from audit_log where params::text like '%'||$1||'%' or detail like '%'||$1||'%'`, token) != 0 {523 t.Fatal("the audit log contains the token")524 }525 if raw := string(admin.must("GET", "/agents/enrollment-tokens", nil, 200).Raw); strings.Contains(raw, token) || strings.Contains(raw, "token_hash") {526 t.Fatalf("the list leaks the token: %s", raw)527 }528529 // With the CA published by ingest and endpoints configured, the hint is a complete command.530 fp := strings.Repeat("ab12", 16)531 h.dbExec(`insert into settings(key, value) values ('pki.ca', jsonb_build_object('pem', '-----BEGIN CERTIFICATE-----', 'fingerprint_sha256', $1::text))`, fp)532 h.Cfg.IngestAdvertise = []string{"192.168.2.20:8731", "10.67.0.20:8731"}533 c2 := admin.must("POST", "/agents/enrollment-tokens", map[string]any{"node_alias": "bhs64", "node_kind": "linux", "ttl_minutes": 60}, 201).jsonObj(t)534 if want := "mcc-agent enroll --endpoints 192.168.2.20:8731,10.67.0.20:8731 --token " + c2["token"].(string) + " --ca-fingerprint sha256:" + fp; c2["install_hint"] != want || c2["ca_fingerprint_sha256"] != fp {535 t.Fatalf("install hint: %v\nwant %s", c2["install_hint"], want)536 }537 id2 := dig(c2, "enrollment_token", "id").(string)538 if c2["token"] == token {539 t.Fatal("tokens must be unique")540 }541542 // Lifecycle: pending, expired, revoked, consumed.543 status := func() map[string]string {544 out := map[string]string{}545 for _, it := range admin.must("GET", "/agents/enrollment-tokens", nil, 200).jsonItems(t) {546 out[it["id"].(string)] = it["status"].(string)547 }548 return out549 }550 if s := status(); s[id1] != "pending" || s[id2] != "pending" {551 t.Fatalf("statuses: %v", s)552 }553 h.dbExec(`update agent_enrollment_tokens set expires_at = now() - interval '1 second' where id = $1`, id1)554 if s := status(); s[id1] != "expired" {555 t.Fatalf("expired token: %v", s)556 }557 h.expireStepUp("ada")558 if r := admin.do("DELETE", "/agents/enrollment-tokens/"+id2, nil); r.code() != "step_up_required" {559 t.Fatalf("revoke without step-up: %d %s", r.Status, r.Raw)560 }561 admin.stepUp()562 admin.must("DELETE", "/agents/enrollment-tokens/"+id2, nil, 204)563 admin.must("DELETE", "/agents/enrollment-tokens/"+id2, nil, 204) // idempotent564 if s := status(); s[id2] != "revoked" || h.count(`select count(*) from audit_log where action = 'agent.enrollment.revoke' and target_id = $1`, id2) != 1 {565 t.Fatalf("revoked token: %v", s)566 }567 c3 := admin.must("POST", "/agents/enrollment-tokens", map[string]any{"node_alias": "used", "node_kind": "mac"}, 201).jsonObj(t)568 id3 := dig(c3, "enrollment_token", "id").(string)569 h.dbExec(`update agent_enrollment_tokens set consumed_at = now() where id = $1`, id3) // what ingest does at enrollment570 if r := admin.do("DELETE", "/agents/enrollment-tokens/"+id3, nil); r.Status != 404 || status()[id3] != "consumed" {571 t.Fatalf("a consumed token is not an unused token: %d %s", r.Status, r.Raw)572 }573 if r := admin.do("DELETE", "/agents/enrollment-tokens/"+ids.New().String(), nil); r.Status != 404 {574 t.Fatalf("unknown token: %d", r.Status)575 }576 if got := admin.walkPages("/agents/enrollment-tokens", 1, "id"); strings.Join(got, ",") != id3+","+id2+","+id1 {577 t.Fatalf("tokens, most recent first: %v", got)578 }579 if h.count(`select count(*) from audit_log where action = 'agent.enrollment.create' and outcome = 'success'`) != 3 {580 t.Fatal("token creation not audited")581 }582583 // Agents.584 n1 := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second})585 n2 := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Hour})586 a1 := h.seedAgent(n1, 5*time.Second)587 a2 := h.seedAgent(n2, time.Hour)588 if r := admin.do("POST", "/agents/enrollment-tokens", map[string]any{"node_alias": "ALPHA", "node_kind": "mac"}); r.Status != 409 || !strings.Contains(string(r.Raw), "agent_enrolled") {589 t.Fatalf("token for a node that already has an agent: %d %s", r.Status, r.Raw)590 }591 agents := admin.must("GET", "/agents", nil, 200).jsonItems(t)592 if len(agents) != 2 || agents[0]["id"] != a2 {593 t.Fatalf("agents, most recently enrolled first: %v", agents)594 }595 ag := rowBy(agents, "id", a1)596 if ag["node_alias"] != "alpha" || ag["version"] != "1.0.0" || ag["os"] != "darwin" || ag["freshness"] != "fresh" || ag["cert_fingerprint_sha256"] != "ab12" ||597 fmt.Sprint(ag["capabilities"]) != "[inventory.refresh service.restart]" || dig(ag, "overhead", "cpu_pct") != 0.4 || dig(ag, "overhead", "rss_bytes") != 20971520.0 ||598 rowBy(agents, "id", a2)["freshness"] != "offline" {599 t.Fatalf("agent mapping: %v", ag)600 }601 wantExactlyOnce(t, "by node", admin.walkPages("/agents?node_id="+n2, 5, "id"), a2)602 wantExactlyOnce(t, "paged", admin.walkPages("/agents", 1, "id"), a1, a2)603604 for name, bad := range map[string]map[string]any{"no reason": {}, "blank": {"reason": " "}, "long": {"reason": strings.Repeat("r", 501)}} {605 if r := admin.do("POST", "/agents/"+a1+"/revoke", bad); r.Status != 400 {606 t.Fatalf("revoke %s: %d %s", name, r.Status, r.Raw)607 }608 }609 h.expireStepUp("ada")610 if r := admin.do("POST", "/agents/"+a1+"/revoke", map[string]any{"reason": "lost laptop"}); r.code() != "step_up_required" {611 t.Fatalf("agent revocation without step-up: %d %s", r.Status, r.Raw)612 }613 admin.stepUp()614 rev := admin.must("POST", "/agents/"+a1+"/revoke", map[string]any{"reason": "lost laptop"}, 200).jsonObj(t)615 if rev["revoked_at"] == nil || rev["revoked_reason"] != "lost laptop" || rev["freshness"] != "offline" {616 t.Fatalf("revoked agent: %v", rev)617 }618 if r := admin.do("POST", "/agents/"+a1+"/revoke", map[string]any{"reason": "again"}); r.Status != 409 {619 t.Fatalf("revoking twice: %d", r.Status)620 }621 if r := admin.do("POST", "/agents/"+ids.New().String()+"/revoke", map[string]any{"reason": "x"}); r.Status != 404 {622 t.Fatalf("unknown agent: %d", r.Status)623 }624 if h.count(`select count(*) from agents where id = $1 and revoked_at is not null and revoked_reason = 'lost laptop'`, a1) != 1 ||625 h.count(`select count(*) from audit_log where action = 'agent.revoke' and target_id = $1 and detail = 'lost laptop' and params->>'node_id' = $2`, a1, n1) != 1 ||626 h.count(`select count(*) from events_outbox where topic = 'node.updated' and payload->>'node_id' = $1 and payload->>'change' = 'agent'`, n1) != 1 {627 t.Fatal("agent revocation must be stored, audited and announced")628 }629 wantExactlyOnce(t, "active only by default", admin.walkPages("/agents", 5, "id"), a2)630 wantExactlyOnce(t, "revoked=true", admin.walkPages("/agents?revoked=true", 5, "id"), a1, a2)631 if d := admin.must("GET", "/nodes/"+n1, nil, 200).jsonObj(t); d["agent"] != nil {632 t.Fatalf("a revoked agent is not the node's agent: %v", d["agent"])633 }634 admin.must("POST", "/agents/enrollment-tokens", map[string]any{"node_alias": "alpha", "node_kind": "mac"}, 201) // re-enrollment is possible again635}636637func TestSettingsAllowlistAndAdminStatus(t *testing.T) {638 h := newHarness(t)639 root := h.owner()640 admin := h.member(root, "admin", "ada", false)641 h.dbExec(`insert into settings(key, value) values ('pki.ca', '{"pem":"x","fingerprint_sha256":"y"}'), ('events.pruned', '{"pruned_through": 1}')`)642643 // Only allowlisted keys exist for the API; unset ones report their default.644 list := admin.must("GET", "/settings", nil, 200).jsonItems(t)645 keys := map[string]any{}646 for _, s := range list {647 keys[s["key"].(string)] = s["value"]648 if s["description"] == nil || s["updated_at"] == nil {649 t.Fatalf("setting without description: %v", s)650 }651 }652 if len(list) != 7 || keys["retention.raw_days"] != 7.0 || keys["retention.1m_days"] != 30.0 || keys["retention.1h_days"] != 400.0 ||653 keys["orchestrator.frozen"] != false || keys["alerts.default_severity"] != "warning" || keys["pki.ca"] != nil {654 t.Fatalf("settings: %v", keys)655 }656 put := func(c *client, key string, value any) resp {657 return c.do("PUT", "/settings/"+key, map[string]any{"value": value})658 }659 for _, key := range []string{"pki.ca", "events.pruned", "made.up", "retention.raw_days.x"} {660 if r := put(admin, key, 1); r.Status != 404 {661 t.Fatalf("unknown or internal key %s: %d %s", key, r.Status, r.Raw)662 }663 }664 for name, c := range map[string][2]any{665 "too low": {"retention.raw_days", 0}, "too high": {"retention.raw_days", 31}, "fraction": {"retention.raw_days", 7.5}, "string": {"retention.raw_days", "7"},666 "null": {"retention.raw_days", nil}, "object": {"retention.1h_days", map[string]any{"days": 400}}, "enum": {"alerts.default_severity", "fatal"},667 "bool as string": {"orchestrator.frozen", "true"}, "negative": {"alerts.default_for_seconds", -1},668 } {669 if r := put(root, c[0].(string), c[1]); r.Status != 400 || r.code() != "validation_failed" {670 t.Fatalf("%s: %d %s", name, r.Status, r.Raw)671 }672 }673 if r := admin.do("PUT", "/settings/retention.raw_days", map[string]any{}); r.Status != 400 {674 t.Fatalf("missing value: %d", r.Status)675 }676 s := put(admin, "retention.raw_days", 14).jsonObj(t)677 if s["value"] != 14.0 || s["updated_by"] != h.userIDOf("ada") || s["key"] != "retention.raw_days" {678 t.Fatalf("updated setting: %v", s)679 }680 if put(admin, "alerts.default_severity", "critical").Status != 200 || put(admin, "retention.raw_days", 10).Status != 200 {681 t.Fatal("valid settings refused")682 }683 if h.count(`select count(*) from settings where key = 'retention.raw_days' and value = '10'::jsonb`) != 1 ||684 h.count(`select count(*) from settings where key = 'alerts.default_severity' and value = '"critical"'::jsonb`) != 1 {685 t.Fatal("settings not stored as JSON values")686 }687 if h.count(`select count(*) from audit_log where action = 'setting.update' and target_id = 'retention.raw_days' and outcome = 'success'688 and (params->>'from')::int = 14 and (params->>'to')::int = 10`) != 1 {689 t.Fatal("setting change must be audited with old and new value")690 }691 h.expireStepUp("ada")692 if r := put(admin, "retention.raw_days", 9); r.code() != "step_up_required" {693 t.Fatalf("setting without step-up: %d %s", r.Status, r.Raw)694 }695 admin.stepUp()696697 // The emergency stop is the owner's.698 if r := put(admin, "orchestrator.frozen", true); r.Status != 403 || r.code() != "forbidden" {699 t.Fatalf("admin freezing the orchestrator: %d %s", r.Status, r.Raw)700 }701 st := admin.must("GET", "/admin/status", nil, 200).jsonObj(t)702 if st["frozen"] != false {703 t.Fatalf("frozen: %v", st)704 }705 if r := put(root, "orchestrator.frozen", true); r.Status != 200 {706 t.Fatalf("owner freezing the orchestrator: %d %s", r.Status, r.Raw)707 }708 if h.count(`select count(*) from audit_log where action = 'setting.update' and target_id = 'orchestrator.frozen' and outcome = 'denied'`) != 1 ||709 h.count(`select count(*) from settings where key = 'orchestrator.frozen' and value = 'true'::jsonb`) != 1 {710 t.Fatal("freeze not stored or the refusal not audited")711 }712713 // Status: nothing has reported yet.714 st = admin.must("GET", "/admin/status", nil, 200).jsonObj(t)715 if st["frozen"] != true || dig(st, "database", "status") != "ok" || dig(st, "database", "latency_ms") == nil || dig(st, "ingest", "status") != "unknown" ||716 dig(st, "orchestrator", "status") != "unknown" || dig(st, "ingest", "heartbeat_at") != nil || dig(st, "backup", "status") != "unknown" ||717 st["agents_connected"] != 0.0 || st["last_report_at"] != nil || st["version"] == "" || st["sse_clients"] != 0.0 {718 t.Fatalf("status without heartbeats: %v", st)719 }720 beat := func(service string, age time.Duration) {721 h.dbExec(`insert into service_heartbeats(service, instance, version, started_at, beat_at) values ($1, 'i', '1.2.3', now(), now() - make_interval(secs => $2))722 on conflict (service) do update set beat_at = excluded.beat_at`, service, age.Seconds())723 }724 for age, want := range map[time.Duration]string{2 * time.Second: "ok", 60 * time.Second: "degraded", 10 * time.Minute: "down"} {725 beat("telemetry-ingest", age)726 beat("action-orchestrator", 3*time.Second)727 beat("control-api", time.Second) // not a monitored dependency of itself728 st := admin.must("GET", "/admin/status", nil, 200).jsonObj(t)729 if dig(st, "ingest", "status") != want || dig(st, "orchestrator", "status") != "ok" || dig(st, "ingest", "heartbeat_at") == nil ||730 !strings.Contains(fmt.Sprint(dig(st, "ingest", "detail")), "1.2.3") {731 t.Fatalf("heartbeat %v old: %v", age, st)732 }733 }734 // Agents, last report, backups.735 n1 := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: 4 * time.Second})736 n2 := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Hour})737 h.seedAgent(n1, 4*time.Second)738 h.seedAgent(n2, time.Hour)739 h.dbExec(`insert into backups(id, name, kind, last_status, last_run_at, last_success_at) values ($1, 'app', 'app', 'failed', now(), null)`, ids.New().String())740 st = admin.must("GET", "/admin/status", nil, 200).jsonObj(t)741 if st["agents_connected"] != 1.0 || st["last_report_at"] == nil || !strings.Contains(fmt.Sprint(dig(st, "ingest", "detail")), "1 of 2") || dig(st, "backup", "status") != "failed" {742 t.Fatalf("status: %v", st)743 }744 // The Command Center's own backup, once observed, is the one reported.745 h.dbExec(`insert into backups(id, name, kind, last_status, last_run_at, last_success_at) values ($1, 'mcc-db', 'mcc', 'ok', now(), now())`, ids.New().String())746 if st := admin.must("GET", "/admin/status", nil, 200).jsonObj(t); dig(st, "backup", "status") != "ok" || dig(st, "backup", "last_success_at") == nil {747 t.Fatalf("backup status: %v", st["backup"])748 }749 h.dbExec(`update backups set last_status = 'ok', last_success_at = null where kind = 'mcc'`) // "ok" without a success on record is not evidence750 if st := admin.must("GET", "/admin/status", nil, 200).jsonObj(t); dig(st, "backup", "status") != "unknown" {751 t.Fatalf("backup without evidence: %v", st["backup"])752 }753}754