SPB Git

Repository Public

admin_test.go 38.8 KB 754 lines go

1package api_test23import (4	"crypto/sha256"5	"encoding/json"6	"fmt"7	"sort"8	"strings"9	"testing"10	"time"1112	"clustr.spboucher.ai/mcc/internal/api"13	"clustr.spboucher.ai/mcc/internal/identity/vauth"14	"clustr.spboucher.ai/mcc/internal/ids"15)1617// opPath fills the path parameters of a contract path with plausible values.18func opPath(p string) string {19	r := strings.NewReplacer("{id}", ids.New().String(), "{passkeyId}", ids.New().String(), "{key}", "retention.raw_days", "{name}", "default")20	return r.Replace(p)21}2223// Operations of the read side and of administration (this file's scope),24// with the query parameters the contract makes mandatory.25var clusterAndAdminOps = map[string]string{26	"GetClusterOverview": "", "GetClusterTrends": "?metric=cpu.usage", "ListNodes": "", "CreateNode": "", "GetNode": "", "UpdateNode": "", "RetireNode": "",27	"GetNodeMetrics": "?metrics=cpu.usage", "CompareMetrics": "?metric=cpu.usage&node_ids=" + ids.New().String(), "GetMetricCatalog": "",28	"ListNodeProcesses": "", "ListNodeServices": "", "GetNodeHealthHistory": "", "ListServices": "", "GetService": "", "UpdateService": "",29	"GetServiceHistory": "", "ListInferenceServers": "", "GetInferenceServer": "", "GetNetworkTopology": "", "ListNetworkInterfaces": "",30	"ListVolumes": "", "ListNasTargets": "", "CreateNasTarget": "", "DeleteNasTarget": "", "ListBackups": "", "ListMyDashboards": "",31	"PutMyDashboard": "", "DeleteMyDashboard": "", "ListActivity": "", "Search": "?q=x",32	"ListUsers": "", "GetUser": "", "UpdateUser": "", "ListUserPasskeys": "", "RevokeUserPasskey": "", "ListUserSessions": "", "RevokeSession": "",33	"ListUserDevices": "", "RevokeDevice": "", "ListPolicyGrants": "", "CreatePolicyGrant": "", "DeletePolicyGrant": "", "ListAgents": "",34	"ListEnrollmentTokens": "", "CreateEnrollmentToken": "", "RevokeEnrollmentToken": "", "RevokeAgent": "", "ListSettings": "", "PutSetting": "",35	"GetAdminStatus": "",36}3738func TestEveryClusterAndAdminOperationIsImplemented(t *testing.T) {39	h := newHarness(t)40	root := h.owner()41	h.seedPartitions()42	if len(clusterAndAdminOps) != 51 {43		t.Fatalf("%d operations listed", len(clusterAndAdminOps))44	}45	for name, query := range clusterAndAdminOps {46		op, ok := api.Operations[name]47		if !ok {48			t.Fatalf("operation %s is not in the contract", name)49		}50		var body any51		if op.Method != "GET" && op.Method != "DELETE" {52			body = map[string]any{}53		}54		r := root.do(op.Method, opPath(op.Path)+query, body)55		if r.Status == 501 || r.Status >= 500 || r.Status == 401 || r.code() == "forbidden" || r.code() == "step_up_required" {56			t.Errorf("%s %s as owner: %d %s", op.Method, op.Path, r.Status, r.Raw)57		}58	}59}6061func TestAdministrationIsRefusedToViewersAndOperators(t *testing.T) {62	h := newHarness(t)63	root := h.owner()64	viewer := h.member(root, "viewer", "vera", false)65	operator := h.member(root, "operator", "oscar", true)66	checked := 067	var names []string68	for name, op := range api.Operations {69		switch op.Permission {70		case "users.manage", "agents.manage", "settings.manage":71			names = append(names, name)72		}73	}74	sort.Strings(names)75	for _, name := range names {76		op := api.Operations[name]77		var body any78		if op.Method != "GET" && op.Method != "DELETE" {79			body = map[string]any{}80		}81		for who, c := range map[string]*client{"viewer": viewer, "operator": operator} {82			r := c.do(op.Method, opPath(op.Path), body)83			if r.Status != 403 || r.code() != "forbidden" || !strings.Contains(string(r.Raw), op.Permission) {84				t.Errorf("%s %s (%s) as %s: %d %s", op.Method, op.Path, name, who, r.Status, r.Raw)85			}86		}87		checked++88	}89	// users (9), policy grants (3), agents and enrollment tokens (5), settings and status (3), plus the settings.manage operations90	// outside the Admin tag (service update, NAS create / delete).91	if checked != 23 {92		t.Fatalf("%d administration operations checked: %v", checked, names)93	}94	if n := h.count(`select count(*) from audit_log where action = 'authz.denied' and outcome = 'denied'`); n != 2*checked {95		t.Fatalf("%d refusals audited, want %d", n, 2*checked)96	}97	if h.count(`select count(*) from settings where key not in ('events.pruned')`)+h.count(`select count(*) from policy_grants`)+98		h.count(`select count(*) from agent_enrollment_tokens`)+h.count(`select count(*) from nas_targets`) != 0 {99		t.Fatal("a refused request changed state")100	}101}102103func TestUserAdministrationRules(t *testing.T) {104	h := newHarness(t)105	root := h.owner()106	admin := h.member(root, "admin", "ada", false)107	viewer := h.member(root, "viewer", "vera", false)108	h.member(root, "operator", "oscar", true)109	id := h.userIDOf110111	// Listing, filtering, paging.112	if got := admin.walkPages("/users", 1, "handle"); strings.Join(got, ",") != "ada,oscar,root,vera" {113		t.Fatalf("users: %v", got)114	}115	wantExactlyOnce(t, "role filter", admin.walkPages("/users?role=viewer", 5, "handle"), "vera")116	wantExactlyOnce(t, "q", admin.walkPages("/users?q=USER%20o", 5, "handle"), "oscar")117	wantExactlyOnce(t, "disabled", admin.walkPages("/users?disabled=true", 5, "handle"))118	u := admin.must("GET", "/users/"+id("vera"), nil, 200).jsonObj(t)119	if u["handle"] != "vera" || u["role"] != "viewer" || u["passkey_count"] != 1.0 || u["last_seen_at"] == nil || u["disabled"] != false {120		t.Fatalf("user: %v", u)121	}122	for _, bad := range []string{"role=king", "limit=0", "cursor=x"} {123		if r := admin.do("GET", "/users?"+bad, nil); r.Status != 400 {124			t.Fatalf("%s: %d", bad, r.Status)125		}126	}127	if r := admin.do("GET", "/users/"+ids.New().String(), nil); r.Status != 404 {128		t.Fatalf("unknown user: %d", r.Status)129	}130131	// Role changes.132	patch := func(c *client, handle string, body map[string]any) resp {133		return c.do("PATCH", "/users/"+id(handle), body)134	}135	if r := patch(admin, "root", map[string]any{"role": "viewer"}); r.Status != 403 {136		t.Fatalf("admin demoting an owner: %d %s", r.Status, r.Raw)137	}138	if r := patch(admin, "root", map[string]any{"disabled": true}); r.Status != 403 {139		t.Fatalf("admin disabling an owner: %d %s", r.Status, r.Raw)140	}141	if r := patch(admin, "vera", map[string]any{"role": "owner"}); r.Status != 403 {142		t.Fatalf("admin granting owner: %d %s", r.Status, r.Raw)143	}144	if r := patch(admin, "ada", map[string]any{"role": "viewer"}); r.Status != 409 || r.code() != "self_modification" {145		t.Fatalf("self-modification: %d %s", r.Status, r.Raw)146	}147	if r := patch(root, "root", map[string]any{"disabled": true}); r.Status != 409 || r.code() != "self_modification" {148		t.Fatalf("owner disabling itself: %d %s", r.Status, r.Raw)149	}150	for name, body := range map[string]map[string]any{"empty": {}, "bad role": {"role": "king"}} {151		if r := patch(admin, "vera", body); r.Status != 400 {152			t.Fatalf("%s: %d %s", name, r.Status, r.Raw)153		}154	}155	if r := admin.do("PATCH", "/users/"+ids.New().String(), map[string]any{"role": "viewer"}); r.Status != 404 {156		t.Fatalf("unknown user: %d", r.Status)157	}158	if got := patch(admin, "vera", map[string]any{"role": "operator"}); got.Status != 200 || got.jsonObj(t)["role"] != "operator" {159		t.Fatalf("admin promoting a viewer: %d %s", got.Status, got.Raw)160	}161	h.expireStepUp("ada")162	if r := patch(admin, "vera", map[string]any{"role": "viewer"}); r.Status != 403 || r.code() != "step_up_required" {163		t.Fatalf("role change without step-up: %d %s", r.Status, r.Raw)164	}165	admin.stepUp()166	patch(admin, "vera", map[string]any{"role": "viewer"})167	if h.count(`select count(*) from audit_log where action = 'user.update' and outcome = 'success' and target_id = $1`, id("vera")) != 2 ||168		h.count(`select count(*) from audit_log where action = 'user.update' and outcome = 'denied'`) != 3 {169		t.Fatal("role changes and refusals must be audited")170	}171172	// Disabling revokes the sessions; enabling does not restore them.173	viewer.must("GET", "/me", nil, 200)174	if d := patch(admin, "vera", map[string]any{"disabled": true}).jsonObj(t); d["disabled"] != true || d["disabled_at"] == nil {175		t.Fatalf("disabled user: %v", d)176	}177	if r := viewer.do("GET", "/me", nil); r.Status != 401 {178		t.Fatalf("disabled user still signed in: %d", r.Status)179	}180	if r := viewer.login(); r.Status == 200 {181		t.Fatal("a disabled user signed in")182	}183	wantExactlyOnce(t, "disabled", admin.walkPages("/users?disabled=true", 5, "handle"), "vera")184	patch(admin, "vera", map[string]any{"disabled": false})185	if r := viewer.login(); r.Status != 200 {186		t.Fatalf("enabled user cannot sign in: %d %s", r.Status, r.Raw)187	}188189	// The last owner able to sign in cannot be demoted or disabled, even by another owner session.190	second := h.member(root, "owner", "olga", false)191	_ = second192	h.dbExec(`update passkeys set revoked_at = now() where user_id = $1`, id("root")) // root's session survives, but root can no longer sign in193	for _, body := range []map[string]any{{"role": "admin"}, {"disabled": true}} {194		if r := patch(root, "olga", body); r.Status != 409 || r.code() != "last_owner" {195			t.Fatalf("demoting the last active owner (%v): %d %s", body, r.Status, r.Raw)196		}197	}198	if h.count(`select count(*) from users where handle = 'olga' and role = 'owner' and disabled_at is null`) != 1 {199		t.Fatal("the last owner was modified")200	}201}202203func TestAdminRevokesSessionsDevicesAndPasskeys(t *testing.T) {204	h := newHarness(t)205	root := h.owner()206	admin := h.member(root, "admin", "ada", false)207	viewer := h.member(root, "viewer", "vera", false)208	vera, rootID := h.userIDOf("vera"), h.userIDOf("root")209210	// Sessions.211	ss := admin.must("GET", "/users/"+vera+"/sessions", nil, 200).jsonItems(t)212	if len(ss) != 1 || ss[0]["user_id"] != vera || ss[0]["current"] != false || ss[0]["ip"] == nil || ss[0]["kind"] != "cookie" {213		t.Fatalf("sessions of another user: %v", ss)214	}215	sid := ss[0]["id"].(string)216	h.expireStepUp("ada")217	if r := admin.do("DELETE", "/sessions/"+sid, nil); r.Status != 403 || r.code() != "step_up_required" {218		t.Fatalf("revoking a session without step-up: %d %s", r.Status, r.Raw)219	}220	viewer.must("GET", "/me", nil, 200)221	admin.stepUp()222	admin.must("DELETE", "/sessions/"+sid, nil, 204)223	if r := viewer.do("GET", "/me", nil); r.Status != 401 {224		t.Fatalf("revoked session still valid: %d", r.Status)225	}226	if r := admin.do("DELETE", "/sessions/"+sid, nil); r.Status != 404 {227		t.Fatalf("revoking twice: %d", r.Status)228	}229	if h.count(`select count(*) from audit_log where action = 'session.revoke' and target_id = $1 and actor_label = 'ada' and detail = 'revoked_by_admin'`, sid) != 1 {230		t.Fatal("session revocation not audited")231	}232233	// Devices: revoking one ends its sessions.234	if viewer.login().Status != 200 {235		t.Fatal("sign-in failed")236	}237	ds := admin.must("GET", "/users/"+vera+"/devices", nil, 200).jsonItems(t)238	if len(ds) != 1 || ds[0]["kind"] != "web" || ds[0]["user_id"] != vera || ds[0]["current"] != false || strings.Contains(fmt.Sprint(ds[0]), "push_token") {239		t.Fatalf("devices: %v", ds)240	}241	admin.must("DELETE", "/devices/"+ds[0]["id"].(string), nil, 204)242	if r := viewer.do("GET", "/me", nil); r.Status != 401 {243		t.Fatalf("session of a revoked device still valid: %d", r.Status)244	}245	if d := admin.must("GET", "/users/"+vera+"/devices", nil, 200).jsonItems(t); d[0]["revoked_at"] == nil {246		t.Fatalf("revoked device: %v", d)247	}248249	// Passkeys: the last one cannot be revoked; a second one can, and stays listed as revoked.250	if viewer.login().Status != 200 {251		t.Fatal("sign-in failed")252	}253	keys := admin.must("GET", "/users/"+vera+"/passkeys", nil, 200).jsonItems(t)254	if len(keys) != 1 {255		t.Fatalf("passkeys: %v", keys)256	}257	if r := admin.do("DELETE", "/users/"+vera+"/passkeys/"+keys[0]["id"].(string), nil); r.Status != 409 || r.code() != "last_passkey" {258		t.Fatalf("revoking the last passkey: %d %s", r.Status, r.Raw)259	}260	other := vauth.New(h.URL)261	var ch struct {262		ChallengeID string          `json:"challenge_id"`263		PublicKey   json.RawMessage `json:"public_key"`264	}265	viewer.must("POST", "/auth/webauthn/registration/begin", map[string]any{}, 200).json(t, &ch)266	cred, err := other.Create([]byte(`{"publicKey":` + string(ch.PublicKey) + `}`))267	if err != nil {268		t.Fatal(err)269	}270	viewer.must("POST", "/auth/webauthn/registration/finish", map[string]any{"challenge_id": ch.ChallengeID, "credential": json.RawMessage(cred), "client": viewer.clientInfo()}, 200)271	keys = admin.must("GET", "/users/"+vera+"/passkeys", nil, 200).jsonItems(t)272	if len(keys) != 2 {273		t.Fatalf("passkeys after adding one: %v", keys)274	}275	first := keys[0]["id"].(string)276	if r := admin.do("DELETE", "/users/"+rootID+"/passkeys/"+first, nil); r.Status != 403 {277		t.Fatalf("admin on an owner's passkeys: %d", r.Status)278	}279	if r := admin.do("DELETE", "/users/"+h.userIDOf("ada")+"/passkeys/"+first, nil); r.Status != 404 {280		t.Fatalf("a passkey is only revocable under its own user: %d", r.Status)281	}282	admin.must("DELETE", "/users/"+vera+"/passkeys/"+first, nil, 204)283	keys = admin.must("GET", "/users/"+vera+"/passkeys", nil, 200).jsonItems(t)284	if len(keys) != 2 || rowBy(keys, "id", first)["revoked_at"] == nil {285		t.Fatalf("revoked passkeys stay in the administration view: %v", keys)286	}287	if r := viewer.do("GET", "/me", nil); r.Status != 401 {288		t.Fatalf("session opened with a revoked passkey still valid: %d", r.Status)289	}290	if r := viewer.login(); r.Status == 200 {291		t.Fatal("signed in with a revoked passkey")292	}293	if h.count(`select count(*) from audit_log where action = 'passkey.revoke' and outcome = 'success' and actor_label = 'ada'`) != 1 {294		t.Fatal("passkey revocation not audited")295	}296297	// An admin does not touch an owner's credentials; an owner does.298	rs := root.must("GET", "/users/"+rootID+"/sessions", nil, 200).jsonItems(t)299	if rs[0]["current"] != true {300		t.Fatalf("own session must be flagged current: %v", rs)301	}302	if r := admin.do("DELETE", "/sessions/"+rs[0]["id"].(string), nil); r.Status != 403 {303		t.Fatalf("admin revoking an owner's session: %d", r.Status)304	}305	rd := root.must("GET", "/users/"+rootID+"/devices", nil, 200).jsonItems(t)306	if r := admin.do("DELETE", "/devices/"+rd[0]["id"].(string), nil); r.Status != 403 {307		t.Fatalf("admin revoking an owner's device: %d", r.Status)308	}309	root.must("GET", "/me", nil, 200)310	if h.count(`select count(*) from audit_log where outcome = 'denied' and action in ('session.revoke', 'device.revoke', 'passkey.revoke')`) != 3 {311		t.Fatal("refused attempts on an owner must be audited")312	}313	as := root.must("GET", "/users/"+h.userIDOf("ada")+"/sessions", nil, 200).jsonItems(t)314	root.must("DELETE", "/sessions/"+as[0]["id"].(string), nil, 204)315	if r := admin.do("GET", "/users", nil); r.Status != 401 {316		t.Fatalf("admin session revoked by the owner: %d", r.Status)317	}318	for _, p := range []string{"/users/" + ids.New().String() + "/sessions", "/users/" + ids.New().String() + "/devices", "/users/" + ids.New().String() + "/passkeys"} {319		if r := root.do("GET", p, nil); r.Status != 404 {320			t.Fatalf("%s: %d", p, r.Status)321		}322	}323}324325func TestPolicyGrantsTakeEffectAndAreValidated(t *testing.T) {326	h := newHarness(t)327	root := h.owner()328	admin := h.member(root, "admin", "ada", false)329	operator := h.member(root, "operator", "oscar", false)330	viewer := h.member(root, "viewer", "vera", false)331	x := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second, IP: "192.168.2.50", Site: "lab"})332	y := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Second, IP: "192.168.2.51"})333	oscar, vera := h.userIDOf("oscar"), h.userIDOf("vera")334	g := func(kind, subject, perm, rk, rid, effect string) map[string]any {335		b := map[string]any{"subject_kind": kind, "subject": subject, "permission": perm, "effect": effect}336		if rk != "" {337			b["resource_kind"] = rk338		}339		if rid != "" {340			b["resource_id"] = rid341		}342		return b343	}344345	// A scoped deny blocks, at once, and only that resource.346	operator.must("GET", "/nodes/"+x, nil, 200)347	deny := admin.must("POST", "/policy-grants", g("user", oscar, "nodes.read", "node", x, "deny"), 201).jsonObj(t)348	if deny["effect"] != "deny" || deny["resource_kind"] != "node" || deny["resource_id"] != x || deny["created_by"] != h.userIDOf("ada") {349		t.Fatalf("created grant: %v", deny)350	}351	if r := operator.do("GET", "/nodes/"+x, nil); r.Status != 404 {352		t.Fatalf("scoped deny not applied: %d", r.Status)353	}354	operator.must("GET", "/nodes/"+y, nil, 200)355	if got := operator.walkPages("/nodes", 5, "alias"); strings.Join(got, ",") != "bravo" {356		t.Fatalf("list under a scoped deny: %v", got)357	}358	admin.must("DELETE", "/policy-grants/"+deny["id"].(string), nil, 204)359	operator.must("GET", "/nodes/"+x, nil, 200)360	if r := admin.do("DELETE", "/policy-grants/"+deny["id"].(string), nil); r.Status != 404 {361		t.Fatalf("deleting a grant twice: %d", r.Status)362	}363	// A deny on a site covers the nodes of that site.364	site := admin.must("POST", "/policy-grants", g("role", "operator", "nodes.read", "site", "lab", "deny"), 201).jsonObj(t)365	if r := operator.do("GET", "/nodes/"+x, nil); r.Status != 404 {366		t.Fatalf("site-scoped deny not applied: %d", r.Status)367	}368	admin.must("DELETE", "/policy-grants/"+site["id"].(string), nil, 204)369370	// An allow extends: private fields for viewers, visible in /me too.371	if n := viewer.must("GET", "/nodes/"+x, nil, 200).jsonObj(t); dig(n, "node", "primary_ip") != nil {372		t.Fatal("viewer sees a private field by default")373	}374	allow := admin.must("POST", "/policy-grants", g("role", "viewer", "nodes.private.read", "", "", "allow"), 201).jsonObj(t)375	if n := viewer.must("GET", "/nodes/"+x, nil, 200).jsonObj(t); dig(n, "node", "primary_ip") != "192.168.2.50" {376		t.Fatal("allow grant not applied")377	}378	if me := string(viewer.must("GET", "/me", nil, 200).Raw); !strings.Contains(me, "nodes.private.read") {379		t.Fatalf("/me must reflect the grant: %s", me)380	}381	// Deny wins over allow.382	admin.must("POST", "/policy-grants", g("user", vera, "nodes.private.read", "node", x, "deny"), 201)383	if n := viewer.must("GET", "/nodes/"+x, nil, 200).jsonObj(t); dig(n, "node", "primary_ip") != nil {384		t.Fatal("deny must win over allow")385	}386	if n := viewer.must("GET", "/nodes/"+y, nil, 200).jsonObj(t); dig(n, "node", "primary_ip") != "192.168.2.51" {387		t.Fatal("the deny is scoped to one node")388	}389390	// Invalid or pointless grants are refused with a precise code.391	cases := []struct {392		name   string393		body   map[string]any394		status int395		code   string396		reason string397	}{398		{"unknown permission", g("role", "viewer", "nodes.fly", "", "", "allow"), 400, "validation_failed", ""},399		{"unknown role", g("role", "king", "nodes.read", "", "", "deny"), 400, "validation_failed", ""},400		{"bad subject kind", g("group", "x", "nodes.read", "", "", "deny"), 400, "validation_failed", ""},401		{"bad effect", g("role", "viewer", "nodes.read", "", "", "maybe"), 400, "validation_failed", ""},402		{"user id shape", g("user", "vera", "nodes.read", "", "", "deny"), 400, "validation_failed", ""},403		{"resource id without kind", g("role", "viewer", "nodes.read", "", x, "deny"), 400, "validation_failed", ""},404		{"bad resource kind", g("role", "viewer", "nodes.read", "rack", "r1", "deny"), 400, "validation_failed", ""},405		{"node id shape", g("role", "viewer", "nodes.read", "node", "alpha", "deny"), 400, "validation_failed", ""},406		{"unknown user", g("user", ids.New().String(), "nodes.read", "", "", "deny"), 422, "unprocessable", "unknown_subject"},407		{"unknown node", g("role", "viewer", "nodes.read", "node", ids.New().String(), "deny"), 422, "unprocessable", "unknown_resource"},408		{"critical to admins", g("role", "admin", "actions.run.critical", "", "", "allow"), 422, "unprocessable", "grant_ineffective"},409		{"critical to an operator", g("user", oscar, "actions.run.critical", "", "", "allow"), 422, "unprocessable", "grant_ineffective"},410		{"user management below admin", g("role", "operator", "users.manage", "", "", "allow"), 422, "unprocessable", "grant_ineffective"},411		{"settings below admin", g("user", vera, "settings.manage", "", "", "allow"), 422, "unprocessable", "grant_ineffective"},412		{"scoped cluster-wide permission", g("role", "admin", "users.manage", "node", x, "deny"), 422, "unprocessable", "permission_not_scopable"},413		{"duplicate", g("role", "viewer", "nodes.private.read", "", "", "allow"), 409, "conflict", ""},414		{"opposite", g("role", "viewer", "nodes.private.read", "", "", "deny"), 409, "conflict", ""},415	}416	before := h.count(`select count(*) from policy_grants`)417	for _, c := range cases {418		r := admin.do("POST", "/policy-grants", c.body)419		if r.Status != c.status || r.code() != c.code || (c.reason != "" && !strings.Contains(string(r.Raw), `"reason":"`+c.reason+`"`)) {420			t.Errorf("%s: %d %s, want %d %s %s", c.name, r.Status, r.Raw, c.status, c.code, c.reason)421		}422	}423	if r := root.do("POST", "/policy-grants", g("role", "owner", "users.manage", "", "", "deny")); r.Status != 422 || !strings.Contains(string(r.Raw), "lockout") {424		t.Errorf("locking owners out: %d %s", r.Status, r.Raw)425	}426	if h.count(`select count(*) from policy_grants`) != before {427		t.Fatal("a refused grant was stored")428	}429430	// An admin neither targets owners nor hands out what it does not hold.431	for name, body := range map[string]map[string]any{432		"owner role": g("role", "owner", "nodes.read", "", "", "deny"), "owner user": g("user", h.userIDOf("root"), "logs.read", "", "", "deny")} {433		if r := admin.do("POST", "/policy-grants", body); r.Status != 403 {434			t.Errorf("admin targeting an %s: %d %s", name, r.Status, r.Raw)435		}436	}437	ownerGrant := root.grant("role", "owner", "nodes.read", "node", y, "deny")438	if r := admin.do("DELETE", "/policy-grants/"+ownerGrant, nil); r.Status != 403 {439		t.Errorf("admin deleting a grant on owners: %d", r.Status)440	}441	root.must("DELETE", "/policy-grants/"+ownerGrant, nil, 204)442	root.grant("user", h.userIDOf("ada"), "logs.read", "", "", "deny")443	if r := admin.do("POST", "/policy-grants", g("user", vera, "logs.read", "", "", "allow")); r.Status != 403 {444		t.Errorf("admin granting a permission it does not hold: %d %s", r.Status, r.Raw)445	}446	root.must("POST", "/policy-grants", g("user", vera, "logs.read", "", "", "allow"), 201)447448	// Step-up and roles.449	if r := operator.do("POST", "/policy-grants", g("role", "viewer", "nodes.read", "", "", "deny")); r.Status != 403 || r.code() != "forbidden" {450		t.Fatalf("operator creating a grant: %d", r.Status)451	}452	h.expireStepUp("ada")453	if r := admin.do("POST", "/policy-grants", g("role", "viewer", "jobs.read", "", "", "deny")); r.code() != "step_up_required" {454		t.Fatalf("grant without step-up: %d %s", r.Status, r.Raw)455	}456	if r := admin.do("DELETE", "/policy-grants/"+allow["id"].(string), nil); r.code() != "step_up_required" {457		t.Fatalf("grant deletion without step-up: %d %s", r.Status, r.Raw)458	}459460	// Listing: most recent first, filters, paging.461	all := admin.must("GET", "/policy-grants", nil, 200).jsonItems(t)462	if len(all) != 4 || all[0]["permission"] != "logs.read" || all[0]["effect"] != "allow" || all[3]["id"] != allow["id"] {463		t.Fatalf("grants: %v", all)464	}465	if got := admin.walkPages("/policy-grants", 1, "id"); len(got) != 4 || got[3] != allow["id"] {466		t.Fatalf("paged grants: %v", got)467	}468	wantExactlyOnce(t, "by subject", admin.walkPages("/policy-grants?subject_kind=user&subject="+vera, 5, "permission"), "nodes.private.read", "logs.read")469	wantExactlyOnce(t, "by permission", admin.walkPages("/policy-grants?permission=nodes.private.read", 5, "effect"), "allow", "deny")470	for _, bad := range []string{"permission=nope", "subject_kind=group", "cursor=x"} {471		if r := admin.do("GET", "/policy-grants?"+bad, nil); r.Status != 400 {472			t.Fatalf("%s: %d", bad, r.Status)473		}474	}475	if h.count(`select count(*) from audit_log where action = 'policy.grant.create' and outcome = 'success'`) != 7 ||476		h.count(`select count(*) from audit_log where action = 'policy.grant.delete' and outcome = 'success'`) != 3 ||477		h.count(`select count(*) from audit_log where action like 'policy.grant.%' and outcome = 'denied'`) != 4 {478		t.Fatalf("grant audit trail: %d created, %d deleted, %d denied",479			h.count(`select count(*) from audit_log where action = 'policy.grant.create' and outcome = 'success'`),480			h.count(`select count(*) from audit_log where action = 'policy.grant.delete' and outcome = 'success'`),481			h.count(`select count(*) from audit_log where action like 'policy.grant.%' and outcome = 'denied'`))482	}483}484485func TestEnrollmentTokensAndAgents(t *testing.T) {486	h := newHarness(t)487	root := h.owner()488	admin := h.member(root, "admin", "ada", false)489	body := map[string]any{"node_alias": "m4mn", "node_kind": "mac"}490491	h.expireStepUp("ada")492	if r := admin.do("POST", "/agents/enrollment-tokens", body); r.Status != 403 || r.code() != "step_up_required" {493		t.Fatalf("token without step-up: %d %s", r.Status, r.Raw)494	}495	admin.stepUp()496	for name, bad := range map[string]map[string]any{"alias": {"node_alias": "bad alias", "node_kind": "mac"}, "kind": {"node_alias": "ok", "node_kind": "bsd"},497		"ttl low": {"node_alias": "ok", "node_kind": "mac", "ttl_minutes": 0}, "ttl high": {"node_alias": "ok", "node_kind": "mac", "ttl_minutes": 1441}} {498		if r := admin.do("POST", "/agents/enrollment-tokens", bad); r.Status != 400 {499			t.Fatalf("%s: %d %s", name, r.Status, r.Raw)500		}501	}502503	// Ingest never started: no CA exists, and the response says so instead of inventing a fingerprint.504	c1 := admin.must("POST", "/agents/enrollment-tokens", body, 201).jsonObj(t)505	token := c1["token"].(string)506	hint := c1["install_hint"].(string)507	if len(token) != 43 || c1["ca_fingerprint_sha256"] != "" || !strings.Contains(hint, "not available") || !strings.Contains(hint, "<CA_FINGERPRINT_UNAVAILABLE>") ||508		!strings.Contains(hint, "MCC_INGEST_ADVERTISE") || !strings.Contains(hint, "--token "+token) {509		t.Fatalf("token without CA: %v", c1)510	}511	et := c1["enrollment_token"].(map[string]any)512	id1 := et["id"].(string)513	exp, _ := time.Parse(time.RFC3339Nano, et["expires_at"].(string))514	if et["status"] != "pending" || et["node_alias"] != "m4mn" || et["created_by"] != h.userIDOf("ada") || time.Until(exp) < 14*time.Minute || time.Until(exp) > 15*time.Minute {515		t.Fatalf("enrollment token: %v (expires in %v)", et, time.Until(exp))516	}517	// Only the hash is stored; the clear token appears nowhere else, ever.518	sum := sha256.Sum256([]byte(token))519	if h.count(`select count(*) from agent_enrollment_tokens where id = $1 and token_hash = $2`, id1, sum[:]) != 1 {520		t.Fatal("the stored value must be the SHA-256 of the token")521	}522	if h.count(`select count(*) from audit_log where params::text like '%'||$1||'%' or detail like '%'||$1||'%'`, token) != 0 {523		t.Fatal("the audit log contains the token")524	}525	if raw := string(admin.must("GET", "/agents/enrollment-tokens", nil, 200).Raw); strings.Contains(raw, token) || strings.Contains(raw, "token_hash") {526		t.Fatalf("the list leaks the token: %s", raw)527	}528529	// With the CA published by ingest and endpoints configured, the hint is a complete command.530	fp := strings.Repeat("ab12", 16)531	h.dbExec(`insert into settings(key, value) values ('pki.ca', jsonb_build_object('pem', '-----BEGIN CERTIFICATE-----', 'fingerprint_sha256', $1::text))`, fp)532	h.Cfg.IngestAdvertise = []string{"192.168.2.20:8731", "10.67.0.20:8731"}533	c2 := admin.must("POST", "/agents/enrollment-tokens", map[string]any{"node_alias": "bhs64", "node_kind": "linux", "ttl_minutes": 60}, 201).jsonObj(t)534	if want := "mcc-agent enroll --endpoints 192.168.2.20:8731,10.67.0.20:8731 --token " + c2["token"].(string) + " --ca-fingerprint sha256:" + fp; c2["install_hint"] != want || c2["ca_fingerprint_sha256"] != fp {535		t.Fatalf("install hint: %v\nwant %s", c2["install_hint"], want)536	}537	id2 := dig(c2, "enrollment_token", "id").(string)538	if c2["token"] == token {539		t.Fatal("tokens must be unique")540	}541542	// Lifecycle: pending, expired, revoked, consumed.543	status := func() map[string]string {544		out := map[string]string{}545		for _, it := range admin.must("GET", "/agents/enrollment-tokens", nil, 200).jsonItems(t) {546			out[it["id"].(string)] = it["status"].(string)547		}548		return out549	}550	if s := status(); s[id1] != "pending" || s[id2] != "pending" {551		t.Fatalf("statuses: %v", s)552	}553	h.dbExec(`update agent_enrollment_tokens set expires_at = now() - interval '1 second' where id = $1`, id1)554	if s := status(); s[id1] != "expired" {555		t.Fatalf("expired token: %v", s)556	}557	h.expireStepUp("ada")558	if r := admin.do("DELETE", "/agents/enrollment-tokens/"+id2, nil); r.code() != "step_up_required" {559		t.Fatalf("revoke without step-up: %d %s", r.Status, r.Raw)560	}561	admin.stepUp()562	admin.must("DELETE", "/agents/enrollment-tokens/"+id2, nil, 204)563	admin.must("DELETE", "/agents/enrollment-tokens/"+id2, nil, 204) // idempotent564	if s := status(); s[id2] != "revoked" || h.count(`select count(*) from audit_log where action = 'agent.enrollment.revoke' and target_id = $1`, id2) != 1 {565		t.Fatalf("revoked token: %v", s)566	}567	c3 := admin.must("POST", "/agents/enrollment-tokens", map[string]any{"node_alias": "used", "node_kind": "mac"}, 201).jsonObj(t)568	id3 := dig(c3, "enrollment_token", "id").(string)569	h.dbExec(`update agent_enrollment_tokens set consumed_at = now() where id = $1`, id3) // what ingest does at enrollment570	if r := admin.do("DELETE", "/agents/enrollment-tokens/"+id3, nil); r.Status != 404 || status()[id3] != "consumed" {571		t.Fatalf("a consumed token is not an unused token: %d %s", r.Status, r.Raw)572	}573	if r := admin.do("DELETE", "/agents/enrollment-tokens/"+ids.New().String(), nil); r.Status != 404 {574		t.Fatalf("unknown token: %d", r.Status)575	}576	if got := admin.walkPages("/agents/enrollment-tokens", 1, "id"); strings.Join(got, ",") != id3+","+id2+","+id1 {577		t.Fatalf("tokens, most recent first: %v", got)578	}579	if h.count(`select count(*) from audit_log where action = 'agent.enrollment.create' and outcome = 'success'`) != 3 {580		t.Fatal("token creation not audited")581	}582583	// Agents.584	n1 := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second})585	n2 := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Hour})586	a1 := h.seedAgent(n1, 5*time.Second)587	a2 := h.seedAgent(n2, time.Hour)588	if r := admin.do("POST", "/agents/enrollment-tokens", map[string]any{"node_alias": "ALPHA", "node_kind": "mac"}); r.Status != 409 || !strings.Contains(string(r.Raw), "agent_enrolled") {589		t.Fatalf("token for a node that already has an agent: %d %s", r.Status, r.Raw)590	}591	agents := admin.must("GET", "/agents", nil, 200).jsonItems(t)592	if len(agents) != 2 || agents[0]["id"] != a2 {593		t.Fatalf("agents, most recently enrolled first: %v", agents)594	}595	ag := rowBy(agents, "id", a1)596	if ag["node_alias"] != "alpha" || ag["version"] != "1.0.0" || ag["os"] != "darwin" || ag["freshness"] != "fresh" || ag["cert_fingerprint_sha256"] != "ab12" ||597		fmt.Sprint(ag["capabilities"]) != "[inventory.refresh service.restart]" || dig(ag, "overhead", "cpu_pct") != 0.4 || dig(ag, "overhead", "rss_bytes") != 20971520.0 ||598		rowBy(agents, "id", a2)["freshness"] != "offline" {599		t.Fatalf("agent mapping: %v", ag)600	}601	wantExactlyOnce(t, "by node", admin.walkPages("/agents?node_id="+n2, 5, "id"), a2)602	wantExactlyOnce(t, "paged", admin.walkPages("/agents", 1, "id"), a1, a2)603604	for name, bad := range map[string]map[string]any{"no reason": {}, "blank": {"reason": "  "}, "long": {"reason": strings.Repeat("r", 501)}} {605		if r := admin.do("POST", "/agents/"+a1+"/revoke", bad); r.Status != 400 {606			t.Fatalf("revoke %s: %d %s", name, r.Status, r.Raw)607		}608	}609	h.expireStepUp("ada")610	if r := admin.do("POST", "/agents/"+a1+"/revoke", map[string]any{"reason": "lost laptop"}); r.code() != "step_up_required" {611		t.Fatalf("agent revocation without step-up: %d %s", r.Status, r.Raw)612	}613	admin.stepUp()614	rev := admin.must("POST", "/agents/"+a1+"/revoke", map[string]any{"reason": "lost laptop"}, 200).jsonObj(t)615	if rev["revoked_at"] == nil || rev["revoked_reason"] != "lost laptop" || rev["freshness"] != "offline" {616		t.Fatalf("revoked agent: %v", rev)617	}618	if r := admin.do("POST", "/agents/"+a1+"/revoke", map[string]any{"reason": "again"}); r.Status != 409 {619		t.Fatalf("revoking twice: %d", r.Status)620	}621	if r := admin.do("POST", "/agents/"+ids.New().String()+"/revoke", map[string]any{"reason": "x"}); r.Status != 404 {622		t.Fatalf("unknown agent: %d", r.Status)623	}624	if h.count(`select count(*) from agents where id = $1 and revoked_at is not null and revoked_reason = 'lost laptop'`, a1) != 1 ||625		h.count(`select count(*) from audit_log where action = 'agent.revoke' and target_id = $1 and detail = 'lost laptop' and params->>'node_id' = $2`, a1, n1) != 1 ||626		h.count(`select count(*) from events_outbox where topic = 'node.updated' and payload->>'node_id' = $1 and payload->>'change' = 'agent'`, n1) != 1 {627		t.Fatal("agent revocation must be stored, audited and announced")628	}629	wantExactlyOnce(t, "active only by default", admin.walkPages("/agents", 5, "id"), a2)630	wantExactlyOnce(t, "revoked=true", admin.walkPages("/agents?revoked=true", 5, "id"), a1, a2)631	if d := admin.must("GET", "/nodes/"+n1, nil, 200).jsonObj(t); d["agent"] != nil {632		t.Fatalf("a revoked agent is not the node's agent: %v", d["agent"])633	}634	admin.must("POST", "/agents/enrollment-tokens", map[string]any{"node_alias": "alpha", "node_kind": "mac"}, 201) // re-enrollment is possible again635}636637func TestSettingsAllowlistAndAdminStatus(t *testing.T) {638	h := newHarness(t)639	root := h.owner()640	admin := h.member(root, "admin", "ada", false)641	h.dbExec(`insert into settings(key, value) values ('pki.ca', '{"pem":"x","fingerprint_sha256":"y"}'), ('events.pruned', '{"pruned_through": 1}')`)642643	// Only allowlisted keys exist for the API; unset ones report their default.644	list := admin.must("GET", "/settings", nil, 200).jsonItems(t)645	keys := map[string]any{}646	for _, s := range list {647		keys[s["key"].(string)] = s["value"]648		if s["description"] == nil || s["updated_at"] == nil {649			t.Fatalf("setting without description: %v", s)650		}651	}652	if len(list) != 7 || keys["retention.raw_days"] != 7.0 || keys["retention.1m_days"] != 30.0 || keys["retention.1h_days"] != 400.0 ||653		keys["orchestrator.frozen"] != false || keys["alerts.default_severity"] != "warning" || keys["pki.ca"] != nil {654		t.Fatalf("settings: %v", keys)655	}656	put := func(c *client, key string, value any) resp {657		return c.do("PUT", "/settings/"+key, map[string]any{"value": value})658	}659	for _, key := range []string{"pki.ca", "events.pruned", "made.up", "retention.raw_days.x"} {660		if r := put(admin, key, 1); r.Status != 404 {661			t.Fatalf("unknown or internal key %s: %d %s", key, r.Status, r.Raw)662		}663	}664	for name, c := range map[string][2]any{665		"too low": {"retention.raw_days", 0}, "too high": {"retention.raw_days", 31}, "fraction": {"retention.raw_days", 7.5}, "string": {"retention.raw_days", "7"},666		"null": {"retention.raw_days", nil}, "object": {"retention.1h_days", map[string]any{"days": 400}}, "enum": {"alerts.default_severity", "fatal"},667		"bool as string": {"orchestrator.frozen", "true"}, "negative": {"alerts.default_for_seconds", -1},668	} {669		if r := put(root, c[0].(string), c[1]); r.Status != 400 || r.code() != "validation_failed" {670			t.Fatalf("%s: %d %s", name, r.Status, r.Raw)671		}672	}673	if r := admin.do("PUT", "/settings/retention.raw_days", map[string]any{}); r.Status != 400 {674		t.Fatalf("missing value: %d", r.Status)675	}676	s := put(admin, "retention.raw_days", 14).jsonObj(t)677	if s["value"] != 14.0 || s["updated_by"] != h.userIDOf("ada") || s["key"] != "retention.raw_days" {678		t.Fatalf("updated setting: %v", s)679	}680	if put(admin, "alerts.default_severity", "critical").Status != 200 || put(admin, "retention.raw_days", 10).Status != 200 {681		t.Fatal("valid settings refused")682	}683	if h.count(`select count(*) from settings where key = 'retention.raw_days' and value = '10'::jsonb`) != 1 ||684		h.count(`select count(*) from settings where key = 'alerts.default_severity' and value = '"critical"'::jsonb`) != 1 {685		t.Fatal("settings not stored as JSON values")686	}687	if h.count(`select count(*) from audit_log where action = 'setting.update' and target_id = 'retention.raw_days' and outcome = 'success'688		and (params->>'from')::int = 14 and (params->>'to')::int = 10`) != 1 {689		t.Fatal("setting change must be audited with old and new value")690	}691	h.expireStepUp("ada")692	if r := put(admin, "retention.raw_days", 9); r.code() != "step_up_required" {693		t.Fatalf("setting without step-up: %d %s", r.Status, r.Raw)694	}695	admin.stepUp()696697	// The emergency stop is the owner's.698	if r := put(admin, "orchestrator.frozen", true); r.Status != 403 || r.code() != "forbidden" {699		t.Fatalf("admin freezing the orchestrator: %d %s", r.Status, r.Raw)700	}701	st := admin.must("GET", "/admin/status", nil, 200).jsonObj(t)702	if st["frozen"] != false {703		t.Fatalf("frozen: %v", st)704	}705	if r := put(root, "orchestrator.frozen", true); r.Status != 200 {706		t.Fatalf("owner freezing the orchestrator: %d %s", r.Status, r.Raw)707	}708	if h.count(`select count(*) from audit_log where action = 'setting.update' and target_id = 'orchestrator.frozen' and outcome = 'denied'`) != 1 ||709		h.count(`select count(*) from settings where key = 'orchestrator.frozen' and value = 'true'::jsonb`) != 1 {710		t.Fatal("freeze not stored or the refusal not audited")711	}712713	// Status: nothing has reported yet.714	st = admin.must("GET", "/admin/status", nil, 200).jsonObj(t)715	if st["frozen"] != true || dig(st, "database", "status") != "ok" || dig(st, "database", "latency_ms") == nil || dig(st, "ingest", "status") != "unknown" ||716		dig(st, "orchestrator", "status") != "unknown" || dig(st, "ingest", "heartbeat_at") != nil || dig(st, "backup", "status") != "unknown" ||717		st["agents_connected"] != 0.0 || st["last_report_at"] != nil || st["version"] == "" || st["sse_clients"] != 0.0 {718		t.Fatalf("status without heartbeats: %v", st)719	}720	beat := func(service string, age time.Duration) {721		h.dbExec(`insert into service_heartbeats(service, instance, version, started_at, beat_at) values ($1, 'i', '1.2.3', now(), now() - make_interval(secs => $2))722			on conflict (service) do update set beat_at = excluded.beat_at`, service, age.Seconds())723	}724	for age, want := range map[time.Duration]string{2 * time.Second: "ok", 60 * time.Second: "degraded", 10 * time.Minute: "down"} {725		beat("telemetry-ingest", age)726		beat("action-orchestrator", 3*time.Second)727		beat("control-api", time.Second) // not a monitored dependency of itself728		st := admin.must("GET", "/admin/status", nil, 200).jsonObj(t)729		if dig(st, "ingest", "status") != want || dig(st, "orchestrator", "status") != "ok" || dig(st, "ingest", "heartbeat_at") == nil ||730			!strings.Contains(fmt.Sprint(dig(st, "ingest", "detail")), "1.2.3") {731			t.Fatalf("heartbeat %v old: %v", age, st)732		}733	}734	// Agents, last report, backups.735	n1 := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: 4 * time.Second})736	n2 := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Hour})737	h.seedAgent(n1, 4*time.Second)738	h.seedAgent(n2, time.Hour)739	h.dbExec(`insert into backups(id, name, kind, last_status, last_run_at, last_success_at) values ($1, 'app', 'app', 'failed', now(), null)`, ids.New().String())740	st = admin.must("GET", "/admin/status", nil, 200).jsonObj(t)741	if st["agents_connected"] != 1.0 || st["last_report_at"] == nil || !strings.Contains(fmt.Sprint(dig(st, "ingest", "detail")), "1 of 2") || dig(st, "backup", "status") != "failed" {742		t.Fatalf("status: %v", st)743	}744	// The Command Center's own backup, once observed, is the one reported.745	h.dbExec(`insert into backups(id, name, kind, last_status, last_run_at, last_success_at) values ($1, 'mcc-db', 'mcc', 'ok', now(), now())`, ids.New().String())746	if st := admin.must("GET", "/admin/status", nil, 200).jsonObj(t); dig(st, "backup", "status") != "ok" || dig(st, "backup", "last_success_at") == nil {747		t.Fatalf("backup status: %v", st["backup"])748	}749	h.dbExec(`update backups set last_status = 'ok', last_success_at = null where kind = 'mcc'`) // "ok" without a success on record is not evidence750	if st := admin.must("GET", "/admin/status", nil, 200).jsonObj(t); dig(st, "backup", "status") != "unknown" {751		t.Fatalf("backup without evidence: %v", st["backup"])752	}753}754
Files main
Menu

Theme