SPB Git

Repository Public

ops_actions_test.go 29.1 KB 688 lines go

1package api_test23import (4	"context"5	"path/filepath"6	"sync"7	"testing"8	"time"910	"github.com/google/uuid"1112	"clustr.spboucher.ai/mcc/internal/orchestrator"13	"clustr.spboucher.ai/mcc/internal/orchestrator/fakeagent"14	"clustr.spboucher.ai/mcc/internal/protocol"15	"clustr.spboucher.ai/mcc/internal/testdb"16)1718// opsEnv is a control-api plus a real orchestrator and a fake transport+agent19// on the same throwaway database.20type opsEnv struct {21	*harness22	root, admin, oper, viewer *client23	node, node2               uuid.UUID24	agentID                   uuid.UUID25	svc, rawSvc               uuid.UUID26	orch                      *orchestrator.Orchestrator27	agent                     *fakeagent.Agent28}2930var testCaps = []string{"inventory.refresh", "diagnostic.run", "service.restart", "service.stop", "service.logs.tail", "system.reboot",31	"cache.purge", "job.run:disk.report", "job.run:sleep"}3233func newOpsEnv(t *testing.T) *opsEnv {34	t.Helper()35	h := newHarness(t)36	e := &opsEnv{harness: h}37	e.root = h.owner()38	e.admin = h.member(e.root, "admin", "alice", false)39	e.oper = h.member(e.root, "operator", "oscar", true)40	e.viewer = h.member(e.root, "viewer", "victor", false)41	now := time.Now().UTC()42	e.node = testdb.Node(t, h.Pool, "T-ALPHA")43	e.node2 = testdb.Node(t, h.Pool, "T-BETA")44	e.agentID = testdb.Agent(t, h.Pool, e.node, now, testCaps...)45	testdb.Agent(t, h.Pool, e.node2, now, testCaps...)46	testdb.State(t, h.Pool, e.node, now, map[string]any{"load.1": 1.0})47	testdb.State(t, h.Pool, e.node2, now, map[string]any{"load.1": 5.0})48	e.svc = testdb.Service(t, h.Pool, e.node, "web", true, "online")49	e.rawSvc = testdb.Service(t, h.Pool, e.node, "scratch", false, "online")50	signer, err := orchestrator.LoadSigner(context.Background(), h.Pool, filepath.Join(h.Cfg.SecretsDir, "command-signing.key"))51	if err != nil {52		t.Fatal(err)53	}54	e.orch = orchestrator.New(h.Pool, signer, orchestrator.Options{})55	e.agent = fakeagent.New(h.Pool, e.agentID)56	return e57}5859func (e *opsEnv) step() {60	e.t.Helper()61	if err := e.orch.Step(context.Background()); err != nil {62		e.t.Fatal(err)63	}64}6566// run lets the orchestrator and the agent work until the action is final.67func (e *opsEnv) run(id string) string {68	e.t.Helper()69	for i := 0; i < 20; i++ {70		e.step()71		if _, err := e.agent.Run(context.Background()); err != nil {72			e.t.Fatal(err)73		}74		e.step()75		var st string76		e.Pool.QueryRow(context.Background(), `select state::text from actions where id=$1`, id).Scan(&st)77		switch st {78		case "succeeded", "failed", "cancelled", "timed_out", "rejected", "expired":79			return st80		}81	}82	e.t.Fatal("action never finished")83	return ""84}8586type actionBody struct {87	ID             string         `json:"id"`88	State          string         `json:"state"`89	Type           string         `json:"type"`90	OutcomeUnknown bool           `json:"outcome_unknown"`91	ErrorCode      string         `json:"error_code"`92	TargetLabel    string         `json:"target_label"`93	NodeAlias      string         `json:"node_alias"`94	Result         map[string]any `json:"result"`95	Approval       bool           `json:"approval_required"`96}9798func act(typ, kind string, target uuid.UUID, extra ...any) map[string]any {99	b := map[string]any{"type": typ, "target": map[string]any{"kind": kind, "id": target}}100	for i := 0; i+1 < len(extra); i += 2 {101		b[extra[i].(string)] = extra[i+1]102	}103	return b104}105106func (c *client) post(body any) resp {107	return c.do("POST", "/actions", body, "Idempotency-Key", uuid.NewString())108}109110func (c *client) create(body any) actionBody {111	c.h.t.Helper()112	r := c.post(body)113	if r.Status != 201 {114		c.h.t.Fatalf("create action: %d %s", r.Status, r.Raw)115	}116	var a actionBody117	r.json(c.h.t, &a)118	return a119}120121func TestActionCatalogEndpoint(t *testing.T) {122	e := newOpsEnv(t)123	var out struct {124		Items []struct {125			Type, Criticality, Permission string126			Available                     bool127			RequiresStepUp                bool           `json:"requires_step_up"`128			RequiresApproval              bool           `json:"requires_approval"`129			RequiresConfirmation          bool           `json:"requires_confirmation"`130			RequiresJustification         bool           `json:"requires_justification"`131			ParamsSchema                  map[string]any `json:"params_schema"`132			LockScope                     string         `json:"lock_scope"`133		}134	}135	e.viewer.must("GET", "/action-types", nil, 200).json(t, &out)136	by := map[string]int{}137	for i, it := range out.Items {138		by[it.Type] = i139		if it.Permission != "actions.run."+it.Criticality {140			t.Errorf("%s: permission %s", it.Type, it.Permission)141		}142		if it.RequiresStepUp != (it.Criticality == "high" || it.Criticality == "critical") || it.RequiresApproval != (it.Criticality == "critical") {143			t.Errorf("%s: flags inconsistent with criticality", it.Type)144		}145		if it.ParamsSchema["additionalProperties"] != false {146			t.Errorf("%s: schema must refuse unknown fields", it.Type)147		}148	}149	want := map[string]string{"inventory.refresh": "low", "diagnostic.run": "low", "service.health_check": "low", "service.logs.tail": "low",150		"service.restart": "medium", "service.start": "medium", "service.stop": "high", "agent.restart": "medium", "cache.purge": "medium",151		"node.maintenance.set": "medium", "job.run": "medium", "job.cancel": "medium", "model.server.start": "high", "model.server.stop": "high",152		"deploy.release": "high", "system.update.schedule": "high", "system.reboot": "critical", "system.shutdown": "critical"}153	if len(out.Items) != len(want) {154		t.Fatalf("%d types, want %d", len(out.Items), len(want))155	}156	for typ, crit := range want {157		i, ok := by[typ]158		if !ok || out.Items[i].Criticality != crit {159			t.Errorf("%s: missing or wrong criticality", typ)160		}161	}162	for _, typ := range []string{"deploy.release", "system.update.schedule"} {163		if out.Items[by[typ]].Available {164			t.Errorf("%s must not be available", typ)165		}166	}167	if rb := out.Items[by["system.reboot"]]; !rb.RequiresConfirmation || !rb.RequiresJustification || !rb.Available {168		t.Errorf("system.reboot flags: %+v", rb)169	}170}171172func TestCreateActionRefusals(t *testing.T) {173	e := newOpsEnv(t)174	ctx := context.Background()175	offline := testdb.Node(t, e.Pool, "T-OFFLINE")176	testdb.Agent(t, e.Pool, offline, time.Now().Add(-10*time.Minute), testCaps...)177	noAgent := testdb.Node(t, e.Pool, "T-NOAGENT")178	bare := testdb.Node(t, e.Pool, "T-BARE")179	testdb.Agent(t, e.Pool, bare, time.Now(), "inventory.refresh")180	maint := testdb.Node(t, e.Pool, "T-MAINT")181	testdb.Agent(t, e.Pool, maint, time.Now(), testCaps...)182	e.Pool.Exec(ctx, `update nodes set maintenance=true where id=$1`, maint)183	reboot := func(extra ...any) map[string]any {184		return act("system.reboot", "node", e.node, extra...)185	}186187	cases := []struct {188		name   string189		who    *client190		body   map[string]any191		status int192		code   string193		audit  string // audit action expected with outcome denied/failure194		prep   func()195	}{196		{"viewer cannot run a low action", e.viewer, act("inventory.refresh", "node", e.node), 403, "forbidden", "authz.denied", nil},197		{"operator cannot run a high action", e.oper, act("service.stop", "service", e.svc), 403, "forbidden", "authz.denied", nil},198		{"admin cannot run a critical action", e.admin, reboot("confirm", "T-ALPHA", "justification", "because"), 403, "forbidden", "authz.denied", nil},199		{"high action without step-up", e.admin, act("service.stop", "service", e.svc), 403, "step_up_required", "action.create",200			func() { e.expireStepUp("alice") }},201		{"critical action without step-up", e.root, reboot("confirm", "T-ALPHA", "justification", "because"), 403, "step_up_required", "action.create",202			func() { e.expireStepUp("root") }},203		{"missing confirmation", e.root, reboot("justification", "because"), 422, "confirmation_mismatch", "action.create", func() { e.root.stepUp() }},204		{"wrong confirmation", e.root, reboot("confirm", "t-alpha", "justification", "because"), 422, "confirmation_mismatch", "action.create", nil},205		{"confirmation of another node", e.root, reboot("confirm", "T-BETA", "justification", "because"), 422, "confirmation_mismatch", "action.create", nil},206		{"missing justification", e.root, reboot("confirm", "T-ALPHA"), 422, "justification_required", "action.create", nil},207		{"blank justification", e.root, reboot("confirm", "T-ALPHA", "justification", "   "), 422, "justification_required", "action.create", nil},208		{"unknown type", e.oper, act("shell.exec", "node", e.node, "params", map[string]any{"cmd": "id"}), 400, "validation_failed", "action.create", nil},209		{"unknown parameter", e.oper, act("inventory.refresh", "node", e.node, "params", map[string]any{"path": "/etc"}), 422, "precheck_failed", "action.create", nil},210		{"parameter outside its enum", e.oper, act("diagnostic.run", "node", e.node, "params", map[string]any{"profile": "everything"}), 422, "precheck_failed", "action.create", nil},211		{"missing required parameter", e.oper, act("diagnostic.run", "node", e.node), 422, "precheck_failed", "action.create", nil},212		{"parameter looking like a path", e.oper, act("cache.purge", "node", e.node, "params", map[string]any{"cache": "../../etc"}), 422, "precheck_failed", "action.create", nil},213		{"parameter of the wrong type", e.oper, act("service.logs.tail", "service", e.svc, "params", map[string]any{"lines": "all"}), 422, "precheck_failed", "action.create", nil},214		{"parameter out of range", e.oper, act("service.logs.tail", "service", e.svc, "params", map[string]any{"lines": 100000}), 422, "precheck_failed", "action.create", nil},215		{"unavailable type", e.root, act("deploy.release", "service", e.svc), 422, "action_unavailable", "action.create", nil},216		{"internal type", e.oper, act("job.run", "job", uuid.New(), "params", map[string]any{"template": "sleep"}), 422, "unprocessable", "action.create", nil},217		{"wrong kind of target", e.oper, act("inventory.refresh", "service", e.svc), 422, "precheck_failed", "action.create", nil},218		{"unknown target", e.oper, act("inventory.refresh", "node", uuid.New()), 404, "not_found", "action.create", nil},219		{"agent offline", e.oper, act("inventory.refresh", "node", offline), 422, "agent_offline", "action.create", nil},220		{"no agent enrolled", e.oper, act("inventory.refresh", "node", noAgent), 422, "agent_offline", "action.create", nil},221		{"capability not declared by the agent", e.oper, act("cache.purge", "node", bare, "params", map[string]any{"cache": "pip"}), 422, "capability_missing", "action.create", nil},222		{"unmanaged service", e.oper, act("service.restart", "service", e.rawSvc), 422, "precheck_failed", "action.create", nil},223		{"node in maintenance", e.oper, act("inventory.refresh", "node", maint), 422, "node_in_maintenance", "action.create", nil},224		{"orchestrator frozen", e.oper, act("inventory.refresh", "node", e.node), 503, "orchestrator_frozen", "action.create",225			func() { e.Pool.Exec(ctx, `insert into settings(key,value) values ('orchestrator.frozen','true')`) }},226	}227	for _, c := range cases {228		if c.prep != nil {229			c.prep()230		}231		before := e.count(`select count(*) from audit_log where action=$1 and outcome in ('denied','failure')`, c.audit)232		r := c.who.post(c.body)233		if r.Status != c.status || r.code() != c.code {234			t.Errorf("%s: got %d %s (%s), want %d %s", c.name, r.Status, r.code(), r.Raw, c.status, c.code)235		}236		if n := e.count(`select count(*) from actions`); n != 0 {237			t.Fatalf("%s: left %d action row(s)", c.name, n)238		}239		if after := e.count(`select count(*) from audit_log where action=$1 and outcome in ('denied','failure')`, c.audit); after != before+1 {240			t.Errorf("%s: refusal not audited (%s: %d -> %d)", c.name, c.audit, before, after)241		}242	}243	if n := e.count(`select count(*) from action_locks`) + e.count(`select count(*) from agent_commands`); n != 0 {244		t.Fatalf("refusals left locks or commands: %d", n)245	}246	e.Pool.Exec(ctx, `delete from settings where key='orchestrator.frozen'`)247248	// Missing or malformed Idempotency-Key.249	if r := e.oper.do("POST", "/actions", act("inventory.refresh", "node", e.node)); r.Status != 400 {250		t.Errorf("missing Idempotency-Key: %d %s", r.Status, r.Raw)251	}252	if r := e.oper.do("POST", "/actions", act("inventory.refresh", "node", e.node), "Idempotency-Key", "short"); r.Status != 400 {253		t.Errorf("short Idempotency-Key: %d %s", r.Status, r.Raw)254	}255	// The documented override lets an action through on a node in maintenance, and is audited.256	a := e.oper.create(act("inventory.refresh", "node", maint, "override_maintenance", true))257	if a.State != "queued" || e.count(`select count(*) from audit_log where action='action.create' and outcome='success' and (params->>'override_maintenance')::boolean`) != 1 {258		t.Fatalf("override: %+v", a)259	}260	// Maintenance-related actions need no override.261	if a := e.oper.create(act("node.maintenance.set", "node", e.node, "params", map[string]any{"enabled": true})); a.State != "queued" {262		t.Fatalf("%+v", a)263	}264}265266func TestScopedDenyGrant(t *testing.T) {267	e := newOpsEnv(t)268	// Oscar may not run medium actions on T-ALPHA; everything else is unchanged. Written before269	// his first request, so the policy cache cannot hold an older view.270	if _, err := e.Pool.Exec(context.Background(), `insert into policy_grants(id, subject_kind, subject, permission, resource_kind, resource_id, effect)271		select $1, 'user', id::text, 'actions.run.medium', 'node', $2, 'deny' from users where handle='oscar'`, uuid.New(), e.node.String()); err != nil {272		t.Fatal(err)273	}274	time.Sleep(3100 * time.Millisecond) // policy cache lifetime275	if r := e.oper.post(act("service.restart", "service", e.svc)); r.Status != 403 || r.code() != "forbidden" {276		t.Fatalf("scoped deny not applied: %d %s", r.Status, r.Raw)277	}278	if r := e.oper.post(act("cache.purge", "node", e.node, "params", map[string]any{"cache": "pip"})); r.Status != 403 {279		t.Fatalf("scoped deny not applied to the node itself: %d", r.Status)280	}281	if e.count(`select count(*) from actions`) != 0 || e.count(`select count(*) from audit_log where action='authz.denied' and target_kind='node' and target_id=$1`, e.node.String()) != 2 {282		t.Fatal("denied requests left rows or were not audited against the node")283	}284	e.oper.create(act("cache.purge", "node", e.node2, "params", map[string]any{"cache": "pip"})) // other node: allowed285	e.oper.create(act("inventory.refresh", "node", e.node))                                      // other criticality: allowed286}287288func TestIdempotency(t *testing.T) {289	e := newOpsEnv(t)290	body := act("service.restart", "service", e.svc)291	key := uuid.NewString()292	r1 := e.admin.do("POST", "/actions", body, "Idempotency-Key", key)293	r2 := e.admin.do("POST", "/actions", body, "Idempotency-Key", key)294	var a1, a2 actionBody295	r1.json(t, &a1)296	r2.json(t, &a2)297	if r1.Status != 201 || r2.Status != 200 || a1.ID == "" || a1.ID != a2.ID {298		t.Fatalf("replay: %d %d %s %s", r1.Status, r2.Status, a1.ID, a2.ID)299	}300	if r := e.admin.do("POST", "/actions", act("service.restart", "service", e.rawSvc), "Idempotency-Key", key); r.Status != 409 || r.code() != "idempotency_conflict" {301		t.Fatalf("conflicting body: %d %s", r.Status, r.Raw)302	}303	if r := e.admin.do("POST", "/actions", act("service.restart", "service", e.svc, "justification", "x"), "Idempotency-Key", key); r.Status != 409 || r.code() != "idempotency_conflict" {304		t.Fatalf("conflicting justification: %d %s", r.Status, r.Raw)305	}306	// The same key used by another user is another request (target is locked by the first one).307	if r := e.oper.do("POST", "/actions", body, "Idempotency-Key", key); r.Status != 409 || r.code() != "target_locked" {308		t.Fatalf("other user: %d %s", r.Status, r.Raw)309	}310	if e.run(a1.ID) != "succeeded" {311		t.Fatal("not succeeded")312	}313	// A replay after completion still returns the original, finished action: nothing runs twice.314	r3 := e.admin.do("POST", "/actions", body, "Idempotency-Key", key)315	var a3 actionBody316	r3.json(t, &a3)317	if r3.Status != 200 || a3.ID != a1.ID || a3.State != "succeeded" {318		t.Fatalf("replay after completion: %d %+v", r3.Status, a3)319	}320321	// 30 concurrent identical requests: one action, one command, one execution.322	key = uuid.NewString()323	body = act("diagnostic.run", "node", e.node, "params", map[string]any{"profile": "basic"})324	var wg sync.WaitGroup325	var mu sync.Mutex326	codes := map[int]int{}327	idsSeen := map[string]bool{}328	for i := 0; i < 30; i++ {329		wg.Add(1)330		go func() {331			defer wg.Done()332			r := e.root.do("POST", "/actions", body, "Idempotency-Key", key)333			var a actionBody334			r.json(t, &a)335			mu.Lock()336			codes[r.Status]++337			idsSeen[a.ID] = true338			mu.Unlock()339		}()340	}341	wg.Wait()342	if codes[201] != 1 || codes[200] != 29 || len(idsSeen) != 1 {343		t.Fatalf("concurrent identical requests: %v, %d distinct ids", codes, len(idsSeen))344	}345	var id string346	for k := range idsSeen {347		id = k348	}349	if e.run(id) != "succeeded" {350		t.Fatal("not succeeded")351	}352	if n := e.count(`select count(*) from actions where type='diagnostic.run'`); n != 1 {353		t.Fatalf("%d actions", n)354	}355	if n := e.count(`select count(*) from agent_commands where action_id=$1`, id); n != 1 || len(e.agent.Executed()) != 2 {356		t.Fatalf("%d commands, %d executions in total", n, len(e.agent.Executed()))357	}358	if n := e.count(`select count(*) from audit_log where action='action.create' and outcome='success' and params->>'action_id'=$1`, id); n != 1 {359		t.Fatalf("%d audit entries for one creation", n)360	}361}362363func TestActionRateLimit(t *testing.T) {364	e := newOpsEnv(t)365	limited := 0366	for i := 0; i < 33; i++ {367		r := e.oper.post(act("service.logs.tail", "service", e.svc))368		if r.Status == 429 {369			limited++370			if r.Header.Get("Retry-After") == "" {371				t.Fatal("429 without Retry-After")372			}373		}374	}375	if limited != 3 || e.count(`select count(*) from actions`) != 30 {376		t.Fatalf("limited=%d actions=%d", limited, e.count(`select count(*) from actions`))377	}378}379380func TestTargetLockAndRelease(t *testing.T) {381	e := newOpsEnv(t)382	ctx := context.Background()383	body := func() map[string]any { return act("inventory.refresh", "node", e.node) }384	blocked := func(holder string) {385		t.Helper()386		r := e.admin.post(act("cache.purge", "node", e.node, "params", map[string]any{"cache": "pip"}))387		var er struct {388			Error struct {389				Details map[string]any390			}391		}392		r.json(t, &er)393		if r.Status != 409 || r.code() != "target_locked" || er.Error.Details["action_id"] != holder {394			t.Fatalf("expected target_locked by %s: %d %s", holder, r.Status, r.Raw)395		}396	}397	free := func(why string) {398		t.Helper()399		if n := e.count(`select count(*) from action_locks`); n != 0 {400			t.Fatalf("lock not released after %s", why)401		}402	}403	// success404	a := e.oper.create(body())405	blocked(a.ID)406	e.oper.create(act("inventory.refresh", "node", e.node2)) // another node is not locked407	e.oper.create(act("service.restart", "service", e.svc))  // a service lock is distinct from its node lock408	if e.run(a.ID) != "succeeded" {409		t.Fatal("not succeeded")410	}411	e.Pool.Exec(ctx, `delete from action_locks`) // the two side actions412	// cancel while queued413	a = e.oper.create(body())414	blocked(a.ID)415	var got actionBody416	e.oper.must("POST", "/actions/"+a.ID+"/cancel", nil, 200).json(t, &got)417	if got.State != "cancelled" {418		t.Fatalf("%+v", got)419	}420	free("cancel")421	if r := e.oper.do("POST", "/actions/"+a.ID+"/cancel", nil); r.Status != 409 {422		t.Fatalf("cancel of a finished action: %d", r.Status)423	}424	// agent refusal425	a = e.oper.create(body())426	e.agent.Handle = func(*protocol.Command) fakeagent.Behavior { return fakeagent.Behavior{Refuse: "not allowed here"} }427	if e.run(a.ID) != "failed" {428		t.Fatal("not failed")429	}430	free("refusal")431	e.agent.Handle = nil432	// never delivered433	a = e.oper.create(body())434	e.step()435	e.Pool.Exec(ctx, `update agent_commands set expires_at=now()-interval '1 second' where action_id=$1`, a.ID)436	e.step()437	e.oper.must("GET", "/actions/"+a.ID, nil, 200).json(t, &struct{ Action *actionBody }{&got})438	if got.State != "failed" || got.ErrorCode != "agent_unreachable" || got.OutcomeUnknown {439		t.Fatalf("%+v", got)440	}441	free("agent_unreachable")442	// delivered, no result443	a = e.oper.create(body())444	e.step()445	ds, _ := e.agent.Deliver(ctx, nil)446	e.agent.Ack(ctx, ds[0], true, "")447	e.step()448	blocked(a.ID)449	e.Pool.Exec(ctx, `update actions set deadline_at=now()-interval '1 second' where id=$1`, a.ID)450	e.step()451	e.oper.must("GET", "/actions/"+a.ID, nil, 200).json(t, &struct{ Action *actionBody }{&got})452	if got.State != "timed_out" || !got.OutcomeUnknown || got.Result["outcome_unknown"] != true {453		t.Fatalf("%+v", got)454	}455	free("timeout")456	// rejection and expiry of an approval457	e.root.stepUp()458	rb := e.root.create(act("system.reboot", "node", e.node, "confirm", "T-ALPHA", "justification", "kernel panic loop"))459	blocked(rb.ID)460	e.admin.must("POST", "/actions/"+rb.ID+"/reject", map[string]any{"reason": "not now"}, 200)461	free("reject")462	rb = e.root.create(act("system.reboot", "node", e.node, "confirm", "T-ALPHA", "justification", "kernel panic loop"))463	e.Pool.Exec(ctx, `update actions set created_at=now()-interval '31 minutes' where id=$1`, rb.ID)464	e.step()465	free("expiry")466	if e.count(`select count(*) from actions where id=$1 and state='expired'`, rb.ID) != 1 {467		t.Fatal("not expired")468	}469}470471func TestApprovalFlow(t *testing.T) {472	e := newOpsEnv(t)473	ctx := context.Background()474	e.root.stepUp()475	req := func() actionBody {476		return e.root.create(act("system.reboot", "node", e.node, "confirm", "T-ALPHA", "justification", "kernel panic loop after the disk filled up"))477	}478	a := req()479	if a.State != "pending_approval" || !a.Approval || a.NodeAlias != "T-ALPHA" || a.TargetLabel != "T-ALPHA" {480		t.Fatalf("%+v", a)481	}482	e.step()483	if e.count(`select count(*) from agent_commands`) != 0 {484		t.Fatal("a command was issued before approval")485	}486	path := "/actions/" + a.ID487	// The requester cannot approve or reject their own request.488	if r := e.root.do("POST", path+"/approve", map[string]any{}); r.Status != 403 || r.code() != "forbidden" {489		t.Fatalf("self approval: %d %s", r.Status, r.Raw)490	}491	if r := e.root.do("POST", path+"/reject", map[string]any{"reason": "changed my mind"}); r.Status != 403 {492		t.Fatalf("self rejection: %d", r.Status)493	}494	if e.count(`select count(*) from audit_log where action in ('action.approve','action.reject') and outcome='denied'`) != 2 {495		t.Fatal("self decisions not audited")496	}497	// Approving needs the permission…498	if r := e.oper.do("POST", path+"/approve", map[string]any{}); r.Status != 403 || r.code() != "forbidden" {499		t.Fatalf("operator approval: %d %s", r.Status, r.Raw)500	}501	// …and a recent passkey verification.502	e.expireStepUp("alice")503	if r := e.admin.do("POST", path+"/approve", map[string]any{}); r.Status != 403 || r.code() != "step_up_required" {504		t.Fatalf("approval without step-up: %d %s", r.Status, r.Raw)505	}506	e.admin.stepUp()507	// Not while frozen.508	e.Pool.Exec(ctx, `insert into settings(key,value) values ('orchestrator.frozen','true')`)509	if r := e.admin.do("POST", path+"/approve", map[string]any{}); r.Status != 503 || r.code() != "orchestrator_frozen" {510		t.Fatalf("approval while frozen: %d %s", r.Status, r.Raw)511	}512	e.Pool.Exec(ctx, `delete from settings where key='orchestrator.frozen'`)513	var got struct {514		State      string `json:"state"`515		ApprovedBy string `json:"approved_by"`516		ApprovedAt string `json:"approved_at"`517		Name       string `json:"approved_by_display_name"`518	}519	e.admin.must("POST", path+"/approve", map[string]any{"comment": "agreed by phone"}, 200).json(t, &got)520	if got.State != "queued" || got.ApprovedBy == "" || got.ApprovedAt == "" || got.Name != "User alice" {521		t.Fatalf("%+v", got)522	}523	if r := e.admin.do("POST", path+"/approve", map[string]any{}); r.Status != 409 {524		t.Fatalf("second approval: %d", r.Status)525	}526	if e.run(a.ID) != "succeeded" || len(e.agent.Executed()) != 1 {527		t.Fatal("approved action did not run exactly once")528	}529530	// Rejection.531	a = req()532	if r := e.admin.do("POST", "/actions/"+a.ID+"/reject", map[string]any{"reason": ""}); r.Status != 400 {533		t.Fatalf("reject without reason: %d", r.Status)534	}535	var rj struct {536		State  string `json:"state"`537		Reason string `json:"rejection_reason"`538	}539	e.admin.must("POST", "/actions/"+a.ID+"/reject", map[string]any{"reason": "wait for the backup"}, 200).json(t, &rj)540	if rj.State != "rejected" || rj.Reason != "wait for the backup" {541		t.Fatalf("%+v", rj)542	}543	e.step()544	// Expiry: an approval that comes too late is refused and the action is closed.545	a = req()546	e.Pool.Exec(ctx, `update actions set created_at=now()-interval '31 minutes' where id=$1`, a.ID)547	if r := e.admin.do("POST", "/actions/"+a.ID+"/approve", map[string]any{}); r.Status != 409 {548		t.Fatalf("late approval: %d %s", r.Status, r.Raw)549	}550	if e.count(`select count(*) from actions where id=$1 and state='expired'`, a.ID) != 1 || e.count(`select count(*) from action_locks`) != 0 {551		t.Fatal("late approval did not expire the action")552	}553	if e.count(`select count(*) from agent_commands`) != 1 {554		t.Fatal("only the approved action may have produced a command")555	}556}557558func TestActionOutputListAndCancelPermissions(t *testing.T) {559	e := newOpsEnv(t)560	ctx := context.Background()561	a := e.oper.create(act("diagnostic.run", "node", e.node, "params", map[string]any{"profile": "network"}))562	e.agent.Handle = func(*protocol.Command) fakeagent.Behavior {563		return fakeagent.Behavior{Output: []protocol.CommandOutput{{Stream: "stdout", Data: "ping ok"}, {Stream: "stdout", Data: "dns ok"}, {Stream: "stderr", Data: "1 warning"}},564			Result: &protocol.CommandResult{Status: "succeeded", Result: map[string]any{"checks": 2}}}565	}566	if e.run(a.ID) != "succeeded" {567		t.Fatal("not succeeded")568	}569	var det struct {570		Action actionBody571		Events []struct{ Kind, Data string }572	}573	e.viewer.must("GET", "/actions/"+a.ID, nil, 200).json(t, &det)574	if det.Action.Result["checks"] != float64(2) || len(det.Events) < 3 {575		t.Fatalf("%+v", det)576	}577	for _, ev := range det.Events {578		if ev.Kind == "output" {579			t.Fatal("output leaked into the lifecycle events")580		}581	}582	var out struct {583		Items []struct {584			ID           int64585			Stream, Data string586		}587		NextAfter int64 `json:"next_after"`588		Complete  bool589	}590	e.oper.must("GET", "/actions/"+a.ID+"/output?limit=2", nil, 200).json(t, &out)591	if len(out.Items) != 2 || out.Complete || out.Items[0].Data != "ping ok" {592		t.Fatalf("%+v", out)593	}594	e.oper.must("GET", "/actions/"+a.ID+"/output?after="+itoa(out.NextAfter), nil, 200).json(t, &out)595	if len(out.Items) != 1 || !out.Complete || out.Items[0].Stream != "stderr" {596		t.Fatalf("%+v", out)597	}598	// Output needs logs.read: a viewer sees the action, not what the node printed.599	if r := e.viewer.do("GET", "/actions/"+a.ID+"/output", nil); r.Status != 403 {600		t.Fatalf("viewer read output: %d", r.Status)601	}602	// Filters and pagination.603	for i := 0; i < 4; i++ {604		b := e.oper.create(act("service.logs.tail", "service", e.svc))605		e.run(b.ID)606	}607	var list struct {608		Items      []actionBody609		NextCursor string `json:"next_cursor"`610	}611	e.viewer.must("GET", "/actions?type=service.logs.tail&limit=3", nil, 200).json(t, &list)612	if len(list.Items) != 3 || list.NextCursor == "" {613		t.Fatalf("page 1: %d %q", len(list.Items), list.NextCursor)614	}615	first, cursor := list.Items[0].ID, list.NextCursor616	list.NextCursor = ""617	e.viewer.must("GET", "/actions?type=service.logs.tail&limit=3&cursor="+cursor, nil, 200).json(t, &list)618	if len(list.Items) != 1 || list.NextCursor != "" || list.Items[0].ID == first {619		t.Fatalf("page 2: %d", len(list.Items))620	}621	e.viewer.must("GET", "/actions?state=succeeded&node_id="+e.node.String()+"&target_kind=node&criticality=low", nil, 200).json(t, &list)622	if len(list.Items) != 1 || list.Items[0].ID != a.ID {623		t.Fatalf("filters: %d", len(list.Items))624	}625	if r := e.viewer.do("GET", "/actions?cursor=zzz", nil); r.Status != 400 {626		t.Fatalf("bad cursor: %d", r.Status)627	}628	if r := e.viewer.do("GET", "/actions/"+uuid.NewString(), nil); r.Status != 404 {629		t.Fatalf("unknown action: %d", r.Status)630	}631632	// Cancel: not by a bystander without the run permission; by the requester; cooperatively when running.633	e.agent.Handle = nil634	c := e.oper.create(act("inventory.refresh", "node", e.node))635	if r := e.viewer.do("POST", "/actions/"+c.ID+"/cancel", nil); r.Status != 403 {636		t.Fatalf("viewer cancel: %d", r.Status)637	}638	e.step()639	ds, _ := e.agent.Deliver(ctx, nil)640	e.agent.Ack(ctx, ds[0], true, "")641	e.step()642	var got struct {643		State  string `json:"state"`644		Cancel string `json:"cancel_requested_at"`645	}646	e.admin.must("POST", "/actions/"+c.ID+"/cancel", nil, 200).json(t, &got) // admin holds the run permission647	if got.State != "running" || got.Cancel == "" {648		t.Fatalf("%+v", got)649	}650	if asked, _ := e.agent.CancelRequested(ctx, ds[0]); !asked {651		t.Fatal("cancellation not passed to the agent")652	}653	e.agent.Result(ctx, ds[0], protocol.CommandResult{Status: "cancelled"})654	e.step()655	if e.count(`select count(*) from actions where id=$1 and state='cancelled'`, c.ID) != 1 {656		t.Fatal("not cancelled")657	}658	if e.count(`select count(*) from audit_log where action='action.cancel' and outcome='success'`) != 1 {659		t.Fatal("cancel not audited")660	}661}662663func TestMaintenanceSetThroughAPI(t *testing.T) {664	e := newOpsEnv(t)665	a := e.oper.create(act("node.maintenance.set", "node", e.node, "params", map[string]any{"enabled": true, "reason": "RAM upgrade"}))666	if e.run(a.ID) != "succeeded" {667		t.Fatal("not succeeded")668	}669	if e.count(`select count(*) from nodes where id=$1 and maintenance and maintenance_reason='RAM upgrade'`, e.node) != 1 || e.count(`select count(*) from agent_commands`) != 0 {670		t.Fatal("maintenance not set server-side")671	}672	if r := e.oper.post(act("inventory.refresh", "node", e.node)); r.code() != "node_in_maintenance" {673		t.Fatalf("%d %s", r.Status, r.Raw)674	}675}676677func itoa(n int64) string {678	b := []byte{}679	if n == 0 {680		return "0"681	}682	for n > 0 {683		b = append([]byte{byte('0' + n%10)}, b...)684		n /= 10685	}686	return string(b)687}688
Files main
Menu

Theme