Repository Public
1package api_test23import (4 "context"5 "path/filepath"6 "sync"7 "testing"8 "time"910 "github.com/google/uuid"1112 "clustr.spboucher.ai/mcc/internal/orchestrator"13 "clustr.spboucher.ai/mcc/internal/orchestrator/fakeagent"14 "clustr.spboucher.ai/mcc/internal/protocol"15 "clustr.spboucher.ai/mcc/internal/testdb"16)1718// opsEnv is a control-api plus a real orchestrator and a fake transport+agent19// on the same throwaway database.20type opsEnv struct {21 *harness22 root, admin, oper, viewer *client23 node, node2 uuid.UUID24 agentID uuid.UUID25 svc, rawSvc uuid.UUID26 orch *orchestrator.Orchestrator27 agent *fakeagent.Agent28}2930var testCaps = []string{"inventory.refresh", "diagnostic.run", "service.restart", "service.stop", "service.logs.tail", "system.reboot",31 "cache.purge", "job.run:disk.report", "job.run:sleep"}3233func newOpsEnv(t *testing.T) *opsEnv {34 t.Helper()35 h := newHarness(t)36 e := &opsEnv{harness: h}37 e.root = h.owner()38 e.admin = h.member(e.root, "admin", "alice", false)39 e.oper = h.member(e.root, "operator", "oscar", true)40 e.viewer = h.member(e.root, "viewer", "victor", false)41 now := time.Now().UTC()42 e.node = testdb.Node(t, h.Pool, "T-ALPHA")43 e.node2 = testdb.Node(t, h.Pool, "T-BETA")44 e.agentID = testdb.Agent(t, h.Pool, e.node, now, testCaps...)45 testdb.Agent(t, h.Pool, e.node2, now, testCaps...)46 testdb.State(t, h.Pool, e.node, now, map[string]any{"load.1": 1.0})47 testdb.State(t, h.Pool, e.node2, now, map[string]any{"load.1": 5.0})48 e.svc = testdb.Service(t, h.Pool, e.node, "web", true, "online")49 e.rawSvc = testdb.Service(t, h.Pool, e.node, "scratch", false, "online")50 signer, err := orchestrator.LoadSigner(context.Background(), h.Pool, filepath.Join(h.Cfg.SecretsDir, "command-signing.key"))51 if err != nil {52 t.Fatal(err)53 }54 e.orch = orchestrator.New(h.Pool, signer, orchestrator.Options{})55 e.agent = fakeagent.New(h.Pool, e.agentID)56 return e57}5859func (e *opsEnv) step() {60 e.t.Helper()61 if err := e.orch.Step(context.Background()); err != nil {62 e.t.Fatal(err)63 }64}6566// run lets the orchestrator and the agent work until the action is final.67func (e *opsEnv) run(id string) string {68 e.t.Helper()69 for i := 0; i < 20; i++ {70 e.step()71 if _, err := e.agent.Run(context.Background()); err != nil {72 e.t.Fatal(err)73 }74 e.step()75 var st string76 e.Pool.QueryRow(context.Background(), `select state::text from actions where id=$1`, id).Scan(&st)77 switch st {78 case "succeeded", "failed", "cancelled", "timed_out", "rejected", "expired":79 return st80 }81 }82 e.t.Fatal("action never finished")83 return ""84}8586type actionBody struct {87 ID string `json:"id"`88 State string `json:"state"`89 Type string `json:"type"`90 OutcomeUnknown bool `json:"outcome_unknown"`91 ErrorCode string `json:"error_code"`92 TargetLabel string `json:"target_label"`93 NodeAlias string `json:"node_alias"`94 Result map[string]any `json:"result"`95 Approval bool `json:"approval_required"`96}9798func act(typ, kind string, target uuid.UUID, extra ...any) map[string]any {99 b := map[string]any{"type": typ, "target": map[string]any{"kind": kind, "id": target}}100 for i := 0; i+1 < len(extra); i += 2 {101 b[extra[i].(string)] = extra[i+1]102 }103 return b104}105106func (c *client) post(body any) resp {107 return c.do("POST", "/actions", body, "Idempotency-Key", uuid.NewString())108}109110func (c *client) create(body any) actionBody {111 c.h.t.Helper()112 r := c.post(body)113 if r.Status != 201 {114 c.h.t.Fatalf("create action: %d %s", r.Status, r.Raw)115 }116 var a actionBody117 r.json(c.h.t, &a)118 return a119}120121func TestActionCatalogEndpoint(t *testing.T) {122 e := newOpsEnv(t)123 var out struct {124 Items []struct {125 Type, Criticality, Permission string126 Available bool127 RequiresStepUp bool `json:"requires_step_up"`128 RequiresApproval bool `json:"requires_approval"`129 RequiresConfirmation bool `json:"requires_confirmation"`130 RequiresJustification bool `json:"requires_justification"`131 ParamsSchema map[string]any `json:"params_schema"`132 LockScope string `json:"lock_scope"`133 }134 }135 e.viewer.must("GET", "/action-types", nil, 200).json(t, &out)136 by := map[string]int{}137 for i, it := range out.Items {138 by[it.Type] = i139 if it.Permission != "actions.run."+it.Criticality {140 t.Errorf("%s: permission %s", it.Type, it.Permission)141 }142 if it.RequiresStepUp != (it.Criticality == "high" || it.Criticality == "critical") || it.RequiresApproval != (it.Criticality == "critical") {143 t.Errorf("%s: flags inconsistent with criticality", it.Type)144 }145 if it.ParamsSchema["additionalProperties"] != false {146 t.Errorf("%s: schema must refuse unknown fields", it.Type)147 }148 }149 want := map[string]string{"inventory.refresh": "low", "diagnostic.run": "low", "service.health_check": "low", "service.logs.tail": "low",150 "service.restart": "medium", "service.start": "medium", "service.stop": "high", "agent.restart": "medium", "cache.purge": "medium",151 "node.maintenance.set": "medium", "job.run": "medium", "job.cancel": "medium", "model.server.start": "high", "model.server.stop": "high",152 "deploy.release": "high", "system.update.schedule": "high", "system.reboot": "critical", "system.shutdown": "critical"}153 if len(out.Items) != len(want) {154 t.Fatalf("%d types, want %d", len(out.Items), len(want))155 }156 for typ, crit := range want {157 i, ok := by[typ]158 if !ok || out.Items[i].Criticality != crit {159 t.Errorf("%s: missing or wrong criticality", typ)160 }161 }162 for _, typ := range []string{"deploy.release", "system.update.schedule"} {163 if out.Items[by[typ]].Available {164 t.Errorf("%s must not be available", typ)165 }166 }167 if rb := out.Items[by["system.reboot"]]; !rb.RequiresConfirmation || !rb.RequiresJustification || !rb.Available {168 t.Errorf("system.reboot flags: %+v", rb)169 }170}171172func TestCreateActionRefusals(t *testing.T) {173 e := newOpsEnv(t)174 ctx := context.Background()175 offline := testdb.Node(t, e.Pool, "T-OFFLINE")176 testdb.Agent(t, e.Pool, offline, time.Now().Add(-10*time.Minute), testCaps...)177 noAgent := testdb.Node(t, e.Pool, "T-NOAGENT")178 bare := testdb.Node(t, e.Pool, "T-BARE")179 testdb.Agent(t, e.Pool, bare, time.Now(), "inventory.refresh")180 maint := testdb.Node(t, e.Pool, "T-MAINT")181 testdb.Agent(t, e.Pool, maint, time.Now(), testCaps...)182 e.Pool.Exec(ctx, `update nodes set maintenance=true where id=$1`, maint)183 reboot := func(extra ...any) map[string]any {184 return act("system.reboot", "node", e.node, extra...)185 }186187 cases := []struct {188 name string189 who *client190 body map[string]any191 status int192 code string193 audit string // audit action expected with outcome denied/failure194 prep func()195 }{196 {"viewer cannot run a low action", e.viewer, act("inventory.refresh", "node", e.node), 403, "forbidden", "authz.denied", nil},197 {"operator cannot run a high action", e.oper, act("service.stop", "service", e.svc), 403, "forbidden", "authz.denied", nil},198 {"admin cannot run a critical action", e.admin, reboot("confirm", "T-ALPHA", "justification", "because"), 403, "forbidden", "authz.denied", nil},199 {"high action without step-up", e.admin, act("service.stop", "service", e.svc), 403, "step_up_required", "action.create",200 func() { e.expireStepUp("alice") }},201 {"critical action without step-up", e.root, reboot("confirm", "T-ALPHA", "justification", "because"), 403, "step_up_required", "action.create",202 func() { e.expireStepUp("root") }},203 {"missing confirmation", e.root, reboot("justification", "because"), 422, "confirmation_mismatch", "action.create", func() { e.root.stepUp() }},204 {"wrong confirmation", e.root, reboot("confirm", "t-alpha", "justification", "because"), 422, "confirmation_mismatch", "action.create", nil},205 {"confirmation of another node", e.root, reboot("confirm", "T-BETA", "justification", "because"), 422, "confirmation_mismatch", "action.create", nil},206 {"missing justification", e.root, reboot("confirm", "T-ALPHA"), 422, "justification_required", "action.create", nil},207 {"blank justification", e.root, reboot("confirm", "T-ALPHA", "justification", " "), 422, "justification_required", "action.create", nil},208 {"unknown type", e.oper, act("shell.exec", "node", e.node, "params", map[string]any{"cmd": "id"}), 400, "validation_failed", "action.create", nil},209 {"unknown parameter", e.oper, act("inventory.refresh", "node", e.node, "params", map[string]any{"path": "/etc"}), 422, "precheck_failed", "action.create", nil},210 {"parameter outside its enum", e.oper, act("diagnostic.run", "node", e.node, "params", map[string]any{"profile": "everything"}), 422, "precheck_failed", "action.create", nil},211 {"missing required parameter", e.oper, act("diagnostic.run", "node", e.node), 422, "precheck_failed", "action.create", nil},212 {"parameter looking like a path", e.oper, act("cache.purge", "node", e.node, "params", map[string]any{"cache": "../../etc"}), 422, "precheck_failed", "action.create", nil},213 {"parameter of the wrong type", e.oper, act("service.logs.tail", "service", e.svc, "params", map[string]any{"lines": "all"}), 422, "precheck_failed", "action.create", nil},214 {"parameter out of range", e.oper, act("service.logs.tail", "service", e.svc, "params", map[string]any{"lines": 100000}), 422, "precheck_failed", "action.create", nil},215 {"unavailable type", e.root, act("deploy.release", "service", e.svc), 422, "action_unavailable", "action.create", nil},216 {"internal type", e.oper, act("job.run", "job", uuid.New(), "params", map[string]any{"template": "sleep"}), 422, "unprocessable", "action.create", nil},217 {"wrong kind of target", e.oper, act("inventory.refresh", "service", e.svc), 422, "precheck_failed", "action.create", nil},218 {"unknown target", e.oper, act("inventory.refresh", "node", uuid.New()), 404, "not_found", "action.create", nil},219 {"agent offline", e.oper, act("inventory.refresh", "node", offline), 422, "agent_offline", "action.create", nil},220 {"no agent enrolled", e.oper, act("inventory.refresh", "node", noAgent), 422, "agent_offline", "action.create", nil},221 {"capability not declared by the agent", e.oper, act("cache.purge", "node", bare, "params", map[string]any{"cache": "pip"}), 422, "capability_missing", "action.create", nil},222 {"unmanaged service", e.oper, act("service.restart", "service", e.rawSvc), 422, "precheck_failed", "action.create", nil},223 {"node in maintenance", e.oper, act("inventory.refresh", "node", maint), 422, "node_in_maintenance", "action.create", nil},224 {"orchestrator frozen", e.oper, act("inventory.refresh", "node", e.node), 503, "orchestrator_frozen", "action.create",225 func() { e.Pool.Exec(ctx, `insert into settings(key,value) values ('orchestrator.frozen','true')`) }},226 }227 for _, c := range cases {228 if c.prep != nil {229 c.prep()230 }231 before := e.count(`select count(*) from audit_log where action=$1 and outcome in ('denied','failure')`, c.audit)232 r := c.who.post(c.body)233 if r.Status != c.status || r.code() != c.code {234 t.Errorf("%s: got %d %s (%s), want %d %s", c.name, r.Status, r.code(), r.Raw, c.status, c.code)235 }236 if n := e.count(`select count(*) from actions`); n != 0 {237 t.Fatalf("%s: left %d action row(s)", c.name, n)238 }239 if after := e.count(`select count(*) from audit_log where action=$1 and outcome in ('denied','failure')`, c.audit); after != before+1 {240 t.Errorf("%s: refusal not audited (%s: %d -> %d)", c.name, c.audit, before, after)241 }242 }243 if n := e.count(`select count(*) from action_locks`) + e.count(`select count(*) from agent_commands`); n != 0 {244 t.Fatalf("refusals left locks or commands: %d", n)245 }246 e.Pool.Exec(ctx, `delete from settings where key='orchestrator.frozen'`)247248 // Missing or malformed Idempotency-Key.249 if r := e.oper.do("POST", "/actions", act("inventory.refresh", "node", e.node)); r.Status != 400 {250 t.Errorf("missing Idempotency-Key: %d %s", r.Status, r.Raw)251 }252 if r := e.oper.do("POST", "/actions", act("inventory.refresh", "node", e.node), "Idempotency-Key", "short"); r.Status != 400 {253 t.Errorf("short Idempotency-Key: %d %s", r.Status, r.Raw)254 }255 // The documented override lets an action through on a node in maintenance, and is audited.256 a := e.oper.create(act("inventory.refresh", "node", maint, "override_maintenance", true))257 if a.State != "queued" || e.count(`select count(*) from audit_log where action='action.create' and outcome='success' and (params->>'override_maintenance')::boolean`) != 1 {258 t.Fatalf("override: %+v", a)259 }260 // Maintenance-related actions need no override.261 if a := e.oper.create(act("node.maintenance.set", "node", e.node, "params", map[string]any{"enabled": true})); a.State != "queued" {262 t.Fatalf("%+v", a)263 }264}265266func TestScopedDenyGrant(t *testing.T) {267 e := newOpsEnv(t)268 // Oscar may not run medium actions on T-ALPHA; everything else is unchanged. Written before269 // his first request, so the policy cache cannot hold an older view.270 if _, err := e.Pool.Exec(context.Background(), `insert into policy_grants(id, subject_kind, subject, permission, resource_kind, resource_id, effect)271 select $1, 'user', id::text, 'actions.run.medium', 'node', $2, 'deny' from users where handle='oscar'`, uuid.New(), e.node.String()); err != nil {272 t.Fatal(err)273 }274 time.Sleep(3100 * time.Millisecond) // policy cache lifetime275 if r := e.oper.post(act("service.restart", "service", e.svc)); r.Status != 403 || r.code() != "forbidden" {276 t.Fatalf("scoped deny not applied: %d %s", r.Status, r.Raw)277 }278 if r := e.oper.post(act("cache.purge", "node", e.node, "params", map[string]any{"cache": "pip"})); r.Status != 403 {279 t.Fatalf("scoped deny not applied to the node itself: %d", r.Status)280 }281 if e.count(`select count(*) from actions`) != 0 || e.count(`select count(*) from audit_log where action='authz.denied' and target_kind='node' and target_id=$1`, e.node.String()) != 2 {282 t.Fatal("denied requests left rows or were not audited against the node")283 }284 e.oper.create(act("cache.purge", "node", e.node2, "params", map[string]any{"cache": "pip"})) // other node: allowed285 e.oper.create(act("inventory.refresh", "node", e.node)) // other criticality: allowed286}287288func TestIdempotency(t *testing.T) {289 e := newOpsEnv(t)290 body := act("service.restart", "service", e.svc)291 key := uuid.NewString()292 r1 := e.admin.do("POST", "/actions", body, "Idempotency-Key", key)293 r2 := e.admin.do("POST", "/actions", body, "Idempotency-Key", key)294 var a1, a2 actionBody295 r1.json(t, &a1)296 r2.json(t, &a2)297 if r1.Status != 201 || r2.Status != 200 || a1.ID == "" || a1.ID != a2.ID {298 t.Fatalf("replay: %d %d %s %s", r1.Status, r2.Status, a1.ID, a2.ID)299 }300 if r := e.admin.do("POST", "/actions", act("service.restart", "service", e.rawSvc), "Idempotency-Key", key); r.Status != 409 || r.code() != "idempotency_conflict" {301 t.Fatalf("conflicting body: %d %s", r.Status, r.Raw)302 }303 if r := e.admin.do("POST", "/actions", act("service.restart", "service", e.svc, "justification", "x"), "Idempotency-Key", key); r.Status != 409 || r.code() != "idempotency_conflict" {304 t.Fatalf("conflicting justification: %d %s", r.Status, r.Raw)305 }306 // The same key used by another user is another request (target is locked by the first one).307 if r := e.oper.do("POST", "/actions", body, "Idempotency-Key", key); r.Status != 409 || r.code() != "target_locked" {308 t.Fatalf("other user: %d %s", r.Status, r.Raw)309 }310 if e.run(a1.ID) != "succeeded" {311 t.Fatal("not succeeded")312 }313 // A replay after completion still returns the original, finished action: nothing runs twice.314 r3 := e.admin.do("POST", "/actions", body, "Idempotency-Key", key)315 var a3 actionBody316 r3.json(t, &a3)317 if r3.Status != 200 || a3.ID != a1.ID || a3.State != "succeeded" {318 t.Fatalf("replay after completion: %d %+v", r3.Status, a3)319 }320321 // 30 concurrent identical requests: one action, one command, one execution.322 key = uuid.NewString()323 body = act("diagnostic.run", "node", e.node, "params", map[string]any{"profile": "basic"})324 var wg sync.WaitGroup325 var mu sync.Mutex326 codes := map[int]int{}327 idsSeen := map[string]bool{}328 for i := 0; i < 30; i++ {329 wg.Add(1)330 go func() {331 defer wg.Done()332 r := e.root.do("POST", "/actions", body, "Idempotency-Key", key)333 var a actionBody334 r.json(t, &a)335 mu.Lock()336 codes[r.Status]++337 idsSeen[a.ID] = true338 mu.Unlock()339 }()340 }341 wg.Wait()342 if codes[201] != 1 || codes[200] != 29 || len(idsSeen) != 1 {343 t.Fatalf("concurrent identical requests: %v, %d distinct ids", codes, len(idsSeen))344 }345 var id string346 for k := range idsSeen {347 id = k348 }349 if e.run(id) != "succeeded" {350 t.Fatal("not succeeded")351 }352 if n := e.count(`select count(*) from actions where type='diagnostic.run'`); n != 1 {353 t.Fatalf("%d actions", n)354 }355 if n := e.count(`select count(*) from agent_commands where action_id=$1`, id); n != 1 || len(e.agent.Executed()) != 2 {356 t.Fatalf("%d commands, %d executions in total", n, len(e.agent.Executed()))357 }358 if n := e.count(`select count(*) from audit_log where action='action.create' and outcome='success' and params->>'action_id'=$1`, id); n != 1 {359 t.Fatalf("%d audit entries for one creation", n)360 }361}362363func TestActionRateLimit(t *testing.T) {364 e := newOpsEnv(t)365 limited := 0366 for i := 0; i < 33; i++ {367 r := e.oper.post(act("service.logs.tail", "service", e.svc))368 if r.Status == 429 {369 limited++370 if r.Header.Get("Retry-After") == "" {371 t.Fatal("429 without Retry-After")372 }373 }374 }375 if limited != 3 || e.count(`select count(*) from actions`) != 30 {376 t.Fatalf("limited=%d actions=%d", limited, e.count(`select count(*) from actions`))377 }378}379380func TestTargetLockAndRelease(t *testing.T) {381 e := newOpsEnv(t)382 ctx := context.Background()383 body := func() map[string]any { return act("inventory.refresh", "node", e.node) }384 blocked := func(holder string) {385 t.Helper()386 r := e.admin.post(act("cache.purge", "node", e.node, "params", map[string]any{"cache": "pip"}))387 var er struct {388 Error struct {389 Details map[string]any390 }391 }392 r.json(t, &er)393 if r.Status != 409 || r.code() != "target_locked" || er.Error.Details["action_id"] != holder {394 t.Fatalf("expected target_locked by %s: %d %s", holder, r.Status, r.Raw)395 }396 }397 free := func(why string) {398 t.Helper()399 if n := e.count(`select count(*) from action_locks`); n != 0 {400 t.Fatalf("lock not released after %s", why)401 }402 }403 // success404 a := e.oper.create(body())405 blocked(a.ID)406 e.oper.create(act("inventory.refresh", "node", e.node2)) // another node is not locked407 e.oper.create(act("service.restart", "service", e.svc)) // a service lock is distinct from its node lock408 if e.run(a.ID) != "succeeded" {409 t.Fatal("not succeeded")410 }411 e.Pool.Exec(ctx, `delete from action_locks`) // the two side actions412 // cancel while queued413 a = e.oper.create(body())414 blocked(a.ID)415 var got actionBody416 e.oper.must("POST", "/actions/"+a.ID+"/cancel", nil, 200).json(t, &got)417 if got.State != "cancelled" {418 t.Fatalf("%+v", got)419 }420 free("cancel")421 if r := e.oper.do("POST", "/actions/"+a.ID+"/cancel", nil); r.Status != 409 {422 t.Fatalf("cancel of a finished action: %d", r.Status)423 }424 // agent refusal425 a = e.oper.create(body())426 e.agent.Handle = func(*protocol.Command) fakeagent.Behavior { return fakeagent.Behavior{Refuse: "not allowed here"} }427 if e.run(a.ID) != "failed" {428 t.Fatal("not failed")429 }430 free("refusal")431 e.agent.Handle = nil432 // never delivered433 a = e.oper.create(body())434 e.step()435 e.Pool.Exec(ctx, `update agent_commands set expires_at=now()-interval '1 second' where action_id=$1`, a.ID)436 e.step()437 e.oper.must("GET", "/actions/"+a.ID, nil, 200).json(t, &struct{ Action *actionBody }{&got})438 if got.State != "failed" || got.ErrorCode != "agent_unreachable" || got.OutcomeUnknown {439 t.Fatalf("%+v", got)440 }441 free("agent_unreachable")442 // delivered, no result443 a = e.oper.create(body())444 e.step()445 ds, _ := e.agent.Deliver(ctx, nil)446 e.agent.Ack(ctx, ds[0], true, "")447 e.step()448 blocked(a.ID)449 e.Pool.Exec(ctx, `update actions set deadline_at=now()-interval '1 second' where id=$1`, a.ID)450 e.step()451 e.oper.must("GET", "/actions/"+a.ID, nil, 200).json(t, &struct{ Action *actionBody }{&got})452 if got.State != "timed_out" || !got.OutcomeUnknown || got.Result["outcome_unknown"] != true {453 t.Fatalf("%+v", got)454 }455 free("timeout")456 // rejection and expiry of an approval457 e.root.stepUp()458 rb := e.root.create(act("system.reboot", "node", e.node, "confirm", "T-ALPHA", "justification", "kernel panic loop"))459 blocked(rb.ID)460 e.admin.must("POST", "/actions/"+rb.ID+"/reject", map[string]any{"reason": "not now"}, 200)461 free("reject")462 rb = e.root.create(act("system.reboot", "node", e.node, "confirm", "T-ALPHA", "justification", "kernel panic loop"))463 e.Pool.Exec(ctx, `update actions set created_at=now()-interval '31 minutes' where id=$1`, rb.ID)464 e.step()465 free("expiry")466 if e.count(`select count(*) from actions where id=$1 and state='expired'`, rb.ID) != 1 {467 t.Fatal("not expired")468 }469}470471func TestApprovalFlow(t *testing.T) {472 e := newOpsEnv(t)473 ctx := context.Background()474 e.root.stepUp()475 req := func() actionBody {476 return e.root.create(act("system.reboot", "node", e.node, "confirm", "T-ALPHA", "justification", "kernel panic loop after the disk filled up"))477 }478 a := req()479 if a.State != "pending_approval" || !a.Approval || a.NodeAlias != "T-ALPHA" || a.TargetLabel != "T-ALPHA" {480 t.Fatalf("%+v", a)481 }482 e.step()483 if e.count(`select count(*) from agent_commands`) != 0 {484 t.Fatal("a command was issued before approval")485 }486 path := "/actions/" + a.ID487 // The requester cannot approve or reject their own request.488 if r := e.root.do("POST", path+"/approve", map[string]any{}); r.Status != 403 || r.code() != "forbidden" {489 t.Fatalf("self approval: %d %s", r.Status, r.Raw)490 }491 if r := e.root.do("POST", path+"/reject", map[string]any{"reason": "changed my mind"}); r.Status != 403 {492 t.Fatalf("self rejection: %d", r.Status)493 }494 if e.count(`select count(*) from audit_log where action in ('action.approve','action.reject') and outcome='denied'`) != 2 {495 t.Fatal("self decisions not audited")496 }497 // Approving needs the permission…498 if r := e.oper.do("POST", path+"/approve", map[string]any{}); r.Status != 403 || r.code() != "forbidden" {499 t.Fatalf("operator approval: %d %s", r.Status, r.Raw)500 }501 // …and a recent passkey verification.502 e.expireStepUp("alice")503 if r := e.admin.do("POST", path+"/approve", map[string]any{}); r.Status != 403 || r.code() != "step_up_required" {504 t.Fatalf("approval without step-up: %d %s", r.Status, r.Raw)505 }506 e.admin.stepUp()507 // Not while frozen.508 e.Pool.Exec(ctx, `insert into settings(key,value) values ('orchestrator.frozen','true')`)509 if r := e.admin.do("POST", path+"/approve", map[string]any{}); r.Status != 503 || r.code() != "orchestrator_frozen" {510 t.Fatalf("approval while frozen: %d %s", r.Status, r.Raw)511 }512 e.Pool.Exec(ctx, `delete from settings where key='orchestrator.frozen'`)513 var got struct {514 State string `json:"state"`515 ApprovedBy string `json:"approved_by"`516 ApprovedAt string `json:"approved_at"`517 Name string `json:"approved_by_display_name"`518 }519 e.admin.must("POST", path+"/approve", map[string]any{"comment": "agreed by phone"}, 200).json(t, &got)520 if got.State != "queued" || got.ApprovedBy == "" || got.ApprovedAt == "" || got.Name != "User alice" {521 t.Fatalf("%+v", got)522 }523 if r := e.admin.do("POST", path+"/approve", map[string]any{}); r.Status != 409 {524 t.Fatalf("second approval: %d", r.Status)525 }526 if e.run(a.ID) != "succeeded" || len(e.agent.Executed()) != 1 {527 t.Fatal("approved action did not run exactly once")528 }529530 // Rejection.531 a = req()532 if r := e.admin.do("POST", "/actions/"+a.ID+"/reject", map[string]any{"reason": ""}); r.Status != 400 {533 t.Fatalf("reject without reason: %d", r.Status)534 }535 var rj struct {536 State string `json:"state"`537 Reason string `json:"rejection_reason"`538 }539 e.admin.must("POST", "/actions/"+a.ID+"/reject", map[string]any{"reason": "wait for the backup"}, 200).json(t, &rj)540 if rj.State != "rejected" || rj.Reason != "wait for the backup" {541 t.Fatalf("%+v", rj)542 }543 e.step()544 // Expiry: an approval that comes too late is refused and the action is closed.545 a = req()546 e.Pool.Exec(ctx, `update actions set created_at=now()-interval '31 minutes' where id=$1`, a.ID)547 if r := e.admin.do("POST", "/actions/"+a.ID+"/approve", map[string]any{}); r.Status != 409 {548 t.Fatalf("late approval: %d %s", r.Status, r.Raw)549 }550 if e.count(`select count(*) from actions where id=$1 and state='expired'`, a.ID) != 1 || e.count(`select count(*) from action_locks`) != 0 {551 t.Fatal("late approval did not expire the action")552 }553 if e.count(`select count(*) from agent_commands`) != 1 {554 t.Fatal("only the approved action may have produced a command")555 }556}557558func TestActionOutputListAndCancelPermissions(t *testing.T) {559 e := newOpsEnv(t)560 ctx := context.Background()561 a := e.oper.create(act("diagnostic.run", "node", e.node, "params", map[string]any{"profile": "network"}))562 e.agent.Handle = func(*protocol.Command) fakeagent.Behavior {563 return fakeagent.Behavior{Output: []protocol.CommandOutput{{Stream: "stdout", Data: "ping ok"}, {Stream: "stdout", Data: "dns ok"}, {Stream: "stderr", Data: "1 warning"}},564 Result: &protocol.CommandResult{Status: "succeeded", Result: map[string]any{"checks": 2}}}565 }566 if e.run(a.ID) != "succeeded" {567 t.Fatal("not succeeded")568 }569 var det struct {570 Action actionBody571 Events []struct{ Kind, Data string }572 }573 e.viewer.must("GET", "/actions/"+a.ID, nil, 200).json(t, &det)574 if det.Action.Result["checks"] != float64(2) || len(det.Events) < 3 {575 t.Fatalf("%+v", det)576 }577 for _, ev := range det.Events {578 if ev.Kind == "output" {579 t.Fatal("output leaked into the lifecycle events")580 }581 }582 var out struct {583 Items []struct {584 ID int64585 Stream, Data string586 }587 NextAfter int64 `json:"next_after"`588 Complete bool589 }590 e.oper.must("GET", "/actions/"+a.ID+"/output?limit=2", nil, 200).json(t, &out)591 if len(out.Items) != 2 || out.Complete || out.Items[0].Data != "ping ok" {592 t.Fatalf("%+v", out)593 }594 e.oper.must("GET", "/actions/"+a.ID+"/output?after="+itoa(out.NextAfter), nil, 200).json(t, &out)595 if len(out.Items) != 1 || !out.Complete || out.Items[0].Stream != "stderr" {596 t.Fatalf("%+v", out)597 }598 // Output needs logs.read: a viewer sees the action, not what the node printed.599 if r := e.viewer.do("GET", "/actions/"+a.ID+"/output", nil); r.Status != 403 {600 t.Fatalf("viewer read output: %d", r.Status)601 }602 // Filters and pagination.603 for i := 0; i < 4; i++ {604 b := e.oper.create(act("service.logs.tail", "service", e.svc))605 e.run(b.ID)606 }607 var list struct {608 Items []actionBody609 NextCursor string `json:"next_cursor"`610 }611 e.viewer.must("GET", "/actions?type=service.logs.tail&limit=3", nil, 200).json(t, &list)612 if len(list.Items) != 3 || list.NextCursor == "" {613 t.Fatalf("page 1: %d %q", len(list.Items), list.NextCursor)614 }615 first, cursor := list.Items[0].ID, list.NextCursor616 list.NextCursor = ""617 e.viewer.must("GET", "/actions?type=service.logs.tail&limit=3&cursor="+cursor, nil, 200).json(t, &list)618 if len(list.Items) != 1 || list.NextCursor != "" || list.Items[0].ID == first {619 t.Fatalf("page 2: %d", len(list.Items))620 }621 e.viewer.must("GET", "/actions?state=succeeded&node_id="+e.node.String()+"&target_kind=node&criticality=low", nil, 200).json(t, &list)622 if len(list.Items) != 1 || list.Items[0].ID != a.ID {623 t.Fatalf("filters: %d", len(list.Items))624 }625 if r := e.viewer.do("GET", "/actions?cursor=zzz", nil); r.Status != 400 {626 t.Fatalf("bad cursor: %d", r.Status)627 }628 if r := e.viewer.do("GET", "/actions/"+uuid.NewString(), nil); r.Status != 404 {629 t.Fatalf("unknown action: %d", r.Status)630 }631632 // Cancel: not by a bystander without the run permission; by the requester; cooperatively when running.633 e.agent.Handle = nil634 c := e.oper.create(act("inventory.refresh", "node", e.node))635 if r := e.viewer.do("POST", "/actions/"+c.ID+"/cancel", nil); r.Status != 403 {636 t.Fatalf("viewer cancel: %d", r.Status)637 }638 e.step()639 ds, _ := e.agent.Deliver(ctx, nil)640 e.agent.Ack(ctx, ds[0], true, "")641 e.step()642 var got struct {643 State string `json:"state"`644 Cancel string `json:"cancel_requested_at"`645 }646 e.admin.must("POST", "/actions/"+c.ID+"/cancel", nil, 200).json(t, &got) // admin holds the run permission647 if got.State != "running" || got.Cancel == "" {648 t.Fatalf("%+v", got)649 }650 if asked, _ := e.agent.CancelRequested(ctx, ds[0]); !asked {651 t.Fatal("cancellation not passed to the agent")652 }653 e.agent.Result(ctx, ds[0], protocol.CommandResult{Status: "cancelled"})654 e.step()655 if e.count(`select count(*) from actions where id=$1 and state='cancelled'`, c.ID) != 1 {656 t.Fatal("not cancelled")657 }658 if e.count(`select count(*) from audit_log where action='action.cancel' and outcome='success'`) != 1 {659 t.Fatal("cancel not audited")660 }661}662663func TestMaintenanceSetThroughAPI(t *testing.T) {664 e := newOpsEnv(t)665 a := e.oper.create(act("node.maintenance.set", "node", e.node, "params", map[string]any{"enabled": true, "reason": "RAM upgrade"}))666 if e.run(a.ID) != "succeeded" {667 t.Fatal("not succeeded")668 }669 if e.count(`select count(*) from nodes where id=$1 and maintenance and maintenance_reason='RAM upgrade'`, e.node) != 1 || e.count(`select count(*) from agent_commands`) != 0 {670 t.Fatal("maintenance not set server-side")671 }672 if r := e.oper.post(act("inventory.refresh", "node", e.node)); r.code() != "node_in_maintenance" {673 t.Fatalf("%d %s", r.Status, r.Raw)674 }675}676677func itoa(n int64) string {678 b := []byte{}679 if n == 0 {680 return "0"681 }682 for n > 0 {683 b = append([]byte{byte('0' + n%10)}, b...)684 n /= 10685 }686 return string(b)687}688