Repository Public
1package api23import (4 "context"5 "errors"6 "strings"78 "github.com/google/uuid"9 "github.com/jackc/pgx/v5"1011 "clustr.spboucher.ai/mcc/internal/actions"12 "clustr.spboucher.ai/mcc/internal/audit"13 "clustr.spboucher.ai/mcc/internal/authz"14 "clustr.spboucher.ai/mcc/internal/httpx"15)1617// Actions API. The orchestrator executes; this file only validates, records18// and reports. Every refusal leaves no action row and is audited.1920// 30 action requests per minute per user (contract).21var actionLimiter = httpx.NewLimiter(30, 30)2223const actionSelect = `select a.id, a.type, a.criticality, a.target_kind, a.target_id, a.target_label, a.node_id, n.alias::text, a.params, a.state::text,24 a.requested_by, ru.display_name, a.requested_via, a.justification, a.approval_required, a.approved_by, au.display_name, a.approved_at,25 a.rejection_reason, a.created_at, a.queued_at, a.started_at, a.finished_at, a.deadline_at, a.cancel_requested_at, a.timeout_s, a.result,26 a.error_code, a.error_message, a.job_id, a.correlation_id27 from actions a left join nodes n on n.id=a.node_id left join users ru on ru.id=a.requested_by left join users au on au.id=a.approved_by`2829func scanAction(r pgx.Row) (Action, error) {30 var a Action31 var kind, target, state string32 var result map[string]any33 err := r.Scan(&a.Id, &a.Type, &a.Criticality, &kind, &target, &a.TargetLabel, &a.NodeId, &a.NodeAlias, &a.Params, &state,34 &a.RequestedBy, &a.RequestedByDisplayName, &a.RequestedVia, &a.Justification, &a.ApprovalRequired, &a.ApprovedBy, &a.ApprovedByDisplayName, &a.ApprovedAt,35 &a.RejectionReason, &a.CreatedAt, &a.QueuedAt, &a.StartedAt, &a.FinishedAt, &a.DeadlineAt, &a.CancelRequestedAt, &a.TimeoutS, &result,36 &a.ErrorCode, &a.ErrorMessage, &a.JobId, &a.CorrelationId)37 if err != nil {38 return a, err39 }40 a.State = ActionState(state)41 a.Target.Kind = ActionTargetKind(kind)42 a.Target.Id, _ = uuid.Parse(target)43 if a.Params == nil {44 a.Params = map[string]any{}45 }46 if result != nil {47 a.Result = &result48 a.OutcomeUnknown, _ = result["outcome_unknown"].(bool)49 }50 if a.TargetLabel != nil && *a.TargetLabel == "" {51 a.TargetLabel = nil52 }53 return a, nil54}5556func (h *opsHandlers) loadAction(ctx context.Context, q actions.Querier, id uuid.UUID) (Action, error) {57 a, err := scanAction(q.QueryRow(ctx, actionSelect+` where a.id=$1`, id))58 return a, notFound(err)59}6061// actionResources lists the resources an action touches, most specific last.62func actionResources(nodeID *uuid.UUID, kind string, id uuid.UUID) []authz.Resource {63 var out []authz.Resource64 if nodeID != nil {65 out = append(out, authz.Resource{Kind: "node", ID: nodeID.String()})66 }67 if kind != "node" {68 out = append(out, authz.Resource{Kind: kind, ID: id.String()})69 }70 if len(out) == 0 {71 out = append(out, authz.Resource{})72 }73 return out74}7576func (h *opsHandlers) ListActionTypes(ctx context.Context, _ ListActionTypesRequestObject) (ListActionTypesResponseObject, error) {77 out := ActionTypeList{Items: []ActionType{}}78 for _, t := range actions.Catalog {79 kinds := make([]ActionTargetKind, len(t.TargetKinds))80 for i, k := range t.TargetKinds {81 kinds[i] = ActionTargetKind(k)82 }83 at := ActionType{Type: t.Name, Title: t.Title, Description: t.Description, Criticality: Criticality(t.Criticality), Permission: t.Permission(),84 TargetKinds: kinds, ParamsSchema: t.Params.JSONSchema(), RequiresStepUp: t.RequiresStepUp(), RequiresJustification: t.RequiresJustification(),85 RequiresConfirmation: t.RequiresConfirmation(), RequiresApproval: t.RequiresApproval(), TimeoutS: t.TimeoutS,86 LockScope: ActionTypeLockScope(t.LockScope), Available: t.Available && !t.Internal}87 if len(t.ExtraPermissions) > 0 {88 at.ExtraPermissions = &t.ExtraPermissions89 }90 switch {91 case !t.Available:92 at.UnavailableReason = nonEmpty(t.UnavailableReason)93 case t.Internal:94 at.UnavailableReason = ptr("Created by the platform only (submit a job instead).")95 }96 out.Items = append(out.Items, at)97 }98 return ListActionTypes200JSONResponse{Body: out}, nil99}100101func (h *opsHandlers) CreateAction(ctx context.Context, req CreateActionRequestObject) (CreateActionResponseObject, error) {102 if req.Body == nil {103 return nil, httpx.Validation("Body required.")104 }105 b := req.Body106 sess := principalFrom(ctx).Session107 now := nowUTC()108 params := deref(b.Params)109 if params == nil {110 params = map[string]any{}111 }112 auditParams := map[string]any{"type": b.Type, "override_maintenance": deref(b.OverrideMaintenance)}113 refuse := func(err error) (CreateActionResponseObject, error) {114 return nil, h.refused(ctx, "action.create", string(b.Target.Kind), b.Target.Id.String(), auditParams, err)115 }116 if ok, wait := actionLimiter.Allow("act:" + sess.User.ID.String()); !ok {117 e := *httpx.ErrRateLimited118 e.RetryAfter = wait119 return refuse(&e)120 }121 key := strings.TrimSpace(req.Params.IdempotencyKey)122 if len(key) < 8 || len(key) > 128 {123 return refuse(httpx.Validation("Idempotency-Key must be 8 to 128 characters."))124 }125 t, ok := actions.Lookup(b.Type)126 if !ok {127 return refuse(httpx.Validation("Unknown action type.").With("fields", map[string]any{"type": "unknown"}))128 }129 if t.Internal {130 return refuse(actions.Refuse("unprocessable", "This action type is created by the platform only; submit a job instead."))131 }132 if !t.Available {133 return refuse(actions.Refuse("action_unavailable", "This action type is not enabled in this version."))134 }135 if err := t.Params.Validate(params); err != nil {136 var pe *actions.ParamError137 errors.As(err, &pe)138 return refuse(actions.Refuse("precheck_failed", "Invalid parameters: "+err.Error()).With("check", "params").With("field", pe.Field))139 }140 if len(deref(b.Justification)) > 2000 || len(deref(b.Confirm)) > 128 {141 return refuse(httpx.Validation("justification or confirm is too long."))142 }143144 // Authorization for the criticality of the type, on the very node (and target) concerned.145 perms := []authz.Permission{authz.Permission(t.Permission())}146 for _, p := range t.ExtraPermissions {147 perms = append(perms, authz.Permission(p))148 }149 tg, terr := actions.ResolveTarget(ctx, h.Pool, string(b.Target.Kind), b.Target.Id)150 if terr != nil {151 for _, p := range perms { // an unauthorized caller learns nothing about targets152 if err := h.authorize(ctx, p, authz.Resource{}); err != nil {153 return nil, err154 }155 }156 return refuse(terr)157 }158 for _, p := range perms {159 for _, res := range actionResources(tg.NodeID, tg.Kind, tg.ID) {160 if err := h.authorize(ctx, p, res); err != nil {161 return nil, err162 }163 }164 }165 if t.RequiresStepUp() {166 if err := h.requireStepUp(ctx); err != nil {167 return refuse(err)168 }169 }170171 hash := actions.HashRequest(map[string]any{"type": b.Type, "kind": b.Target.Kind, "id": b.Target.Id, "params": params,172 "confirm": deref(b.Confirm), "justification": deref(b.Justification), "override": deref(b.OverrideMaintenance)})173 if id, found, err := actions.Existing(ctx, h.Pool, sess.User.ID, key, hash); err != nil {174 return refuse(err)175 } else if found {176 a, err := h.loadAction(ctx, h.Pool, id)177 if err != nil {178 return nil, err179 }180 return CreateAction200JSONResponse{Body: a}, nil181 }182183 if frozen, err := actions.Frozen(ctx, h.Pool); err != nil {184 return nil, err185 } else if frozen {186 return refuse(actions.ErrFrozen)187 }188 if t.RequiresConfirmation() && (b.Confirm == nil || *b.Confirm != tg.Confirm) {189 return refuse(actions.Refuse("confirmation_mismatch", "The confirmation does not match the target."))190 }191 justification := strings.TrimSpace(deref(b.Justification))192 if t.RequiresJustification() && justification == "" {193 return refuse(actions.Refuse("justification_required", "A justification is required for this action."))194 }195 if _, err := actions.Precheck(ctx, h.Pool, t, tg, params, actions.PrecheckOpts{OverrideMaintenance: deref(b.OverrideMaintenance),196 Now: now, OfflineAfter: h.Cfg.OfflineAfter}); err != nil {197 return refuse(err)198 }199200 var id uuid.UUID201 var created bool202 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) (err error) {203 id, created, err = actions.Insert(ctx, tx, actions.Request{Type: t, Target: tg, Params: params, Justification: justification,204 OverrideMaintenance: deref(b.OverrideMaintenance), RequestedBy: &sess.User.ID, Via: "api", IdempotencyKey: key, RequestHash: hash,205 CorrelationID: httpx.Correlation(ctx), Now: now})206 if err != nil || !created {207 return err208 }209 auditParams["criticality"], auditParams["target_label"], auditParams["action_id"] = t.Criticality, tg.Label, id.String()210 if tg.NodeID != nil {211 auditParams["node_id"] = tg.NodeID.String()212 }213 if justification != "" {214 auditParams["justification"] = justification215 }216 return h.audit(ctx, tx, "action.create", tg.Kind, tg.ID.String(), audit.Success, "", auditParams)217 })218 if err != nil {219 return refuse(err)220 }221 a, err := h.loadAction(ctx, h.Pool, id)222 if err != nil {223 return nil, err224 }225 if !created {226 return CreateAction200JSONResponse{Body: a}, nil227 }228 return CreateAction201JSONResponse{Body: a}, nil229}230231func (h *opsHandlers) ListActions(ctx context.Context, req ListActionsRequestObject) (ListActionsResponseObject, error) {232 p := req.Params233 var w where234 if p.State != nil {235 w.add("a.state = ?::action_state", string(*p.State))236 }237 if p.Type != nil {238 w.add("a.type = ?", *p.Type)239 }240 if p.Criticality != nil {241 w.add("a.criticality = ?", string(*p.Criticality))242 }243 if p.NodeId != nil {244 w.add("a.node_id = ?", *p.NodeId)245 }246 if p.TargetKind != nil {247 w.add("a.target_kind = ?", string(*p.TargetKind))248 }249 if p.TargetId != nil {250 w.add("a.target_id = ?", p.TargetId.String())251 }252 if p.RequestedBy != nil {253 w.add("a.requested_by = ?", *p.RequestedBy)254 }255 if p.From != nil {256 w.add("a.created_at >= ?", *p.From)257 }258 if p.To != nil {259 w.add("a.created_at < ?", *p.To)260 }261 cur, err := uuidCursor(p.Cursor)262 if err != nil {263 return nil, err264 }265 if cur != nil {266 w.add("a.id < ?", *cur)267 }268 limit := pageLimit(p.Limit)269 rows, err := h.Pool.Query(ctx, actionSelect+w.sql()+" order by a.id desc limit "+w.arg(limit+1), w.args...)270 if err != nil {271 return nil, err272 }273 items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Action, error) { return scanAction(r) })274 if err != nil {275 return nil, err276 }277 out := ActionList{Items: items}278 if len(items) > limit {279 out.Items = items[:limit]280 out.NextCursor = ptr(items[limit-1].Id.String())281 }282 if out.Items == nil {283 out.Items = []Action{}284 }285 return ListActions200JSONResponse{Body: out}, nil286}287288func (h *opsHandlers) actionEvents(ctx context.Context, id uuid.UUID, output bool, after int64, limit int) ([]ActionEvent, error) {289 cmp := "<>"290 if output {291 cmp = "="292 }293 rows, err := h.Pool.Query(ctx, `select id, ts, kind, stream, data from action_events where action_id=$1 and kind `+cmp+` 'output' and id > $2 order by id limit $3`, id, after, limit)294 if err != nil {295 return nil, err296 }297 evs, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (ActionEvent, error) {298 var e ActionEvent299 var stream *string300 err := r.Scan(&e.Id, &e.Ts, &e.Kind, &stream, &e.Data)301 if stream != nil && (*stream == "stdout" || *stream == "stderr") {302 e.Stream = ptr(ActionEventStream(*stream))303 }304 return e, err305 })306 if evs == nil {307 evs = []ActionEvent{}308 }309 return evs, err310}311312func (h *opsHandlers) GetAction(ctx context.Context, req GetActionRequestObject) (GetActionResponseObject, error) {313 a, err := h.loadAction(ctx, h.Pool, req.Id)314 if err != nil {315 return nil, err316 }317 evs, err := h.actionEvents(ctx, req.Id, false, 0, 500)318 if err != nil {319 return nil, err320 }321 return GetAction200JSONResponse{Body: ActionDetail{Action: a, Events: evs}}, nil322}323324// GetActionOutput serves what the node printed. Output can contain operational325// detail, so on top of actions.read it requires logs.read on the node.326func (h *opsHandlers) GetActionOutput(ctx context.Context, req GetActionOutputRequestObject) (GetActionOutputResponseObject, error) {327 a, err := h.loadAction(ctx, h.Pool, req.Id)328 if err != nil {329 return nil, err330 }331 for _, res := range actionResources(a.NodeId, string(a.Target.Kind), a.Target.Id) {332 if err := h.authorize(ctx, authz.LogsRead, res); err != nil {333 return nil, err334 }335 }336 after, limit := deref(req.Params.After), 500337 if req.Params.Limit != nil {338 limit = min(max(*req.Params.Limit, 1), 1000)339 }340 if after < 0 {341 return nil, httpx.Validation("after must be >= 0.")342 }343 evs, err := h.actionEvents(ctx, req.Id, true, after, limit)344 if err != nil {345 return nil, err346 }347 out := ActionOutput{Items: evs, NextAfter: after, Complete: actions.IsTerminal(string(a.State)) && len(evs) < limit}348 if len(evs) > 0 {349 out.NextAfter = evs[len(evs)-1].Id350 }351 var truncated bool352 if err := h.Pool.QueryRow(ctx, `select output_truncated from agent_commands where action_id=$1`, req.Id).Scan(&truncated); err == nil && truncated {353 out.Truncated = &truncated354 }355 return GetActionOutput200JSONResponse{Body: out}, nil356}357358// CancelAction: the requester, or anyone allowed to run actions of that criticality on the node.359func (h *opsHandlers) CancelAction(ctx context.Context, req CancelActionRequestObject) (CancelActionResponseObject, error) {360 a, err := h.loadAction(ctx, h.Pool, req.Id)361 if err != nil {362 return nil, err363 }364 if a.RequestedBy == nil || *a.RequestedBy != h.userID(ctx) {365 for _, res := range actionResources(a.NodeId, string(a.Target.Kind), a.Target.Id) {366 if err := h.authorize(ctx, authz.Permission("actions.run."+string(a.Criticality)), res); err != nil {367 return nil, err368 }369 }370 }371 err = pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {372 state, err := actions.RequestCancel(ctx, tx, req.Id, nowUTC())373 if err != nil {374 return err375 }376 detail := "cancelled"377 if state != "cancelled" {378 detail = "cancellation requested from the agent"379 }380 return h.audit(ctx, tx, "action.cancel", string(a.Target.Kind), a.Target.Id.String(), audit.Success, detail, map[string]any{"action_id": req.Id.String(), "type": a.Type})381 })382 if err != nil {383 return nil, h.refused(ctx, "action.cancel", string(a.Target.Kind), a.Target.Id.String(), map[string]any{"action_id": req.Id.String()}, err)384 }385 a, err = h.loadAction(ctx, h.Pool, req.Id)386 if err != nil {387 return nil, err388 }389 return CancelAction200JSONResponse{Body: a}, nil390}391392// decide implements approve and reject: a second person, never the requester.393func (h *opsHandlers) decide(ctx context.Context, id uuid.UUID, approve bool, text string) (Action, error) {394 me := h.userID(ctx)395 verb := "action.reject"396 if approve {397 verb = "action.approve"398 }399 var a Action400 var denied error401 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {402 var err error403 if a, err = scanAction(tx.QueryRow(ctx, actionSelect+` where a.id=$1 for update of a`, id)); err != nil {404 return notFound(err)405 }406 if a.State != ActionStatePendingApproval {407 return httpx.Conflict("conflict", "The action is not pending approval.")408 }409 if a.RequestedBy != nil && *a.RequestedBy == me {410 return httpx.ErrForbidden.With("reason", "The requester cannot approve or reject their own action.")411 }412 for _, res := range actionResources(a.NodeId, string(a.Target.Kind), a.Target.Id) {413 if err := h.authorize(ctx, authz.ActionsApprove, res); err != nil {414 return err415 }416 }417 now := nowUTC()418 ap := map[string]any{"action_id": id.String(), "type": a.Type}419 if now.Sub(a.CreatedAt) >= actions.ApprovalTTL {420 // Too late: close it here rather than let a stale request through.421 if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "expired", ErrorCode: "approval_expired", ErrorMessage: "No approval was given within 30 minutes.", At: now}); err != nil {422 return err423 }424 denied = httpx.Conflict("conflict", "The approval period of this action has expired.")425 return nil426 }427 if !approve {428 if _, err := tx.Exec(ctx, `update actions set rejection_reason=$2 where id=$1`, id, text); err != nil {429 return err430 }431 if err := actions.Event(ctx, tx, id, "approval", "", "rejected: "+text, now); err != nil {432 return err433 }434 if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "rejected", At: now}); err != nil {435 return err436 }437 ap["reason"] = text438 return h.audit(ctx, tx, verb, string(a.Target.Kind), a.Target.Id.String(), audit.Success, "", ap)439 }440 if frozen, err := actions.Frozen(ctx, tx); err != nil {441 return err442 } else if frozen {443 return actions.ErrFrozen444 }445 if _, err := tx.Exec(ctx, `update actions set approved_by=$2, approved_at=$3 where id=$1`, id, me, now); err != nil {446 return err447 }448 if err := actions.Event(ctx, tx, id, "approval", "", strings.TrimSpace("approved "+text), now); err != nil {449 return err450 }451 if _, err := actions.Transition(ctx, tx, id, "pending_approval", "queued", now); err != nil {452 return err453 }454 return h.audit(ctx, tx, verb, string(a.Target.Kind), a.Target.Id.String(), audit.Success, "", ap)455 })456 if err == nil {457 err = denied458 }459 if err != nil {460 var he *httpx.Error461 if errors.As(err, &he) && he.Code == "forbidden" && he.Details["reason"] != nil {462 h.audit(ctx, h.Pool, verb, string(a.Target.Kind), a.Target.Id.String(), audit.Denied, "requester cannot decide on own action", map[string]any{"action_id": id.String()})463 return a, err464 }465 return a, h.refused(ctx, verb, "action", id.String(), map[string]any{"action_id": id.String()}, err)466 }467 return h.loadAction(ctx, h.Pool, id)468}469470func (h *opsHandlers) ApproveAction(ctx context.Context, req ApproveActionRequestObject) (ApproveActionResponseObject, error) {471 comment := ""472 if req.Body != nil {473 comment = strings.TrimSpace(deref(req.Body.Comment))474 }475 if len(comment) > 2000 {476 return nil, httpx.Validation("comment is too long.")477 }478 a, err := h.decide(ctx, req.Id, true, comment)479 if err != nil {480 return nil, err481 }482 return ApproveAction200JSONResponse{Body: a}, nil483}484485func (h *opsHandlers) RejectAction(ctx context.Context, req RejectActionRequestObject) (RejectActionResponseObject, error) {486 if req.Body == nil || strings.TrimSpace(req.Body.Reason) == "" || len(req.Body.Reason) > 2000 {487 return nil, httpx.Validation("reason is required (at most 2000 characters).")488 }489 a, err := h.decide(ctx, req.Id, false, strings.TrimSpace(req.Body.Reason))490 if err != nil {491 return nil, err492 }493 return RejectAction200JSONResponse{Body: a}, nil494}495