SPB Git

Repository Public

ops_actions.go 17.8 KB 495 lines go

1package api23import (4	"context"5	"errors"6	"strings"78	"github.com/google/uuid"9	"github.com/jackc/pgx/v5"1011	"clustr.spboucher.ai/mcc/internal/actions"12	"clustr.spboucher.ai/mcc/internal/audit"13	"clustr.spboucher.ai/mcc/internal/authz"14	"clustr.spboucher.ai/mcc/internal/httpx"15)1617// Actions API. The orchestrator executes; this file only validates, records18// and reports. Every refusal leaves no action row and is audited.1920// 30 action requests per minute per user (contract).21var actionLimiter = httpx.NewLimiter(30, 30)2223const actionSelect = `select a.id, a.type, a.criticality, a.target_kind, a.target_id, a.target_label, a.node_id, n.alias::text, a.params, a.state::text,24	a.requested_by, ru.display_name, a.requested_via, a.justification, a.approval_required, a.approved_by, au.display_name, a.approved_at,25	a.rejection_reason, a.created_at, a.queued_at, a.started_at, a.finished_at, a.deadline_at, a.cancel_requested_at, a.timeout_s, a.result,26	a.error_code, a.error_message, a.job_id, a.correlation_id27	from actions a left join nodes n on n.id=a.node_id left join users ru on ru.id=a.requested_by left join users au on au.id=a.approved_by`2829func scanAction(r pgx.Row) (Action, error) {30	var a Action31	var kind, target, state string32	var result map[string]any33	err := r.Scan(&a.Id, &a.Type, &a.Criticality, &kind, &target, &a.TargetLabel, &a.NodeId, &a.NodeAlias, &a.Params, &state,34		&a.RequestedBy, &a.RequestedByDisplayName, &a.RequestedVia, &a.Justification, &a.ApprovalRequired, &a.ApprovedBy, &a.ApprovedByDisplayName, &a.ApprovedAt,35		&a.RejectionReason, &a.CreatedAt, &a.QueuedAt, &a.StartedAt, &a.FinishedAt, &a.DeadlineAt, &a.CancelRequestedAt, &a.TimeoutS, &result,36		&a.ErrorCode, &a.ErrorMessage, &a.JobId, &a.CorrelationId)37	if err != nil {38		return a, err39	}40	a.State = ActionState(state)41	a.Target.Kind = ActionTargetKind(kind)42	a.Target.Id, _ = uuid.Parse(target)43	if a.Params == nil {44		a.Params = map[string]any{}45	}46	if result != nil {47		a.Result = &result48		a.OutcomeUnknown, _ = result["outcome_unknown"].(bool)49	}50	if a.TargetLabel != nil && *a.TargetLabel == "" {51		a.TargetLabel = nil52	}53	return a, nil54}5556func (h *opsHandlers) loadAction(ctx context.Context, q actions.Querier, id uuid.UUID) (Action, error) {57	a, err := scanAction(q.QueryRow(ctx, actionSelect+` where a.id=$1`, id))58	return a, notFound(err)59}6061// actionResources lists the resources an action touches, most specific last.62func actionResources(nodeID *uuid.UUID, kind string, id uuid.UUID) []authz.Resource {63	var out []authz.Resource64	if nodeID != nil {65		out = append(out, authz.Resource{Kind: "node", ID: nodeID.String()})66	}67	if kind != "node" {68		out = append(out, authz.Resource{Kind: kind, ID: id.String()})69	}70	if len(out) == 0 {71		out = append(out, authz.Resource{})72	}73	return out74}7576func (h *opsHandlers) ListActionTypes(ctx context.Context, _ ListActionTypesRequestObject) (ListActionTypesResponseObject, error) {77	out := ActionTypeList{Items: []ActionType{}}78	for _, t := range actions.Catalog {79		kinds := make([]ActionTargetKind, len(t.TargetKinds))80		for i, k := range t.TargetKinds {81			kinds[i] = ActionTargetKind(k)82		}83		at := ActionType{Type: t.Name, Title: t.Title, Description: t.Description, Criticality: Criticality(t.Criticality), Permission: t.Permission(),84			TargetKinds: kinds, ParamsSchema: t.Params.JSONSchema(), RequiresStepUp: t.RequiresStepUp(), RequiresJustification: t.RequiresJustification(),85			RequiresConfirmation: t.RequiresConfirmation(), RequiresApproval: t.RequiresApproval(), TimeoutS: t.TimeoutS,86			LockScope: ActionTypeLockScope(t.LockScope), Available: t.Available && !t.Internal}87		if len(t.ExtraPermissions) > 0 {88			at.ExtraPermissions = &t.ExtraPermissions89		}90		switch {91		case !t.Available:92			at.UnavailableReason = nonEmpty(t.UnavailableReason)93		case t.Internal:94			at.UnavailableReason = ptr("Created by the platform only (submit a job instead).")95		}96		out.Items = append(out.Items, at)97	}98	return ListActionTypes200JSONResponse{Body: out}, nil99}100101func (h *opsHandlers) CreateAction(ctx context.Context, req CreateActionRequestObject) (CreateActionResponseObject, error) {102	if req.Body == nil {103		return nil, httpx.Validation("Body required.")104	}105	b := req.Body106	sess := principalFrom(ctx).Session107	now := nowUTC()108	params := deref(b.Params)109	if params == nil {110		params = map[string]any{}111	}112	auditParams := map[string]any{"type": b.Type, "override_maintenance": deref(b.OverrideMaintenance)}113	refuse := func(err error) (CreateActionResponseObject, error) {114		return nil, h.refused(ctx, "action.create", string(b.Target.Kind), b.Target.Id.String(), auditParams, err)115	}116	if ok, wait := actionLimiter.Allow("act:" + sess.User.ID.String()); !ok {117		e := *httpx.ErrRateLimited118		e.RetryAfter = wait119		return refuse(&e)120	}121	key := strings.TrimSpace(req.Params.IdempotencyKey)122	if len(key) < 8 || len(key) > 128 {123		return refuse(httpx.Validation("Idempotency-Key must be 8 to 128 characters."))124	}125	t, ok := actions.Lookup(b.Type)126	if !ok {127		return refuse(httpx.Validation("Unknown action type.").With("fields", map[string]any{"type": "unknown"}))128	}129	if t.Internal {130		return refuse(actions.Refuse("unprocessable", "This action type is created by the platform only; submit a job instead."))131	}132	if !t.Available {133		return refuse(actions.Refuse("action_unavailable", "This action type is not enabled in this version."))134	}135	if err := t.Params.Validate(params); err != nil {136		var pe *actions.ParamError137		errors.As(err, &pe)138		return refuse(actions.Refuse("precheck_failed", "Invalid parameters: "+err.Error()).With("check", "params").With("field", pe.Field))139	}140	if len(deref(b.Justification)) > 2000 || len(deref(b.Confirm)) > 128 {141		return refuse(httpx.Validation("justification or confirm is too long."))142	}143144	// Authorization for the criticality of the type, on the very node (and target) concerned.145	perms := []authz.Permission{authz.Permission(t.Permission())}146	for _, p := range t.ExtraPermissions {147		perms = append(perms, authz.Permission(p))148	}149	tg, terr := actions.ResolveTarget(ctx, h.Pool, string(b.Target.Kind), b.Target.Id)150	if terr != nil {151		for _, p := range perms { // an unauthorized caller learns nothing about targets152			if err := h.authorize(ctx, p, authz.Resource{}); err != nil {153				return nil, err154			}155		}156		return refuse(terr)157	}158	for _, p := range perms {159		for _, res := range actionResources(tg.NodeID, tg.Kind, tg.ID) {160			if err := h.authorize(ctx, p, res); err != nil {161				return nil, err162			}163		}164	}165	if t.RequiresStepUp() {166		if err := h.requireStepUp(ctx); err != nil {167			return refuse(err)168		}169	}170171	hash := actions.HashRequest(map[string]any{"type": b.Type, "kind": b.Target.Kind, "id": b.Target.Id, "params": params,172		"confirm": deref(b.Confirm), "justification": deref(b.Justification), "override": deref(b.OverrideMaintenance)})173	if id, found, err := actions.Existing(ctx, h.Pool, sess.User.ID, key, hash); err != nil {174		return refuse(err)175	} else if found {176		a, err := h.loadAction(ctx, h.Pool, id)177		if err != nil {178			return nil, err179		}180		return CreateAction200JSONResponse{Body: a}, nil181	}182183	if frozen, err := actions.Frozen(ctx, h.Pool); err != nil {184		return nil, err185	} else if frozen {186		return refuse(actions.ErrFrozen)187	}188	if t.RequiresConfirmation() && (b.Confirm == nil || *b.Confirm != tg.Confirm) {189		return refuse(actions.Refuse("confirmation_mismatch", "The confirmation does not match the target."))190	}191	justification := strings.TrimSpace(deref(b.Justification))192	if t.RequiresJustification() && justification == "" {193		return refuse(actions.Refuse("justification_required", "A justification is required for this action."))194	}195	if _, err := actions.Precheck(ctx, h.Pool, t, tg, params, actions.PrecheckOpts{OverrideMaintenance: deref(b.OverrideMaintenance),196		Now: now, OfflineAfter: h.Cfg.OfflineAfter}); err != nil {197		return refuse(err)198	}199200	var id uuid.UUID201	var created bool202	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) (err error) {203		id, created, err = actions.Insert(ctx, tx, actions.Request{Type: t, Target: tg, Params: params, Justification: justification,204			OverrideMaintenance: deref(b.OverrideMaintenance), RequestedBy: &sess.User.ID, Via: "api", IdempotencyKey: key, RequestHash: hash,205			CorrelationID: httpx.Correlation(ctx), Now: now})206		if err != nil || !created {207			return err208		}209		auditParams["criticality"], auditParams["target_label"], auditParams["action_id"] = t.Criticality, tg.Label, id.String()210		if tg.NodeID != nil {211			auditParams["node_id"] = tg.NodeID.String()212		}213		if justification != "" {214			auditParams["justification"] = justification215		}216		return h.audit(ctx, tx, "action.create", tg.Kind, tg.ID.String(), audit.Success, "", auditParams)217	})218	if err != nil {219		return refuse(err)220	}221	a, err := h.loadAction(ctx, h.Pool, id)222	if err != nil {223		return nil, err224	}225	if !created {226		return CreateAction200JSONResponse{Body: a}, nil227	}228	return CreateAction201JSONResponse{Body: a}, nil229}230231func (h *opsHandlers) ListActions(ctx context.Context, req ListActionsRequestObject) (ListActionsResponseObject, error) {232	p := req.Params233	var w where234	if p.State != nil {235		w.add("a.state = ?::action_state", string(*p.State))236	}237	if p.Type != nil {238		w.add("a.type = ?", *p.Type)239	}240	if p.Criticality != nil {241		w.add("a.criticality = ?", string(*p.Criticality))242	}243	if p.NodeId != nil {244		w.add("a.node_id = ?", *p.NodeId)245	}246	if p.TargetKind != nil {247		w.add("a.target_kind = ?", string(*p.TargetKind))248	}249	if p.TargetId != nil {250		w.add("a.target_id = ?", p.TargetId.String())251	}252	if p.RequestedBy != nil {253		w.add("a.requested_by = ?", *p.RequestedBy)254	}255	if p.From != nil {256		w.add("a.created_at >= ?", *p.From)257	}258	if p.To != nil {259		w.add("a.created_at < ?", *p.To)260	}261	cur, err := uuidCursor(p.Cursor)262	if err != nil {263		return nil, err264	}265	if cur != nil {266		w.add("a.id < ?", *cur)267	}268	limit := pageLimit(p.Limit)269	rows, err := h.Pool.Query(ctx, actionSelect+w.sql()+" order by a.id desc limit "+w.arg(limit+1), w.args...)270	if err != nil {271		return nil, err272	}273	items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Action, error) { return scanAction(r) })274	if err != nil {275		return nil, err276	}277	out := ActionList{Items: items}278	if len(items) > limit {279		out.Items = items[:limit]280		out.NextCursor = ptr(items[limit-1].Id.String())281	}282	if out.Items == nil {283		out.Items = []Action{}284	}285	return ListActions200JSONResponse{Body: out}, nil286}287288func (h *opsHandlers) actionEvents(ctx context.Context, id uuid.UUID, output bool, after int64, limit int) ([]ActionEvent, error) {289	cmp := "<>"290	if output {291		cmp = "="292	}293	rows, err := h.Pool.Query(ctx, `select id, ts, kind, stream, data from action_events where action_id=$1 and kind `+cmp+` 'output' and id > $2 order by id limit $3`, id, after, limit)294	if err != nil {295		return nil, err296	}297	evs, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (ActionEvent, error) {298		var e ActionEvent299		var stream *string300		err := r.Scan(&e.Id, &e.Ts, &e.Kind, &stream, &e.Data)301		if stream != nil && (*stream == "stdout" || *stream == "stderr") {302			e.Stream = ptr(ActionEventStream(*stream))303		}304		return e, err305	})306	if evs == nil {307		evs = []ActionEvent{}308	}309	return evs, err310}311312func (h *opsHandlers) GetAction(ctx context.Context, req GetActionRequestObject) (GetActionResponseObject, error) {313	a, err := h.loadAction(ctx, h.Pool, req.Id)314	if err != nil {315		return nil, err316	}317	evs, err := h.actionEvents(ctx, req.Id, false, 0, 500)318	if err != nil {319		return nil, err320	}321	return GetAction200JSONResponse{Body: ActionDetail{Action: a, Events: evs}}, nil322}323324// GetActionOutput serves what the node printed. Output can contain operational325// detail, so on top of actions.read it requires logs.read on the node.326func (h *opsHandlers) GetActionOutput(ctx context.Context, req GetActionOutputRequestObject) (GetActionOutputResponseObject, error) {327	a, err := h.loadAction(ctx, h.Pool, req.Id)328	if err != nil {329		return nil, err330	}331	for _, res := range actionResources(a.NodeId, string(a.Target.Kind), a.Target.Id) {332		if err := h.authorize(ctx, authz.LogsRead, res); err != nil {333			return nil, err334		}335	}336	after, limit := deref(req.Params.After), 500337	if req.Params.Limit != nil {338		limit = min(max(*req.Params.Limit, 1), 1000)339	}340	if after < 0 {341		return nil, httpx.Validation("after must be >= 0.")342	}343	evs, err := h.actionEvents(ctx, req.Id, true, after, limit)344	if err != nil {345		return nil, err346	}347	out := ActionOutput{Items: evs, NextAfter: after, Complete: actions.IsTerminal(string(a.State)) && len(evs) < limit}348	if len(evs) > 0 {349		out.NextAfter = evs[len(evs)-1].Id350	}351	var truncated bool352	if err := h.Pool.QueryRow(ctx, `select output_truncated from agent_commands where action_id=$1`, req.Id).Scan(&truncated); err == nil && truncated {353		out.Truncated = &truncated354	}355	return GetActionOutput200JSONResponse{Body: out}, nil356}357358// CancelAction: the requester, or anyone allowed to run actions of that criticality on the node.359func (h *opsHandlers) CancelAction(ctx context.Context, req CancelActionRequestObject) (CancelActionResponseObject, error) {360	a, err := h.loadAction(ctx, h.Pool, req.Id)361	if err != nil {362		return nil, err363	}364	if a.RequestedBy == nil || *a.RequestedBy != h.userID(ctx) {365		for _, res := range actionResources(a.NodeId, string(a.Target.Kind), a.Target.Id) {366			if err := h.authorize(ctx, authz.Permission("actions.run."+string(a.Criticality)), res); err != nil {367				return nil, err368			}369		}370	}371	err = pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {372		state, err := actions.RequestCancel(ctx, tx, req.Id, nowUTC())373		if err != nil {374			return err375		}376		detail := "cancelled"377		if state != "cancelled" {378			detail = "cancellation requested from the agent"379		}380		return h.audit(ctx, tx, "action.cancel", string(a.Target.Kind), a.Target.Id.String(), audit.Success, detail, map[string]any{"action_id": req.Id.String(), "type": a.Type})381	})382	if err != nil {383		return nil, h.refused(ctx, "action.cancel", string(a.Target.Kind), a.Target.Id.String(), map[string]any{"action_id": req.Id.String()}, err)384	}385	a, err = h.loadAction(ctx, h.Pool, req.Id)386	if err != nil {387		return nil, err388	}389	return CancelAction200JSONResponse{Body: a}, nil390}391392// decide implements approve and reject: a second person, never the requester.393func (h *opsHandlers) decide(ctx context.Context, id uuid.UUID, approve bool, text string) (Action, error) {394	me := h.userID(ctx)395	verb := "action.reject"396	if approve {397		verb = "action.approve"398	}399	var a Action400	var denied error401	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {402		var err error403		if a, err = scanAction(tx.QueryRow(ctx, actionSelect+` where a.id=$1 for update of a`, id)); err != nil {404			return notFound(err)405		}406		if a.State != ActionStatePendingApproval {407			return httpx.Conflict("conflict", "The action is not pending approval.")408		}409		if a.RequestedBy != nil && *a.RequestedBy == me {410			return httpx.ErrForbidden.With("reason", "The requester cannot approve or reject their own action.")411		}412		for _, res := range actionResources(a.NodeId, string(a.Target.Kind), a.Target.Id) {413			if err := h.authorize(ctx, authz.ActionsApprove, res); err != nil {414				return err415			}416		}417		now := nowUTC()418		ap := map[string]any{"action_id": id.String(), "type": a.Type}419		if now.Sub(a.CreatedAt) >= actions.ApprovalTTL {420			// Too late: close it here rather than let a stale request through.421			if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "expired", ErrorCode: "approval_expired", ErrorMessage: "No approval was given within 30 minutes.", At: now}); err != nil {422				return err423			}424			denied = httpx.Conflict("conflict", "The approval period of this action has expired.")425			return nil426		}427		if !approve {428			if _, err := tx.Exec(ctx, `update actions set rejection_reason=$2 where id=$1`, id, text); err != nil {429				return err430			}431			if err := actions.Event(ctx, tx, id, "approval", "", "rejected: "+text, now); err != nil {432				return err433			}434			if _, err := actions.Finish(ctx, tx, id, actions.Outcome{State: "rejected", At: now}); err != nil {435				return err436			}437			ap["reason"] = text438			return h.audit(ctx, tx, verb, string(a.Target.Kind), a.Target.Id.String(), audit.Success, "", ap)439		}440		if frozen, err := actions.Frozen(ctx, tx); err != nil {441			return err442		} else if frozen {443			return actions.ErrFrozen444		}445		if _, err := tx.Exec(ctx, `update actions set approved_by=$2, approved_at=$3 where id=$1`, id, me, now); err != nil {446			return err447		}448		if err := actions.Event(ctx, tx, id, "approval", "", strings.TrimSpace("approved "+text), now); err != nil {449			return err450		}451		if _, err := actions.Transition(ctx, tx, id, "pending_approval", "queued", now); err != nil {452			return err453		}454		return h.audit(ctx, tx, verb, string(a.Target.Kind), a.Target.Id.String(), audit.Success, "", ap)455	})456	if err == nil {457		err = denied458	}459	if err != nil {460		var he *httpx.Error461		if errors.As(err, &he) && he.Code == "forbidden" && he.Details["reason"] != nil {462			h.audit(ctx, h.Pool, verb, string(a.Target.Kind), a.Target.Id.String(), audit.Denied, "requester cannot decide on own action", map[string]any{"action_id": id.String()})463			return a, err464		}465		return a, h.refused(ctx, verb, "action", id.String(), map[string]any{"action_id": id.String()}, err)466	}467	return h.loadAction(ctx, h.Pool, id)468}469470func (h *opsHandlers) ApproveAction(ctx context.Context, req ApproveActionRequestObject) (ApproveActionResponseObject, error) {471	comment := ""472	if req.Body != nil {473		comment = strings.TrimSpace(deref(req.Body.Comment))474	}475	if len(comment) > 2000 {476		return nil, httpx.Validation("comment is too long.")477	}478	a, err := h.decide(ctx, req.Id, true, comment)479	if err != nil {480		return nil, err481	}482	return ApproveAction200JSONResponse{Body: a}, nil483}484485func (h *opsHandlers) RejectAction(ctx context.Context, req RejectActionRequestObject) (RejectActionResponseObject, error) {486	if req.Body == nil || strings.TrimSpace(req.Body.Reason) == "" || len(req.Body.Reason) > 2000 {487		return nil, httpx.Validation("reason is required (at most 2000 characters).")488	}489	a, err := h.decide(ctx, req.Id, false, strings.TrimSpace(req.Body.Reason))490	if err != nil {491		return nil, err492	}493	return RejectAction200JSONResponse{Body: a}, nil494}495
Files main
Menu

Theme