SPB Git

Repository Public

store.go 22.5 KB 584 lines go

1package actions23import (4	"context"5	"crypto/sha256"6	"encoding/json"7	"errors"8	"fmt"9	"net/http"10	"slices"11	"time"1213	"github.com/google/uuid"14	"github.com/jackc/pgx/v5"15	"github.com/jackc/pgx/v5/pgconn"1617	"clustr.spboucher.ai/mcc/internal/events"18	"clustr.spboucher.ai/mcc/internal/httpx"19	"clustr.spboucher.ai/mcc/internal/ids"20)2122// Notification channels (PostgreSQL LISTEN/NOTIFY).23const (24	ChanCommands       = "mcc_commands"        // orchestrator -> ingest: payload = agent id (0002 contract)25	ChanCommandUpdates = "mcc_command_updates" // ingest -> orchestrator: payload = command id (0002 contract)26	ChanActions        = "mcc_actions"         // control-api -> orchestrator: an action was queued or needs attention27)2829// ApprovalTTL is how long an action may wait for its second approval.30const ApprovalTTL = 30 * time.Minute3132var terminal = []string{"succeeded", "failed", "cancelled", "timed_out", "rejected", "expired"}3334// IsTerminal reports whether an action state is final.35func IsTerminal(state string) bool { return slices.Contains(terminal, state) }3637// Refuse builds the 422 answer of a failed pre-check.38func Refuse(code, message string) *httpx.Error {39	return httpx.Err(http.StatusUnprocessableEntity, code, message)40}4142// ErrFrozen is answered while the emergency stop is engaged.43var ErrFrozen = httpx.Err(http.StatusServiceUnavailable, "orchestrator_frozen", "The emergency stop is engaged: no action can be started.")4445// Querier is satisfied by *pgxpool.Pool and pgx.Tx.46type Querier interface {47	Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)48	QueryRow(ctx context.Context, sql string, args ...any) pgx.Row49	Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)50}5152// Frozen reads the emergency stop.53func Frozen(ctx context.Context, q Querier) (bool, error) {54	var frozen bool55	err := q.QueryRow(ctx, `select coalesce((select value = 'true'::jsonb from settings where key='orchestrator.frozen'), false)`).Scan(&frozen)56	return frozen, err57}5859// Target is the resolved target of an action.60type Target struct {61	Kind        string62	ID          uuid.UUID63	NodeID      *uuid.UUID64	NodeAlias   string65	Maintenance bool   // node in maintenance mode66	Label       string // alias, manager:name, kind:port or template67	Confirm     string // what the user must type for types that require confirmation68	Managed     bool   // services69	Gone        bool   // service or inference server no longer observed70	JobState    string71	Wire        map[string]any // identity of the target as sent to the agent (params.target)72}7374// ResolveTarget loads the target; an unknown or retired target is 404.75func ResolveTarget(ctx context.Context, q Querier, kind string, id uuid.UUID) (*Target, error) {76	t := &Target{Kind: kind, ID: id}77	var nodeID *uuid.UUID78	var alias *string79	var maint *bool80	var err error81	switch kind {82	case "node":83		var a string84		var m bool85		err = q.QueryRow(ctx, `select alias::text, maintenance from nodes where id=$1 and retired_at is null`, id).Scan(&a, &m)86		nodeID, alias, maint = &id, &a, &m87		t.Label, t.Confirm = a, a88		t.Wire = map[string]any{"kind": "node", "id": id.String()}89	case "service":90		var manager, name string91		var gone *time.Time92		err = q.QueryRow(ctx, `select s.node_id, s.manager::text, s.name, s.managed, s.gone_at, n.alias::text, n.maintenance93			from services s join nodes n on n.id=s.node_id where s.id=$1 and n.retired_at is null`, id).94			Scan(&nodeID, &manager, &name, &t.Managed, &gone, &alias, &maint)95		t.Gone = gone != nil96		t.Label, t.Confirm = manager+":"+name, name97		t.Wire = map[string]any{"kind": "service", "id": id.String(), "manager": manager, "name": name}98	case "inference_server":99		var skind string100		var port int101		var gone *time.Time102		err = q.QueryRow(ctx, `select i.node_id, i.kind, i.port, i.gone_at, n.alias::text, n.maintenance103			from inference_servers i join nodes n on n.id=i.node_id where i.id=$1 and n.retired_at is null`, id).104			Scan(&nodeID, &skind, &port, &gone, &alias, &maint)105		t.Gone = gone != nil106		t.Label = fmt.Sprintf("%s:%d", skind, port)107		t.Wire = map[string]any{"kind": "inference_server", "id": id.String(), "server_kind": skind, "port": port}108	case "job":109		var tpl string110		err = q.QueryRow(ctx, `select j.node_id, j.template, j.state::text, n.alias::text, n.maintenance111			from jobs j left join nodes n on n.id=j.node_id where j.id=$1`, id).Scan(&nodeID, &tpl, &t.JobState, &alias, &maint)112		t.Label = tpl113		t.Wire = map[string]any{"kind": "job", "id": id.String()}114	default:115		return nil, httpx.Validation("target.kind is not supported.")116	}117	if errors.Is(err, pgx.ErrNoRows) {118		return nil, httpx.ErrNotFound119	}120	if err != nil {121		return nil, err122	}123	t.NodeID = nodeID124	if alias != nil {125		t.NodeAlias = *alias126	}127	if maint != nil {128		t.Maintenance = *maint129	}130	if t.Confirm == "" {131		t.Confirm = t.NodeAlias132	}133	return t, nil134}135136// LockKey is the action_locks key for this type on this target ("" = no lock).137func LockKey(t Type, tg *Target) string {138	switch t.LockScope {139	case LockNode:140		if tg.NodeID != nil {141			return "node:" + tg.NodeID.String()142		}143	case LockService:144		return "service:" + tg.ID.String()145	case LockJob:146		return "job:" + tg.ID.String()147	}148	return ""149}150151// PrecheckOpts parametrizes Precheck.152type PrecheckOpts struct {153	OverrideMaintenance bool154	Now                 time.Time155	OfflineAfter        time.Duration156}157158// Precheck verifies everything that must hold for the action to be sent:159// target usable, agent enrolled, reporting and declaring the capability,160// service managed, node not in maintenance. It is run when the action is161// requested AND again by the orchestrator just before the command is signed.162// It returns the agent that would execute (uuid.Nil for server-side types).163func Precheck(ctx context.Context, q Querier, t Type, tg *Target, params map[string]any, o PrecheckOpts) (agentID uuid.UUID, err error) {164	check := func(code, name, msg string) *httpx.Error {165		e := Refuse(code, msg).With("check", name)166		if tg.NodeID != nil {167			e = e.With("node_id", tg.NodeID.String())168		}169		return e170	}171	if !t.Available {172		return uuid.Nil, Refuse("action_unavailable", "This action type is not enabled in this version.")173	}174	if !slices.Contains(t.TargetKinds, tg.Kind) {175		return uuid.Nil, check("precheck_failed", "target_kind", "This action does not apply to this kind of target.")176	}177	if tg.Gone {178		return uuid.Nil, check("precheck_failed", "target_present", "The target is no longer observed on its node.")179	}180	if tg.Kind == "service" && !tg.Managed {181		return uuid.Nil, check("precheck_failed", "service_managed", "Actions are only allowed on managed services.")182	}183	if tg.Maintenance && !t.MaintenanceRelated && !o.OverrideMaintenance {184		return uuid.Nil, check("node_in_maintenance", "maintenance", "The node "+tg.NodeAlias+" is in maintenance mode.")185	}186	if t.ServerSide {187		return uuid.Nil, nil188	}189	if tg.NodeID == nil {190		return uuid.Nil, check("precheck_failed", "node_assigned", "The target is not placed on a node.")191	}192	var caps []string193	var seen *time.Time194	err = q.QueryRow(ctx, `select a.id, a.capabilities, greatest(a.last_seen_at, ns.received_at)195		from agents a left join node_state ns on ns.node_id=a.node_id where a.node_id=$1 and a.revoked_at is null`, *tg.NodeID).196		Scan(&agentID, &caps, &seen)197	if errors.Is(err, pgx.ErrNoRows) {198		return uuid.Nil, check("agent_offline", "agent_enrolled", "No active agent is enrolled on "+tg.NodeAlias+".")199	}200	if err != nil {201		return uuid.Nil, err202	}203	if seen == nil || o.Now.Sub(*seen) > o.OfflineAfter {204		return uuid.Nil, check("agent_offline", "agent_online", "The agent of "+tg.NodeAlias+" is offline.")205	}206	if c := t.Capability(params); !slices.Contains(caps, c) {207		return uuid.Nil, check("capability_missing", "capability", "The agent of "+tg.NodeAlias+" does not allow "+c+".").With("capability", c)208	}209	if need, ok := params["mem_bytes"].(float64); ok && t.Name == "model.server.start" {210		// Memory estimate: only real, recent measurements count; without them the check cannot pass.211		var total, used *float64212		var at *time.Time213		err = q.QueryRow(ctx, `select (values->>'mem.total')::float8, (values->>'mem.used')::float8, received_at from node_state where node_id=$1`, *tg.NodeID).Scan(&total, &used, &at)214		if err != nil && !errors.Is(err, pgx.ErrNoRows) {215			return uuid.Nil, err216		}217		if total == nil || used == nil || at == nil || o.Now.Sub(*at) > o.OfflineAfter {218			return uuid.Nil, check("precheck_failed", "memory", "The free memory of "+tg.NodeAlias+" is not known; the estimate cannot be checked.")219		}220		if free := *total - *used; free < need {221			return uuid.Nil, check("precheck_failed", "memory", "Not enough free memory on "+tg.NodeAlias+".").With("free_bytes", int64(free)).With("needed_bytes", int64(need))222		}223	}224	return agentID, nil225}226227// HashRequest is the idempotency fingerprint of a request body.228func HashRequest(v any) []byte {229	b, _ := json.Marshal(v) // maps are encoded with sorted keys: canonical230	h := sha256.Sum256(b)231	return h[:]232}233234// Request is a validated action about to be stored.235type Request struct {236	Type                Type237	Target              *Target238	Params              map[string]any239	Justification       string240	OverrideMaintenance bool241	RequestedBy         *uuid.UUID // nil: created by the platform (scheduler)242	Via                 string     // api | scheduler | cli243	IdempotencyKey      string244	RequestHash         []byte245	CorrelationID       uuid.UUID246	JobID               *uuid.UUID247	TimeoutS            int // 0 = the type's timeout; never above it248	Now                 time.Time249}250251// ErrIdempotencyConflict: same key, different request.252var ErrIdempotencyConflict = httpx.Conflict("idempotency_conflict", "Idempotency-Key already used with a different request.")253254// Existing returns the action already created by this user with this key, if any.255func Existing(ctx context.Context, q Querier, user uuid.UUID, key string, hash []byte) (uuid.UUID, bool, error) {256	var id uuid.UUID257	var have []byte258	err := q.QueryRow(ctx, `select id, request_hash from actions where requested_by=$1 and idempotency_key=$2`, user, key).Scan(&id, &have)259	if errors.Is(err, pgx.ErrNoRows) {260		return uuid.Nil, false, nil261	}262	if err != nil {263		return uuid.Nil, false, err264	}265	if string(have) != string(hash) {266		return uuid.Nil, false, ErrIdempotencyConflict267	}268	return id, true, nil269}270271// Insert stores the action and takes the target lock, atomically, inside tx.272// created=false means the idempotency key already produced an action (its id273// is returned and nothing was written). A busy target is 409 target_locked;274// the caller must then roll the transaction back.275func Insert(ctx context.Context, tx pgx.Tx, r Request) (id uuid.UUID, created bool, err error) {276	if r.Params == nil {277		r.Params = map[string]any{}278	}279	if r.Now.IsZero() {280		r.Now = time.Now().UTC()281	}282	timeout := r.Type.TimeoutS283	if r.TimeoutS > 0 && r.TimeoutS < timeout {284		timeout = r.TimeoutS285	}286	state := "queued"287	var queuedAt *time.Time288	if r.RequestedBy != nil && r.Type.RequiresApproval() {289		state = "pending_approval"290	} else {291		queuedAt = &r.Now292	}293	var nodeID *uuid.UUID294	if r.Target.NodeID != nil {295		nodeID = r.Target.NodeID296	}297	id = ids.New()298	err = tx.QueryRow(ctx, `insert into actions(id, type, criticality, node_id, target_kind, target_id, target_label, params, state, requested_by,299			requested_via, justification, idempotency_key, request_hash, approval_required, created_at, queued_at, timeout_s, correlation_id, job_id, override_maintenance)300		values ($1,$2,$3,$4,$5,$6,$7,$8,$9::action_state,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21)301		on conflict (requested_by, idempotency_key) do nothing returning id`,302		id, r.Type.Name, r.Type.Criticality, nodeID, r.Target.Kind, r.Target.ID.String(), r.Target.Label, r.Params, state, r.RequestedBy,303		r.Via, r.Justification, r.IdempotencyKey, r.RequestHash, state == "pending_approval", r.Now, queuedAt, timeout, r.CorrelationID, r.JobID,304		r.OverrideMaintenance).Scan(&id)305	if errors.Is(err, pgx.ErrNoRows) {306		// A concurrent (or earlier) request with the same key won; it is committed by now.307		if r.RequestedBy == nil {308			return uuid.Nil, false, errors.New("actions: unexpected idempotency conflict for a system action")309		}310		ex, ok, err := Existing(ctx, tx, *r.RequestedBy, r.IdempotencyKey, r.RequestHash)311		if err != nil {312			return uuid.Nil, false, err313		}314		if !ok {315			return uuid.Nil, false, errors.New("actions: idempotent action vanished")316		}317		return ex, false, nil318	}319	if err != nil {320		return uuid.Nil, false, err321	}322	if key := LockKey(r.Type, r.Target); key != "" {323		// Defensive: a lock whose action is already final is stale (Finish removes locks in the same transaction).324		if _, err := tx.Exec(ctx, `delete from action_locks l using actions a where l.target_key=$1 and a.id=l.action_id325			and a.state in ('succeeded','failed','cancelled','timed_out','rejected','expired')`, key); err != nil {326			return uuid.Nil, false, err327		}328		tag, err := tx.Exec(ctx, `insert into action_locks(target_key, action_id, acquired_at) values ($1,$2,$3) on conflict do nothing`, key, id, r.Now)329		if err != nil {330			return uuid.Nil, false, err331		}332		if tag.RowsAffected() == 0 {333			var holder uuid.UUID334			if err := tx.QueryRow(ctx, `select action_id from action_locks where target_key=$1`, key).Scan(&holder); err != nil && !errors.Is(err, pgx.ErrNoRows) {335				return uuid.Nil, false, err336			}337			return uuid.Nil, false, httpx.Conflict("target_locked", "Another action is in progress on this target.").With("action_id", holder.String())338		}339	}340	if err := Event(ctx, tx, id, "state", "", state, r.Now); err != nil {341		return uuid.Nil, false, err342	}343	return id, true, Changed(ctx, tx, id, state == "queued")344}345346// Event appends a lifecycle event to an action.347func Event(ctx context.Context, tx Querier, id uuid.UUID, kind, stream, data string, at time.Time) error {348	_, err := tx.Exec(ctx, `insert into action_events(action_id, ts, kind, stream, data) values ($1,$2,$3,nullif($4,''),$5)`, id, at, kind, stream, data)349	return err350}351352// Changed publishes action.changed and, when wake is set, wakes the orchestrator.353func Changed(ctx context.Context, tx Querier, id uuid.UUID, wake bool) error {354	var typ, state, kind, target string355	var nodeID, by *uuid.UUID356	var unknown bool357	if err := tx.QueryRow(ctx, `select type, state::text, target_kind, target_id, node_id, requested_by, coalesce((result->>'outcome_unknown')::boolean, false)358		from actions where id=$1`, id).Scan(&typ, &state, &kind, &target, &nodeID, &by, &unknown); err != nil {359		return err360	}361	p := map[string]any{"action_id": id, "type": typ, "state": state, "target": map[string]any{"kind": kind, "id": target}, "outcome_unknown": unknown}362	if nodeID != nil {363		p["node_id"] = *nodeID364	}365	if by != nil {366		p["requested_by"] = *by367	}368	if err := events.Publish(ctx, tx, "action.changed", p); err != nil {369		return err370	}371	if wake {372		_, err := tx.Exec(ctx, `select pg_notify($1, $2)`, ChanActions, id.String())373		return err374	}375	return nil376}377378// Outcome is the final state of an action.379type Outcome struct {380	State        string // succeeded | failed | cancelled | timed_out | rejected | expired381	ErrorCode    string382	ErrorMessage string383	Result       map[string]any384	// OutcomeUnknown: the command reached the agent but no result came back.385	// It may or may not have run; it is never sent again.386	OutcomeUnknown bool387	At             time.Time388}389390// Finish moves a non-terminal action to its final state, releases its lock391// and publishes the change. It returns false when the action was already final.392func Finish(ctx context.Context, tx Querier, id uuid.UUID, o Outcome) (bool, error) {393	if !IsTerminal(o.State) {394		return false, fmt.Errorf("actions: %q is not a final state", o.State)395	}396	if o.At.IsZero() {397		o.At = time.Now().UTC()398	}399	res := map[string]any{}400	for k, v := range o.Result {401		res[k] = v402	}403	res["outcome_unknown"] = o.OutcomeUnknown404	tag, err := tx.Exec(ctx, `update actions set state=$2::action_state, finished_at=$3, result=$4, error_code=nullif($5,''), error_message=nullif($6,'')405		where id=$1 and state not in ('succeeded','failed','cancelled','timed_out','rejected','expired')`,406		id, o.State, o.At, res, o.ErrorCode, truncate(o.ErrorMessage, 2000))407	if err != nil || tag.RowsAffected() == 0 {408		return false, err409	}410	if _, err := tx.Exec(ctx, `delete from action_locks where action_id=$1`, id); err != nil {411		return false, err412	}413	data := o.State414	if o.ErrorCode != "" {415		data += " (" + o.ErrorCode + ")"416	}417	if err := Event(ctx, tx, id, "state", "", data, o.At); err != nil {418		return false, err419	}420	return true, Changed(ctx, tx, id, true)421}422423// Transition moves an action from one non-terminal state to another.424func Transition(ctx context.Context, tx Querier, id uuid.UUID, from, to string, at time.Time) (bool, error) {425	tag, err := tx.Exec(ctx, `update actions set state=$3::action_state,426			queued_at  = case when $3='queued'  then $4 else queued_at end,427			started_at = case when $3='running' then $4 else started_at end428		where id=$1 and state=$2::action_state`, id, from, to, at)429	if err != nil || tag.RowsAffected() == 0 {430		return false, err431	}432	if err := Event(ctx, tx, id, "state", "", to, at); err != nil {433		return false, err434	}435	return true, Changed(ctx, tx, id, to == "queued")436}437438// ErrFinished is answered when a finished action is cancelled, approved or rejected.439var ErrFinished = httpx.Conflict("conflict", "The action is already finished.")440441// RequestCancel cancels an action. Before the command leaves the server the442// cancellation is immediate and certain. Once the command may have reached443// the agent it is only a request: the agent decides, and its result decides444// the final state.445func RequestCancel(ctx context.Context, tx Querier, id uuid.UUID, at time.Time) (state string, err error) {446	if err := tx.QueryRow(ctx, `select state::text from actions where id=$1 for update`, id).Scan(&state); err != nil {447		if errors.Is(err, pgx.ErrNoRows) {448			return "", httpx.ErrNotFound449		}450		return "", err451	}452	switch state {453	case "pending_approval", "queued":454		_, err := Finish(ctx, tx, id, Outcome{State: "cancelled", At: at})455		return "cancelled", err456	case "dispatched", "running":457		// Never delivered: withdraw the command, the action is cancelled for certain.458		tag, err := tx.Exec(ctx, `update agent_commands set state='cancelled', finished_at=$2, cancel_requested_at=coalesce(cancel_requested_at,$2)459			where action_id=$1 and state='pending'`, id, at)460		if err != nil {461			return "", err462		}463		if tag.RowsAffected() == 1 {464			_, err := Finish(ctx, tx, id, Outcome{State: "cancelled", At: at})465			return "cancelled", err466		}467		if _, err := tx.Exec(ctx, `update actions set cancel_requested_at=coalesce(cancel_requested_at,$2) where id=$1`, id, at); err != nil {468			return "", err469		}470		var agent *uuid.UUID471		err = tx.QueryRow(ctx, `update agent_commands set cancel_requested_at=coalesce(cancel_requested_at,$2)472			where action_id=$1 and state in ('delivered','acked') returning agent_id`, id, at).Scan(&agent)473		if err != nil && !errors.Is(err, pgx.ErrNoRows) {474			return "", err475		}476		if agent != nil { // wake the transport so the agent learns about the cancellation477			if _, err := tx.Exec(ctx, `select pg_notify($1,$2)`, ChanCommands, agent.String()); err != nil {478				return "", err479			}480		}481		if err := Event(ctx, tx, id, "note", "", "cancellation requested", at); err != nil {482			return "", err483		}484		return state, Changed(ctx, tx, id, true)485	}486	return "", ErrFinished487}488489// CancelJob cancels a job: at once when it is not running, cooperatively490// (through its job.run action) when it is. It returns the resulting job state.491func CancelJob(ctx context.Context, tx Querier, jobID uuid.UUID, at time.Time) (string, error) {492	var state, tpl string493	var nodeID *uuid.UUID494	if err := tx.QueryRow(ctx, `select state::text, template, node_id from jobs where id=$1 for update`, jobID).Scan(&state, &tpl, &nodeID); err != nil {495		if errors.Is(err, pgx.ErrNoRows) {496			return "", httpx.ErrNotFound497		}498		return "", err499	}500	switch state {501	case "queued":502		if _, err := tx.Exec(ctx, `update jobs set state='cancelled', finished_at=$2, cancel_requested_at=$2, next_attempt_at=null, error='cancelled before start' where id=$1`, jobID, at); err != nil {503			return "", err504		}505		return "cancelled", JobChanged(ctx, tx, jobID)506	case "running":507		if _, err := tx.Exec(ctx, `update jobs set cancel_requested_at=coalesce(cancel_requested_at,$2) where id=$1`, jobID, at); err != nil {508			return "", err509		}510		var actionID uuid.UUID511		err := tx.QueryRow(ctx, `select id from actions where job_id=$1 and type='job.run'512			and state in ('queued','dispatched','running') order by created_at desc limit 1`, jobID).Scan(&actionID)513		if errors.Is(err, pgx.ErrNoRows) {514			return "running", nil // the runner will observe cancel_requested_at515		}516		if err != nil {517			return "", err518		}519		if _, err := RequestCancel(ctx, tx, actionID, at); err != nil && !errors.Is(err, ErrFinished) {520			return "", err521		}522		return "running", nil523	}524	return "", httpx.Conflict("conflict", "The job is already finished.")525}526527// JobChanged publishes job.changed.528func JobChanged(ctx context.Context, tx Querier, jobID uuid.UUID) error {529	var tpl, state string530	var nodeID *uuid.UUID531	var progress *float32532	if err := tx.QueryRow(ctx, `select template, state::text, node_id, progress from jobs where id=$1`, jobID).Scan(&tpl, &state, &nodeID, &progress); err != nil {533		return err534	}535	p := map[string]any{"job_id": jobID, "template": tpl, "state": state}536	if nodeID != nil {537		p["node_id"] = *nodeID538	}539	if progress != nil {540		p["progress"] = *progress541	}542	return events.Publish(ctx, tx, "job.changed", p)543}544545// ExecuteServerSide performs a ServerSide action inside tx and returns its result.546func ExecuteServerSide(ctx context.Context, tx Querier, typ string, tg *Target, params map[string]any, at time.Time) (map[string]any, error) {547	switch typ {548	case "node.maintenance.set":549		enabled, _ := params["enabled"].(bool)550		reason, _ := params["reason"].(string)551		if !enabled {552			reason = ""553		}554		var before bool555		if err := tx.QueryRow(ctx, `select maintenance from nodes where id=$1 for update`, tg.ID).Scan(&before); err != nil {556			return nil, err557		}558		if _, err := tx.Exec(ctx, `update nodes set maintenance=$2, maintenance_reason=$3 where id=$1`, tg.ID, enabled, reason); err != nil {559			return nil, err560		}561		if err := events.Publish(ctx, tx, "node.updated", map[string]any{"node_id": tg.ID, "change": "updated"}); err != nil {562			return nil, err563		}564		return map[string]any{"maintenance": enabled, "previous": before}, nil565	case "job.cancel":566		state, err := CancelJob(ctx, tx, tg.ID, at)567		if err != nil {568			return nil, err569		}570		return map[string]any{"job_state": state}, nil571	}572	return nil, fmt.Errorf("actions: %s is not a server-side type", typ)573}574575func truncate(s string, n int) string {576	if len(s) <= n {577		return s578	}579	for n > 0 && s[n]&0xC0 == 0x80 { // do not cut a UTF-8 sequence580		n--581	}582	return s[:n]583}584
Files main
Menu

Theme