Repository Public
1package actions23import (4 "context"5 "crypto/sha256"6 "encoding/json"7 "errors"8 "fmt"9 "net/http"10 "slices"11 "time"1213 "github.com/google/uuid"14 "github.com/jackc/pgx/v5"15 "github.com/jackc/pgx/v5/pgconn"1617 "clustr.spboucher.ai/mcc/internal/events"18 "clustr.spboucher.ai/mcc/internal/httpx"19 "clustr.spboucher.ai/mcc/internal/ids"20)2122// Notification channels (PostgreSQL LISTEN/NOTIFY).23const (24 ChanCommands = "mcc_commands" // orchestrator -> ingest: payload = agent id (0002 contract)25 ChanCommandUpdates = "mcc_command_updates" // ingest -> orchestrator: payload = command id (0002 contract)26 ChanActions = "mcc_actions" // control-api -> orchestrator: an action was queued or needs attention27)2829// ApprovalTTL is how long an action may wait for its second approval.30const ApprovalTTL = 30 * time.Minute3132var terminal = []string{"succeeded", "failed", "cancelled", "timed_out", "rejected", "expired"}3334// IsTerminal reports whether an action state is final.35func IsTerminal(state string) bool { return slices.Contains(terminal, state) }3637// Refuse builds the 422 answer of a failed pre-check.38func Refuse(code, message string) *httpx.Error {39 return httpx.Err(http.StatusUnprocessableEntity, code, message)40}4142// ErrFrozen is answered while the emergency stop is engaged.43var ErrFrozen = httpx.Err(http.StatusServiceUnavailable, "orchestrator_frozen", "The emergency stop is engaged: no action can be started.")4445// Querier is satisfied by *pgxpool.Pool and pgx.Tx.46type Querier interface {47 Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)48 QueryRow(ctx context.Context, sql string, args ...any) pgx.Row49 Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)50}5152// Frozen reads the emergency stop.53func Frozen(ctx context.Context, q Querier) (bool, error) {54 var frozen bool55 err := q.QueryRow(ctx, `select coalesce((select value = 'true'::jsonb from settings where key='orchestrator.frozen'), false)`).Scan(&frozen)56 return frozen, err57}5859// Target is the resolved target of an action.60type Target struct {61 Kind string62 ID uuid.UUID63 NodeID *uuid.UUID64 NodeAlias string65 Maintenance bool // node in maintenance mode66 Label string // alias, manager:name, kind:port or template67 Confirm string // what the user must type for types that require confirmation68 Managed bool // services69 Gone bool // service or inference server no longer observed70 JobState string71 Wire map[string]any // identity of the target as sent to the agent (params.target)72}7374// ResolveTarget loads the target; an unknown or retired target is 404.75func ResolveTarget(ctx context.Context, q Querier, kind string, id uuid.UUID) (*Target, error) {76 t := &Target{Kind: kind, ID: id}77 var nodeID *uuid.UUID78 var alias *string79 var maint *bool80 var err error81 switch kind {82 case "node":83 var a string84 var m bool85 err = q.QueryRow(ctx, `select alias::text, maintenance from nodes where id=$1 and retired_at is null`, id).Scan(&a, &m)86 nodeID, alias, maint = &id, &a, &m87 t.Label, t.Confirm = a, a88 t.Wire = map[string]any{"kind": "node", "id": id.String()}89 case "service":90 var manager, name string91 var gone *time.Time92 err = q.QueryRow(ctx, `select s.node_id, s.manager::text, s.name, s.managed, s.gone_at, n.alias::text, n.maintenance93 from services s join nodes n on n.id=s.node_id where s.id=$1 and n.retired_at is null`, id).94 Scan(&nodeID, &manager, &name, &t.Managed, &gone, &alias, &maint)95 t.Gone = gone != nil96 t.Label, t.Confirm = manager+":"+name, name97 t.Wire = map[string]any{"kind": "service", "id": id.String(), "manager": manager, "name": name}98 case "inference_server":99 var skind string100 var port int101 var gone *time.Time102 err = q.QueryRow(ctx, `select i.node_id, i.kind, i.port, i.gone_at, n.alias::text, n.maintenance103 from inference_servers i join nodes n on n.id=i.node_id where i.id=$1 and n.retired_at is null`, id).104 Scan(&nodeID, &skind, &port, &gone, &alias, &maint)105 t.Gone = gone != nil106 t.Label = fmt.Sprintf("%s:%d", skind, port)107 t.Wire = map[string]any{"kind": "inference_server", "id": id.String(), "server_kind": skind, "port": port}108 case "job":109 var tpl string110 err = q.QueryRow(ctx, `select j.node_id, j.template, j.state::text, n.alias::text, n.maintenance111 from jobs j left join nodes n on n.id=j.node_id where j.id=$1`, id).Scan(&nodeID, &tpl, &t.JobState, &alias, &maint)112 t.Label = tpl113 t.Wire = map[string]any{"kind": "job", "id": id.String()}114 default:115 return nil, httpx.Validation("target.kind is not supported.")116 }117 if errors.Is(err, pgx.ErrNoRows) {118 return nil, httpx.ErrNotFound119 }120 if err != nil {121 return nil, err122 }123 t.NodeID = nodeID124 if alias != nil {125 t.NodeAlias = *alias126 }127 if maint != nil {128 t.Maintenance = *maint129 }130 if t.Confirm == "" {131 t.Confirm = t.NodeAlias132 }133 return t, nil134}135136// LockKey is the action_locks key for this type on this target ("" = no lock).137func LockKey(t Type, tg *Target) string {138 switch t.LockScope {139 case LockNode:140 if tg.NodeID != nil {141 return "node:" + tg.NodeID.String()142 }143 case LockService:144 return "service:" + tg.ID.String()145 case LockJob:146 return "job:" + tg.ID.String()147 }148 return ""149}150151// PrecheckOpts parametrizes Precheck.152type PrecheckOpts struct {153 OverrideMaintenance bool154 Now time.Time155 OfflineAfter time.Duration156}157158// Precheck verifies everything that must hold for the action to be sent:159// target usable, agent enrolled, reporting and declaring the capability,160// service managed, node not in maintenance. It is run when the action is161// requested AND again by the orchestrator just before the command is signed.162// It returns the agent that would execute (uuid.Nil for server-side types).163func Precheck(ctx context.Context, q Querier, t Type, tg *Target, params map[string]any, o PrecheckOpts) (agentID uuid.UUID, err error) {164 check := func(code, name, msg string) *httpx.Error {165 e := Refuse(code, msg).With("check", name)166 if tg.NodeID != nil {167 e = e.With("node_id", tg.NodeID.String())168 }169 return e170 }171 if !t.Available {172 return uuid.Nil, Refuse("action_unavailable", "This action type is not enabled in this version.")173 }174 if !slices.Contains(t.TargetKinds, tg.Kind) {175 return uuid.Nil, check("precheck_failed", "target_kind", "This action does not apply to this kind of target.")176 }177 if tg.Gone {178 return uuid.Nil, check("precheck_failed", "target_present", "The target is no longer observed on its node.")179 }180 if tg.Kind == "service" && !tg.Managed {181 return uuid.Nil, check("precheck_failed", "service_managed", "Actions are only allowed on managed services.")182 }183 if tg.Maintenance && !t.MaintenanceRelated && !o.OverrideMaintenance {184 return uuid.Nil, check("node_in_maintenance", "maintenance", "The node "+tg.NodeAlias+" is in maintenance mode.")185 }186 if t.ServerSide {187 return uuid.Nil, nil188 }189 if tg.NodeID == nil {190 return uuid.Nil, check("precheck_failed", "node_assigned", "The target is not placed on a node.")191 }192 var caps []string193 var seen *time.Time194 err = q.QueryRow(ctx, `select a.id, a.capabilities, greatest(a.last_seen_at, ns.received_at)195 from agents a left join node_state ns on ns.node_id=a.node_id where a.node_id=$1 and a.revoked_at is null`, *tg.NodeID).196 Scan(&agentID, &caps, &seen)197 if errors.Is(err, pgx.ErrNoRows) {198 return uuid.Nil, check("agent_offline", "agent_enrolled", "No active agent is enrolled on "+tg.NodeAlias+".")199 }200 if err != nil {201 return uuid.Nil, err202 }203 if seen == nil || o.Now.Sub(*seen) > o.OfflineAfter {204 return uuid.Nil, check("agent_offline", "agent_online", "The agent of "+tg.NodeAlias+" is offline.")205 }206 if c := t.Capability(params); !slices.Contains(caps, c) {207 return uuid.Nil, check("capability_missing", "capability", "The agent of "+tg.NodeAlias+" does not allow "+c+".").With("capability", c)208 }209 if need, ok := params["mem_bytes"].(float64); ok && t.Name == "model.server.start" {210 // Memory estimate: only real, recent measurements count; without them the check cannot pass.211 var total, used *float64212 var at *time.Time213 err = q.QueryRow(ctx, `select (values->>'mem.total')::float8, (values->>'mem.used')::float8, received_at from node_state where node_id=$1`, *tg.NodeID).Scan(&total, &used, &at)214 if err != nil && !errors.Is(err, pgx.ErrNoRows) {215 return uuid.Nil, err216 }217 if total == nil || used == nil || at == nil || o.Now.Sub(*at) > o.OfflineAfter {218 return uuid.Nil, check("precheck_failed", "memory", "The free memory of "+tg.NodeAlias+" is not known; the estimate cannot be checked.")219 }220 if free := *total - *used; free < need {221 return uuid.Nil, check("precheck_failed", "memory", "Not enough free memory on "+tg.NodeAlias+".").With("free_bytes", int64(free)).With("needed_bytes", int64(need))222 }223 }224 return agentID, nil225}226227// HashRequest is the idempotency fingerprint of a request body.228func HashRequest(v any) []byte {229 b, _ := json.Marshal(v) // maps are encoded with sorted keys: canonical230 h := sha256.Sum256(b)231 return h[:]232}233234// Request is a validated action about to be stored.235type Request struct {236 Type Type237 Target *Target238 Params map[string]any239 Justification string240 OverrideMaintenance bool241 RequestedBy *uuid.UUID // nil: created by the platform (scheduler)242 Via string // api | scheduler | cli243 IdempotencyKey string244 RequestHash []byte245 CorrelationID uuid.UUID246 JobID *uuid.UUID247 TimeoutS int // 0 = the type's timeout; never above it248 Now time.Time249}250251// ErrIdempotencyConflict: same key, different request.252var ErrIdempotencyConflict = httpx.Conflict("idempotency_conflict", "Idempotency-Key already used with a different request.")253254// Existing returns the action already created by this user with this key, if any.255func Existing(ctx context.Context, q Querier, user uuid.UUID, key string, hash []byte) (uuid.UUID, bool, error) {256 var id uuid.UUID257 var have []byte258 err := q.QueryRow(ctx, `select id, request_hash from actions where requested_by=$1 and idempotency_key=$2`, user, key).Scan(&id, &have)259 if errors.Is(err, pgx.ErrNoRows) {260 return uuid.Nil, false, nil261 }262 if err != nil {263 return uuid.Nil, false, err264 }265 if string(have) != string(hash) {266 return uuid.Nil, false, ErrIdempotencyConflict267 }268 return id, true, nil269}270271// Insert stores the action and takes the target lock, atomically, inside tx.272// created=false means the idempotency key already produced an action (its id273// is returned and nothing was written). A busy target is 409 target_locked;274// the caller must then roll the transaction back.275func Insert(ctx context.Context, tx pgx.Tx, r Request) (id uuid.UUID, created bool, err error) {276 if r.Params == nil {277 r.Params = map[string]any{}278 }279 if r.Now.IsZero() {280 r.Now = time.Now().UTC()281 }282 timeout := r.Type.TimeoutS283 if r.TimeoutS > 0 && r.TimeoutS < timeout {284 timeout = r.TimeoutS285 }286 state := "queued"287 var queuedAt *time.Time288 if r.RequestedBy != nil && r.Type.RequiresApproval() {289 state = "pending_approval"290 } else {291 queuedAt = &r.Now292 }293 var nodeID *uuid.UUID294 if r.Target.NodeID != nil {295 nodeID = r.Target.NodeID296 }297 id = ids.New()298 err = tx.QueryRow(ctx, `insert into actions(id, type, criticality, node_id, target_kind, target_id, target_label, params, state, requested_by,299 requested_via, justification, idempotency_key, request_hash, approval_required, created_at, queued_at, timeout_s, correlation_id, job_id, override_maintenance)300 values ($1,$2,$3,$4,$5,$6,$7,$8,$9::action_state,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21)301 on conflict (requested_by, idempotency_key) do nothing returning id`,302 id, r.Type.Name, r.Type.Criticality, nodeID, r.Target.Kind, r.Target.ID.String(), r.Target.Label, r.Params, state, r.RequestedBy,303 r.Via, r.Justification, r.IdempotencyKey, r.RequestHash, state == "pending_approval", r.Now, queuedAt, timeout, r.CorrelationID, r.JobID,304 r.OverrideMaintenance).Scan(&id)305 if errors.Is(err, pgx.ErrNoRows) {306 // A concurrent (or earlier) request with the same key won; it is committed by now.307 if r.RequestedBy == nil {308 return uuid.Nil, false, errors.New("actions: unexpected idempotency conflict for a system action")309 }310 ex, ok, err := Existing(ctx, tx, *r.RequestedBy, r.IdempotencyKey, r.RequestHash)311 if err != nil {312 return uuid.Nil, false, err313 }314 if !ok {315 return uuid.Nil, false, errors.New("actions: idempotent action vanished")316 }317 return ex, false, nil318 }319 if err != nil {320 return uuid.Nil, false, err321 }322 if key := LockKey(r.Type, r.Target); key != "" {323 // Defensive: a lock whose action is already final is stale (Finish removes locks in the same transaction).324 if _, err := tx.Exec(ctx, `delete from action_locks l using actions a where l.target_key=$1 and a.id=l.action_id325 and a.state in ('succeeded','failed','cancelled','timed_out','rejected','expired')`, key); err != nil {326 return uuid.Nil, false, err327 }328 tag, err := tx.Exec(ctx, `insert into action_locks(target_key, action_id, acquired_at) values ($1,$2,$3) on conflict do nothing`, key, id, r.Now)329 if err != nil {330 return uuid.Nil, false, err331 }332 if tag.RowsAffected() == 0 {333 var holder uuid.UUID334 if err := tx.QueryRow(ctx, `select action_id from action_locks where target_key=$1`, key).Scan(&holder); err != nil && !errors.Is(err, pgx.ErrNoRows) {335 return uuid.Nil, false, err336 }337 return uuid.Nil, false, httpx.Conflict("target_locked", "Another action is in progress on this target.").With("action_id", holder.String())338 }339 }340 if err := Event(ctx, tx, id, "state", "", state, r.Now); err != nil {341 return uuid.Nil, false, err342 }343 return id, true, Changed(ctx, tx, id, state == "queued")344}345346// Event appends a lifecycle event to an action.347func Event(ctx context.Context, tx Querier, id uuid.UUID, kind, stream, data string, at time.Time) error {348 _, err := tx.Exec(ctx, `insert into action_events(action_id, ts, kind, stream, data) values ($1,$2,$3,nullif($4,''),$5)`, id, at, kind, stream, data)349 return err350}351352// Changed publishes action.changed and, when wake is set, wakes the orchestrator.353func Changed(ctx context.Context, tx Querier, id uuid.UUID, wake bool) error {354 var typ, state, kind, target string355 var nodeID, by *uuid.UUID356 var unknown bool357 if err := tx.QueryRow(ctx, `select type, state::text, target_kind, target_id, node_id, requested_by, coalesce((result->>'outcome_unknown')::boolean, false)358 from actions where id=$1`, id).Scan(&typ, &state, &kind, &target, &nodeID, &by, &unknown); err != nil {359 return err360 }361 p := map[string]any{"action_id": id, "type": typ, "state": state, "target": map[string]any{"kind": kind, "id": target}, "outcome_unknown": unknown}362 if nodeID != nil {363 p["node_id"] = *nodeID364 }365 if by != nil {366 p["requested_by"] = *by367 }368 if err := events.Publish(ctx, tx, "action.changed", p); err != nil {369 return err370 }371 if wake {372 _, err := tx.Exec(ctx, `select pg_notify($1, $2)`, ChanActions, id.String())373 return err374 }375 return nil376}377378// Outcome is the final state of an action.379type Outcome struct {380 State string // succeeded | failed | cancelled | timed_out | rejected | expired381 ErrorCode string382 ErrorMessage string383 Result map[string]any384 // OutcomeUnknown: the command reached the agent but no result came back.385 // It may or may not have run; it is never sent again.386 OutcomeUnknown bool387 At time.Time388}389390// Finish moves a non-terminal action to its final state, releases its lock391// and publishes the change. It returns false when the action was already final.392func Finish(ctx context.Context, tx Querier, id uuid.UUID, o Outcome) (bool, error) {393 if !IsTerminal(o.State) {394 return false, fmt.Errorf("actions: %q is not a final state", o.State)395 }396 if o.At.IsZero() {397 o.At = time.Now().UTC()398 }399 res := map[string]any{}400 for k, v := range o.Result {401 res[k] = v402 }403 res["outcome_unknown"] = o.OutcomeUnknown404 tag, err := tx.Exec(ctx, `update actions set state=$2::action_state, finished_at=$3, result=$4, error_code=nullif($5,''), error_message=nullif($6,'')405 where id=$1 and state not in ('succeeded','failed','cancelled','timed_out','rejected','expired')`,406 id, o.State, o.At, res, o.ErrorCode, truncate(o.ErrorMessage, 2000))407 if err != nil || tag.RowsAffected() == 0 {408 return false, err409 }410 if _, err := tx.Exec(ctx, `delete from action_locks where action_id=$1`, id); err != nil {411 return false, err412 }413 data := o.State414 if o.ErrorCode != "" {415 data += " (" + o.ErrorCode + ")"416 }417 if err := Event(ctx, tx, id, "state", "", data, o.At); err != nil {418 return false, err419 }420 return true, Changed(ctx, tx, id, true)421}422423// Transition moves an action from one non-terminal state to another.424func Transition(ctx context.Context, tx Querier, id uuid.UUID, from, to string, at time.Time) (bool, error) {425 tag, err := tx.Exec(ctx, `update actions set state=$3::action_state,426 queued_at = case when $3='queued' then $4 else queued_at end,427 started_at = case when $3='running' then $4 else started_at end428 where id=$1 and state=$2::action_state`, id, from, to, at)429 if err != nil || tag.RowsAffected() == 0 {430 return false, err431 }432 if err := Event(ctx, tx, id, "state", "", to, at); err != nil {433 return false, err434 }435 return true, Changed(ctx, tx, id, to == "queued")436}437438// ErrFinished is answered when a finished action is cancelled, approved or rejected.439var ErrFinished = httpx.Conflict("conflict", "The action is already finished.")440441// RequestCancel cancels an action. Before the command leaves the server the442// cancellation is immediate and certain. Once the command may have reached443// the agent it is only a request: the agent decides, and its result decides444// the final state.445func RequestCancel(ctx context.Context, tx Querier, id uuid.UUID, at time.Time) (state string, err error) {446 if err := tx.QueryRow(ctx, `select state::text from actions where id=$1 for update`, id).Scan(&state); err != nil {447 if errors.Is(err, pgx.ErrNoRows) {448 return "", httpx.ErrNotFound449 }450 return "", err451 }452 switch state {453 case "pending_approval", "queued":454 _, err := Finish(ctx, tx, id, Outcome{State: "cancelled", At: at})455 return "cancelled", err456 case "dispatched", "running":457 // Never delivered: withdraw the command, the action is cancelled for certain.458 tag, err := tx.Exec(ctx, `update agent_commands set state='cancelled', finished_at=$2, cancel_requested_at=coalesce(cancel_requested_at,$2)459 where action_id=$1 and state='pending'`, id, at)460 if err != nil {461 return "", err462 }463 if tag.RowsAffected() == 1 {464 _, err := Finish(ctx, tx, id, Outcome{State: "cancelled", At: at})465 return "cancelled", err466 }467 if _, err := tx.Exec(ctx, `update actions set cancel_requested_at=coalesce(cancel_requested_at,$2) where id=$1`, id, at); err != nil {468 return "", err469 }470 var agent *uuid.UUID471 err = tx.QueryRow(ctx, `update agent_commands set cancel_requested_at=coalesce(cancel_requested_at,$2)472 where action_id=$1 and state in ('delivered','acked') returning agent_id`, id, at).Scan(&agent)473 if err != nil && !errors.Is(err, pgx.ErrNoRows) {474 return "", err475 }476 if agent != nil { // wake the transport so the agent learns about the cancellation477 if _, err := tx.Exec(ctx, `select pg_notify($1,$2)`, ChanCommands, agent.String()); err != nil {478 return "", err479 }480 }481 if err := Event(ctx, tx, id, "note", "", "cancellation requested", at); err != nil {482 return "", err483 }484 return state, Changed(ctx, tx, id, true)485 }486 return "", ErrFinished487}488489// CancelJob cancels a job: at once when it is not running, cooperatively490// (through its job.run action) when it is. It returns the resulting job state.491func CancelJob(ctx context.Context, tx Querier, jobID uuid.UUID, at time.Time) (string, error) {492 var state, tpl string493 var nodeID *uuid.UUID494 if err := tx.QueryRow(ctx, `select state::text, template, node_id from jobs where id=$1 for update`, jobID).Scan(&state, &tpl, &nodeID); err != nil {495 if errors.Is(err, pgx.ErrNoRows) {496 return "", httpx.ErrNotFound497 }498 return "", err499 }500 switch state {501 case "queued":502 if _, err := tx.Exec(ctx, `update jobs set state='cancelled', finished_at=$2, cancel_requested_at=$2, next_attempt_at=null, error='cancelled before start' where id=$1`, jobID, at); err != nil {503 return "", err504 }505 return "cancelled", JobChanged(ctx, tx, jobID)506 case "running":507 if _, err := tx.Exec(ctx, `update jobs set cancel_requested_at=coalesce(cancel_requested_at,$2) where id=$1`, jobID, at); err != nil {508 return "", err509 }510 var actionID uuid.UUID511 err := tx.QueryRow(ctx, `select id from actions where job_id=$1 and type='job.run'512 and state in ('queued','dispatched','running') order by created_at desc limit 1`, jobID).Scan(&actionID)513 if errors.Is(err, pgx.ErrNoRows) {514 return "running", nil // the runner will observe cancel_requested_at515 }516 if err != nil {517 return "", err518 }519 if _, err := RequestCancel(ctx, tx, actionID, at); err != nil && !errors.Is(err, ErrFinished) {520 return "", err521 }522 return "running", nil523 }524 return "", httpx.Conflict("conflict", "The job is already finished.")525}526527// JobChanged publishes job.changed.528func JobChanged(ctx context.Context, tx Querier, jobID uuid.UUID) error {529 var tpl, state string530 var nodeID *uuid.UUID531 var progress *float32532 if err := tx.QueryRow(ctx, `select template, state::text, node_id, progress from jobs where id=$1`, jobID).Scan(&tpl, &state, &nodeID, &progress); err != nil {533 return err534 }535 p := map[string]any{"job_id": jobID, "template": tpl, "state": state}536 if nodeID != nil {537 p["node_id"] = *nodeID538 }539 if progress != nil {540 p["progress"] = *progress541 }542 return events.Publish(ctx, tx, "job.changed", p)543}544545// ExecuteServerSide performs a ServerSide action inside tx and returns its result.546func ExecuteServerSide(ctx context.Context, tx Querier, typ string, tg *Target, params map[string]any, at time.Time) (map[string]any, error) {547 switch typ {548 case "node.maintenance.set":549 enabled, _ := params["enabled"].(bool)550 reason, _ := params["reason"].(string)551 if !enabled {552 reason = ""553 }554 var before bool555 if err := tx.QueryRow(ctx, `select maintenance from nodes where id=$1 for update`, tg.ID).Scan(&before); err != nil {556 return nil, err557 }558 if _, err := tx.Exec(ctx, `update nodes set maintenance=$2, maintenance_reason=$3 where id=$1`, tg.ID, enabled, reason); err != nil {559 return nil, err560 }561 if err := events.Publish(ctx, tx, "node.updated", map[string]any{"node_id": tg.ID, "change": "updated"}); err != nil {562 return nil, err563 }564 return map[string]any{"maintenance": enabled, "previous": before}, nil565 case "job.cancel":566 state, err := CancelJob(ctx, tx, tg.ID, at)567 if err != nil {568 return nil, err569 }570 return map[string]any{"job_state": state}, nil571 }572 return nil, fmt.Errorf("actions: %s is not a server-side type", typ)573}574575func truncate(s string, n int) string {576 if len(s) <= n {577 return s578 }579 for n > 0 && s[n]&0xC0 == 0x80 { // do not cut a UTF-8 sequence580 n--581 }582 return s[:n]583}584