Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/ingest/commands.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package ingest4344import (45 "context"46 "encoding/json"47 "errors"48 "net/http"49 "strconv"50 "strings"51 "sync"52 "time"5354 "github.com/google/uuid"55 "github.com/jackc/pgx/v5"5657 "clustr.spboucher.ai/mcc/internal/events"58 "clustr.spboucher.ai/mcc/internal/httpx"59 "clustr.spboucher.ai/mcc/internal/protocol"60 "clustr.spboucher.ai/mcc/internal/redact"61)6263// Channels of the database contract (see migration 0002).64const (65 chanCommands = "mcc_commands" // orchestrator -> ingest, payload = agent id66 chanCommandUpdates = "mcc_command_updates" // ingest -> orchestrator, payload = command id67)6869// Server-side bounds on the output of one command.70const (71 MaxOutputLines = 200072 MaxOutputBytes = 512 << 1073 maxOutputLineLen = 8 << 1074 truncationMarker = "[output truncated by the server: limit reached]"75)7677// waker wakes the long-polls of an agent when a notification names it.78type waker struct {79 mu sync.Mutex80 subs map[uuid.UUID]map[chan struct{}]struct{}81}8283func newWaker() *waker { return &waker{subs: map[uuid.UUID]map[chan struct{}]struct{}{}} }8485func (w *waker) subscribe(id uuid.UUID) (<-chan struct{}, func()) {86 ch := make(chan struct{}, 1)87 w.mu.Lock()88 if w.subs[id] == nil {89 w.subs[id] = map[chan struct{}]struct{}{}90 }91 w.subs[id][ch] = struct{}{}92 w.mu.Unlock()93 return ch, func() {94 w.mu.Lock()95 delete(w.subs[id], ch)96 if len(w.subs[id]) == 0 {97 delete(w.subs, id)98 }99 w.mu.Unlock()100 }101}102103func (w *waker) wake(id uuid.UUID) {104 w.mu.Lock()105 for ch := range w.subs[id] {106 select {107 case ch <- struct{}{}:108 default:109 }110 }111 w.mu.Unlock()112}113114func (w *waker) wakeAll() {115 w.mu.Lock()116 for _, m := range w.subs {117 for ch := range m {118 select {119 case ch <- struct{}{}:120 default:121 }122 }123 }124 w.mu.Unlock()125}126127// listenCommands holds a LISTEN connection and wakes long-polls. While the128// connection is down, long-polls fall back to periodic polling, so a database129// blip delays a command but never loses it.130func (s *Server) listenCommands(ctx context.Context) {131 backoff := time.Second132 for ctx.Err() == nil {133 err := func() error {134 conn, err := s.pool.Acquire(ctx)135 if err != nil {136 return err137 }138 defer conn.Release()139 if _, err := conn.Exec(ctx, "listen "+chanCommands); err != nil {140 return err141 }142 backoff = time.Second143 s.wake.wakeAll() // anything queued while we were not listening144 for {145 n, err := conn.Conn().WaitForNotification(ctx)146 if err != nil {147 // Do not return a broken connection to the pool.148 conn.Conn().Close(context.Background())149 return err150 }151 if id, err := uuid.Parse(strings.TrimSpace(n.Payload)); err == nil {152 s.wake.wake(id)153 } else {154 s.wake.wakeAll()155 }156 }157 }()158 if ctx.Err() != nil {159 return160 }161 s.log.Warn("ingest: command listener disconnected", "err", err, "retry_in", backoff)162 select {163 case <-ctx.Done():164 return165 case <-time.After(backoff):166 }167 if backoff < 15*time.Second {168 backoff *= 2169 }170 }171}172173// takeCommands marks the agent's pending commands delivered and returns174// them, with the ids of commands whose cancellation was requested. A command175// is handed out exactly once: there is no redelivery (design brief §4).176func (s *Server) takeCommands(ctx context.Context, agentID uuid.UUID) (*protocol.CommandsResponse, error) {177 resp := &protocol.CommandsResponse{Commands: []protocol.SignedCommand{}}178 var delivered []uuid.UUID179 err := s.inTx(ctx, func(tx pgx.Tx) error {180 rows, err := tx.Query(ctx, `update agent_commands set state = 'delivered', delivered_at = now()181 where id in (select id from agent_commands where agent_id = $1 and state = 'pending' and expires_at > now()182 order by issued_at limit 8 for update skip locked)183 returning id, envelope, signature, key_id`, agentID)184 if err != nil {185 return err186 }187 for rows.Next() {188 var id uuid.UUID189 var sc protocol.SignedCommand190 if err := rows.Scan(&id, &sc.Payload, &sc.Signature, &sc.KeyID); err != nil {191 rows.Close()192 return err193 }194 resp.Commands = append(resp.Commands, sc)195 delivered = append(delivered, id)196 }197 rows.Close()198 if err := rows.Err(); err != nil {199 return err200 }201 crow, err := tx.Query(ctx, `update agent_commands set cancel_sent_at = now()202 where agent_id = $1 and cancel_requested_at is not null and cancel_sent_at is null and state in ('delivered','acked')203 returning id::text`, agentID)204 if err != nil {205 return err206 }207 resp.Cancel, err = pgx.CollectRows(crow, pgx.RowTo[string])208 if err != nil {209 return err210 }211 for _, id := range delivered {212 if _, err := tx.Exec(ctx, `select pg_notify($1, $2)`, chanCommandUpdates, id.String()); err != nil {213 return err214 }215 }216 return nil217 })218 if err != nil {219 return nil, err220 }221 return resp, nil222}223224// handleCommands is the long-poll: it answers at once when something is225// waiting, otherwise when notified, and with an empty list at the deadline.226func (s *Server) handleCommands(w http.ResponseWriter, r *http.Request) {227 a := agentFrom(r.Context())228 wait := 25 * time.Second229 if v := r.URL.Query().Get("wait"); v != "" {230 n, err := strconv.Atoi(v)231 if err != nil || n < 0 {232 httpx.WriteError(w, r, httpx.Validation("wait must be a number of seconds."))233 return234 }235 wait = min(time.Duration(n)*time.Second, s.maxLongPoll)236 }237 wakeCh, unsub := s.wake.subscribe(a.ID)238 defer unsub()239 s.longPolls.Add(1)240 defer s.longPolls.Add(-1)241 deadline := time.NewTimer(wait)242 defer deadline.Stop()243 poll := time.NewTicker(5 * time.Second) // safety net for a missed notification244 defer poll.Stop()245 recheck := time.NewTicker(s.authTTL + time.Second)246 defer recheck.Stop()247 for {248 ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)249 resp, err := s.takeCommands(ctx, a.ID)250 cancel()251 if err != nil {252 s.writeErr(w, r, err)253 return254 }255 if len(resp.Commands) > 0 || len(resp.Cancel) > 0 {256 writeJSON(w, http.StatusOK, resp)257 return258 }259 select {260 case <-wakeCh:261 case <-poll.C:262 case <-recheck.C:263 // A revoked agent must not keep a long-poll open.264 ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)265 cur, err := s.lookupAgent(ctx, leafFrom(r.Context()))266 cancel()267 if err == nil && (cur == nil || cur.ID != a.ID) {268 httpx.WriteError(w, r, errAgentRevoked)269 return270 }271 case <-deadline.C:272 writeJSON(w, http.StatusOK, resp)273 return274 case <-s.closing:275 writeJSON(w, http.StatusOK, resp)276 return277 case <-r.Context().Done():278 return279 }280 }281}282283var (284 errCommandNotFound = httpx.Err(http.StatusNotFound, "command_not_found", "No such command for this agent.")285 errCommandState = httpx.Conflict("command_state", "The command is not in a state that accepts this message.")286)287288func commandID(r *http.Request) (uuid.UUID, error) {289 id, err := uuid.Parse(r.PathValue("id"))290 if err != nil {291 return uuid.Nil, errCommandNotFound292 }293 return id, nil294}295296// lockCommand locks a command row, but only if it is addressed to agentID:297// a command of another agent is indistinguishable from a missing one.298func lockCommand(ctx context.Context, tx pgx.Tx, id, agentID uuid.UUID) (state string, actionID uuid.UUID, err error) {299 err = tx.QueryRow(ctx, `select state, action_id from agent_commands where id = $1 and agent_id = $2 for update`, id, agentID).Scan(&state, &actionID)300 if errors.Is(err, pgx.ErrNoRows) {301 return "", uuid.Nil, errCommandNotFound302 }303 return state, actionID, err304}305306func notifyUpdate(ctx context.Context, tx pgx.Tx, id uuid.UUID) error {307 _, err := tx.Exec(ctx, `select pg_notify($1, $2)`, chanCommandUpdates, id.String())308 return err309}310311// handleCommandAck records the agent's decision: accepted (state acked) or312// refused after verification / local policy (state finished).313func (s *Server) handleCommandAck(w http.ResponseWriter, r *http.Request) {314 a := agentFrom(r.Context())315 id, err := commandID(r)316 if err != nil {317 httpx.WriteError(w, r, err)318 return319 }320 var ack protocol.CommandAck321 if err := decodeBody(w, r, 16<<10, &ack); err != nil {322 httpx.WriteError(w, r, err)323 return324 }325 reason := redact.String(clip(ack.Reason, 500))326 ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)327 defer cancel()328 err = s.inTx(ctx, func(tx pgx.Tx) error {329 state, _, err := lockCommand(ctx, tx, id, a.ID)330 if err != nil {331 return err332 }333 if state != "delivered" {334 return errCommandState335 }336 if ack.Accepted {337 _, err = tx.Exec(ctx, `update agent_commands set state = 'acked', acked_at = now(), ack_accepted = true, ack_reason = nullif($2,'') where id = $1`, id, reason)338 } else {339 _, err = tx.Exec(ctx, `update agent_commands set state = 'finished', acked_at = now(), finished_at = now(), ack_accepted = false, ack_reason = $2 where id = $1`, id, reason)340 }341 if err != nil {342 return err343 }344 return notifyUpdate(ctx, tx, id)345 })346 if err != nil {347 s.writeErr(w, r, err)348 return349 }350 w.WriteHeader(http.StatusNoContent)351}352353// decodeOutputs accepts one CommandOutput or an array of them.354func decodeOutputs(raw json.RawMessage) ([]protocol.CommandOutput, error) {355 trim := strings.TrimLeft(string(raw), " \t\r\n")356 if strings.HasPrefix(trim, "[") {357 var out []protocol.CommandOutput358 err := json.Unmarshal(raw, &out)359 return out, err360 }361 var one protocol.CommandOutput362 if err := json.Unmarshal(raw, &one); err != nil {363 return nil, err364 }365 return []protocol.CommandOutput{one}, nil366}367368type outputLine struct {369 ID int64 `json:"id"`370 Stream string `json:"stream"`371 Data string `json:"data"`372}373374// handleCommandEvents stores output lines after a second redaction pass,375// within the per-command bounds, and publishes them as action.output.376func (s *Server) handleCommandEvents(w http.ResponseWriter, r *http.Request) {377 a := agentFrom(r.Context())378 id, err := commandID(r)379 if err != nil {380 httpx.WriteError(w, r, err)381 return382 }383 var raw json.RawMessage384 if err := decodeBody(w, r, 1<<20, &raw); err != nil {385 httpx.WriteError(w, r, err)386 return387 }388 outs, err := decodeOutputs(raw)389 if err != nil {390 httpx.WriteError(w, r, httpx.Validation("Body must be a command output or an array of them."))391 return392 }393 type line struct{ stream, data string }394 var lines []line395 for _, o := range outs {396 switch o.Stream {397 case "stdout", "stderr", "info":398 default:399 httpx.WriteError(w, r, httpx.Validation("stream must be stdout, stderr or info."))400 return401 }402 data := strings.ToValidUTF8(o.Data, "�")403 for _, l := range strings.Split(strings.TrimRight(data, "\n"), "\n") {404 l = strings.TrimRight(l, "\r")405 if len(l) > maxOutputLineLen {406 l = strings.ToValidUTF8(l[:maxOutputLineLen], "") + "…"407 }408 l = strings.Map(func(c rune) rune {409 if c == '\t' || c >= 0x20 && c != 0x7f {410 return c411 }412 return -1413 }, l)414 lines = append(lines, line{o.Stream, redact.String(l)})415 if len(lines) > MaxOutputLines+1 {416 break417 }418 }419 }420 ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)421 defer cancel()422 err = s.inTx(ctx, func(tx pgx.Tx) error {423 state, actionID, err := lockCommand(ctx, tx, id, a.ID)424 if err != nil {425 return err426 }427 if state != "acked" {428 return errCommandState429 }430 var nLines, nBytes int431 var truncated bool432 if err := tx.QueryRow(ctx, `select output_lines, output_bytes, output_truncated from agent_commands where id = $1`, id).Scan(&nLines, &nBytes, &truncated); err != nil {433 return err434 }435 if truncated {436 return nil // silently dropped: the marker is already stored437 }438 var stored []outputLine439 insert := func(stream, data string) error {440 var evID int64441 if err := tx.QueryRow(ctx, `insert into action_events(action_id, kind, stream, data) values ($1,'output',$2,$3) returning id`, actionID, stream, data).Scan(&evID); err != nil {442 return err443 }444 stored = append(stored, outputLine{evID, stream, data})445 return nil446 }447 for _, l := range lines {448 if nLines+1 > MaxOutputLines || nBytes+len(l.data) > MaxOutputBytes {449 truncated = true450 if err := insert("info", truncationMarker); err != nil {451 return err452 }453 break454 }455 if err := insert(l.stream, l.data); err != nil {456 return err457 }458 nLines++459 nBytes += len(l.data)460 }461 if _, err := tx.Exec(ctx, `update agent_commands set output_lines = $2, output_bytes = $3, output_truncated = $4 where id = $1`, id, nLines, nBytes, truncated); err != nil {462 return err463 }464 if len(stored) == 0 {465 return nil466 }467 if err := events.Publish(ctx, tx, "action.output", map[string]any{468 "action_id": actionID, "command_id": id, "node_id": a.NodeID, "lines": stored, "truncated": truncated,469 }); err != nil {470 return err471 }472 return notifyUpdate(ctx, tx, id)473 })474 if err != nil {475 s.writeErr(w, r, err)476 return477 }478 w.WriteHeader(http.StatusNoContent)479}480481var resultStatuses = map[string]bool{"succeeded": true, "failed": true, "timed_out": true, "cancelled": true}482483// handleCommandResult closes a command. The orchestrator, woken by the484// notification, is the only writer of the actions table.485func (s *Server) handleCommandResult(w http.ResponseWriter, r *http.Request) {486 a := agentFrom(r.Context())487 id, err := commandID(r)488 if err != nil {489 httpx.WriteError(w, r, err)490 return491 }492 var res protocol.CommandResult493 if err := decodeBody(w, r, 256<<10, &res); err != nil {494 httpx.WriteError(w, r, err)495 return496 }497 if !resultStatuses[res.Status] {498 httpx.WriteError(w, r, httpx.Validation("status must be succeeded, failed, timed_out or cancelled."))499 return500 }501 if !finite(res.Duration) || res.Duration < 0 {502 httpx.WriteError(w, r, httpx.Validation("duration_s is invalid."))503 return504 }505 res.Error = redact.String(clip(res.Error, 2000))506 if res.Result != nil {507 res.Result, _ = redact.Value(res.Result).(map[string]any)508 }509 doc, err := json.Marshal(res)510 if err != nil || len(doc) > 256<<10 {511 httpx.WriteError(w, r, httpx.Validation("result is not serializable or too large."))512 return513 }514 ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)515 defer cancel()516 err = s.inTx(ctx, func(tx pgx.Tx) error {517 state, _, err := lockCommand(ctx, tx, id, a.ID)518 if err != nil {519 return err520 }521 // 'delivered' is accepted too: after a crash the agent reports commands522 // it had started without having been able to acknowledge them.523 if state != "acked" && state != "delivered" {524 return errCommandState525 }526 if _, err := tx.Exec(ctx, `update agent_commands set state = 'finished', finished_at = now(), result_status = $2, result = $3::jsonb,527 acked_at = coalesce(acked_at, now()), ack_accepted = coalesce(ack_accepted, true)528 where id = $1`, id, res.Status, doc); err != nil {529 return err530 }531 return notifyUpdate(ctx, tx, id)532 })533 if err != nil {534 s.writeErr(w, r, err)535 return536 }537 w.WriteHeader(http.StatusNoContent)538}539