SPB Git

Repository Public

commands.go 16.0 KB 539 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/ingest/commands.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package ingest4344import (45	"context"46	"encoding/json"47	"errors"48	"net/http"49	"strconv"50	"strings"51	"sync"52	"time"5354	"github.com/google/uuid"55	"github.com/jackc/pgx/v5"5657	"clustr.spboucher.ai/mcc/internal/events"58	"clustr.spboucher.ai/mcc/internal/httpx"59	"clustr.spboucher.ai/mcc/internal/protocol"60	"clustr.spboucher.ai/mcc/internal/redact"61)6263// Channels of the database contract (see migration 0002).64const (65	chanCommands       = "mcc_commands"        // orchestrator -> ingest, payload = agent id66	chanCommandUpdates = "mcc_command_updates" // ingest -> orchestrator, payload = command id67)6869// Server-side bounds on the output of one command.70const (71	MaxOutputLines   = 200072	MaxOutputBytes   = 512 << 1073	maxOutputLineLen = 8 << 1074	truncationMarker = "[output truncated by the server: limit reached]"75)7677// waker wakes the long-polls of an agent when a notification names it.78type waker struct {79	mu   sync.Mutex80	subs map[uuid.UUID]map[chan struct{}]struct{}81}8283func newWaker() *waker { return &waker{subs: map[uuid.UUID]map[chan struct{}]struct{}{}} }8485func (w *waker) subscribe(id uuid.UUID) (<-chan struct{}, func()) {86	ch := make(chan struct{}, 1)87	w.mu.Lock()88	if w.subs[id] == nil {89		w.subs[id] = map[chan struct{}]struct{}{}90	}91	w.subs[id][ch] = struct{}{}92	w.mu.Unlock()93	return ch, func() {94		w.mu.Lock()95		delete(w.subs[id], ch)96		if len(w.subs[id]) == 0 {97			delete(w.subs, id)98		}99		w.mu.Unlock()100	}101}102103func (w *waker) wake(id uuid.UUID) {104	w.mu.Lock()105	for ch := range w.subs[id] {106		select {107		case ch <- struct{}{}:108		default:109		}110	}111	w.mu.Unlock()112}113114func (w *waker) wakeAll() {115	w.mu.Lock()116	for _, m := range w.subs {117		for ch := range m {118			select {119			case ch <- struct{}{}:120			default:121			}122		}123	}124	w.mu.Unlock()125}126127// listenCommands holds a LISTEN connection and wakes long-polls. While the128// connection is down, long-polls fall back to periodic polling, so a database129// blip delays a command but never loses it.130func (s *Server) listenCommands(ctx context.Context) {131	backoff := time.Second132	for ctx.Err() == nil {133		err := func() error {134			conn, err := s.pool.Acquire(ctx)135			if err != nil {136				return err137			}138			defer conn.Release()139			if _, err := conn.Exec(ctx, "listen "+chanCommands); err != nil {140				return err141			}142			backoff = time.Second143			s.wake.wakeAll() // anything queued while we were not listening144			for {145				n, err := conn.Conn().WaitForNotification(ctx)146				if err != nil {147					// Do not return a broken connection to the pool.148					conn.Conn().Close(context.Background())149					return err150				}151				if id, err := uuid.Parse(strings.TrimSpace(n.Payload)); err == nil {152					s.wake.wake(id)153				} else {154					s.wake.wakeAll()155				}156			}157		}()158		if ctx.Err() != nil {159			return160		}161		s.log.Warn("ingest: command listener disconnected", "err", err, "retry_in", backoff)162		select {163		case <-ctx.Done():164			return165		case <-time.After(backoff):166		}167		if backoff < 15*time.Second {168			backoff *= 2169		}170	}171}172173// takeCommands marks the agent's pending commands delivered and returns174// them, with the ids of commands whose cancellation was requested. A command175// is handed out exactly once: there is no redelivery (design brief §4).176func (s *Server) takeCommands(ctx context.Context, agentID uuid.UUID) (*protocol.CommandsResponse, error) {177	resp := &protocol.CommandsResponse{Commands: []protocol.SignedCommand{}}178	var delivered []uuid.UUID179	err := s.inTx(ctx, func(tx pgx.Tx) error {180		rows, err := tx.Query(ctx, `update agent_commands set state = 'delivered', delivered_at = now()181			where id in (select id from agent_commands where agent_id = $1 and state = 'pending' and expires_at > now()182			             order by issued_at limit 8 for update skip locked)183			returning id, envelope, signature, key_id`, agentID)184		if err != nil {185			return err186		}187		for rows.Next() {188			var id uuid.UUID189			var sc protocol.SignedCommand190			if err := rows.Scan(&id, &sc.Payload, &sc.Signature, &sc.KeyID); err != nil {191				rows.Close()192				return err193			}194			resp.Commands = append(resp.Commands, sc)195			delivered = append(delivered, id)196		}197		rows.Close()198		if err := rows.Err(); err != nil {199			return err200		}201		crow, err := tx.Query(ctx, `update agent_commands set cancel_sent_at = now()202			where agent_id = $1 and cancel_requested_at is not null and cancel_sent_at is null and state in ('delivered','acked')203			returning id::text`, agentID)204		if err != nil {205			return err206		}207		resp.Cancel, err = pgx.CollectRows(crow, pgx.RowTo[string])208		if err != nil {209			return err210		}211		for _, id := range delivered {212			if _, err := tx.Exec(ctx, `select pg_notify($1, $2)`, chanCommandUpdates, id.String()); err != nil {213				return err214			}215		}216		return nil217	})218	if err != nil {219		return nil, err220	}221	return resp, nil222}223224// handleCommands is the long-poll: it answers at once when something is225// waiting, otherwise when notified, and with an empty list at the deadline.226func (s *Server) handleCommands(w http.ResponseWriter, r *http.Request) {227	a := agentFrom(r.Context())228	wait := 25 * time.Second229	if v := r.URL.Query().Get("wait"); v != "" {230		n, err := strconv.Atoi(v)231		if err != nil || n < 0 {232			httpx.WriteError(w, r, httpx.Validation("wait must be a number of seconds."))233			return234		}235		wait = min(time.Duration(n)*time.Second, s.maxLongPoll)236	}237	wakeCh, unsub := s.wake.subscribe(a.ID)238	defer unsub()239	s.longPolls.Add(1)240	defer s.longPolls.Add(-1)241	deadline := time.NewTimer(wait)242	defer deadline.Stop()243	poll := time.NewTicker(5 * time.Second) // safety net for a missed notification244	defer poll.Stop()245	recheck := time.NewTicker(s.authTTL + time.Second)246	defer recheck.Stop()247	for {248		ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)249		resp, err := s.takeCommands(ctx, a.ID)250		cancel()251		if err != nil {252			s.writeErr(w, r, err)253			return254		}255		if len(resp.Commands) > 0 || len(resp.Cancel) > 0 {256			writeJSON(w, http.StatusOK, resp)257			return258		}259		select {260		case <-wakeCh:261		case <-poll.C:262		case <-recheck.C:263			// A revoked agent must not keep a long-poll open.264			ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)265			cur, err := s.lookupAgent(ctx, leafFrom(r.Context()))266			cancel()267			if err == nil && (cur == nil || cur.ID != a.ID) {268				httpx.WriteError(w, r, errAgentRevoked)269				return270			}271		case <-deadline.C:272			writeJSON(w, http.StatusOK, resp)273			return274		case <-s.closing:275			writeJSON(w, http.StatusOK, resp)276			return277		case <-r.Context().Done():278			return279		}280	}281}282283var (284	errCommandNotFound = httpx.Err(http.StatusNotFound, "command_not_found", "No such command for this agent.")285	errCommandState    = httpx.Conflict("command_state", "The command is not in a state that accepts this message.")286)287288func commandID(r *http.Request) (uuid.UUID, error) {289	id, err := uuid.Parse(r.PathValue("id"))290	if err != nil {291		return uuid.Nil, errCommandNotFound292	}293	return id, nil294}295296// lockCommand locks a command row, but only if it is addressed to agentID:297// a command of another agent is indistinguishable from a missing one.298func lockCommand(ctx context.Context, tx pgx.Tx, id, agentID uuid.UUID) (state string, actionID uuid.UUID, err error) {299	err = tx.QueryRow(ctx, `select state, action_id from agent_commands where id = $1 and agent_id = $2 for update`, id, agentID).Scan(&state, &actionID)300	if errors.Is(err, pgx.ErrNoRows) {301		return "", uuid.Nil, errCommandNotFound302	}303	return state, actionID, err304}305306func notifyUpdate(ctx context.Context, tx pgx.Tx, id uuid.UUID) error {307	_, err := tx.Exec(ctx, `select pg_notify($1, $2)`, chanCommandUpdates, id.String())308	return err309}310311// handleCommandAck records the agent's decision: accepted (state acked) or312// refused after verification / local policy (state finished).313func (s *Server) handleCommandAck(w http.ResponseWriter, r *http.Request) {314	a := agentFrom(r.Context())315	id, err := commandID(r)316	if err != nil {317		httpx.WriteError(w, r, err)318		return319	}320	var ack protocol.CommandAck321	if err := decodeBody(w, r, 16<<10, &ack); err != nil {322		httpx.WriteError(w, r, err)323		return324	}325	reason := redact.String(clip(ack.Reason, 500))326	ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)327	defer cancel()328	err = s.inTx(ctx, func(tx pgx.Tx) error {329		state, _, err := lockCommand(ctx, tx, id, a.ID)330		if err != nil {331			return err332		}333		if state != "delivered" {334			return errCommandState335		}336		if ack.Accepted {337			_, err = tx.Exec(ctx, `update agent_commands set state = 'acked', acked_at = now(), ack_accepted = true, ack_reason = nullif($2,'') where id = $1`, id, reason)338		} else {339			_, err = tx.Exec(ctx, `update agent_commands set state = 'finished', acked_at = now(), finished_at = now(), ack_accepted = false, ack_reason = $2 where id = $1`, id, reason)340		}341		if err != nil {342			return err343		}344		return notifyUpdate(ctx, tx, id)345	})346	if err != nil {347		s.writeErr(w, r, err)348		return349	}350	w.WriteHeader(http.StatusNoContent)351}352353// decodeOutputs accepts one CommandOutput or an array of them.354func decodeOutputs(raw json.RawMessage) ([]protocol.CommandOutput, error) {355	trim := strings.TrimLeft(string(raw), " \t\r\n")356	if strings.HasPrefix(trim, "[") {357		var out []protocol.CommandOutput358		err := json.Unmarshal(raw, &out)359		return out, err360	}361	var one protocol.CommandOutput362	if err := json.Unmarshal(raw, &one); err != nil {363		return nil, err364	}365	return []protocol.CommandOutput{one}, nil366}367368type outputLine struct {369	ID     int64  `json:"id"`370	Stream string `json:"stream"`371	Data   string `json:"data"`372}373374// handleCommandEvents stores output lines after a second redaction pass,375// within the per-command bounds, and publishes them as action.output.376func (s *Server) handleCommandEvents(w http.ResponseWriter, r *http.Request) {377	a := agentFrom(r.Context())378	id, err := commandID(r)379	if err != nil {380		httpx.WriteError(w, r, err)381		return382	}383	var raw json.RawMessage384	if err := decodeBody(w, r, 1<<20, &raw); err != nil {385		httpx.WriteError(w, r, err)386		return387	}388	outs, err := decodeOutputs(raw)389	if err != nil {390		httpx.WriteError(w, r, httpx.Validation("Body must be a command output or an array of them."))391		return392	}393	type line struct{ stream, data string }394	var lines []line395	for _, o := range outs {396		switch o.Stream {397		case "stdout", "stderr", "info":398		default:399			httpx.WriteError(w, r, httpx.Validation("stream must be stdout, stderr or info."))400			return401		}402		data := strings.ToValidUTF8(o.Data, "�")403		for _, l := range strings.Split(strings.TrimRight(data, "\n"), "\n") {404			l = strings.TrimRight(l, "\r")405			if len(l) > maxOutputLineLen {406				l = strings.ToValidUTF8(l[:maxOutputLineLen], "") + "…"407			}408			l = strings.Map(func(c rune) rune {409				if c == '\t' || c >= 0x20 && c != 0x7f {410					return c411				}412				return -1413			}, l)414			lines = append(lines, line{o.Stream, redact.String(l)})415			if len(lines) > MaxOutputLines+1 {416				break417			}418		}419	}420	ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)421	defer cancel()422	err = s.inTx(ctx, func(tx pgx.Tx) error {423		state, actionID, err := lockCommand(ctx, tx, id, a.ID)424		if err != nil {425			return err426		}427		if state != "acked" {428			return errCommandState429		}430		var nLines, nBytes int431		var truncated bool432		if err := tx.QueryRow(ctx, `select output_lines, output_bytes, output_truncated from agent_commands where id = $1`, id).Scan(&nLines, &nBytes, &truncated); err != nil {433			return err434		}435		if truncated {436			return nil // silently dropped: the marker is already stored437		}438		var stored []outputLine439		insert := func(stream, data string) error {440			var evID int64441			if err := tx.QueryRow(ctx, `insert into action_events(action_id, kind, stream, data) values ($1,'output',$2,$3) returning id`, actionID, stream, data).Scan(&evID); err != nil {442				return err443			}444			stored = append(stored, outputLine{evID, stream, data})445			return nil446		}447		for _, l := range lines {448			if nLines+1 > MaxOutputLines || nBytes+len(l.data) > MaxOutputBytes {449				truncated = true450				if err := insert("info", truncationMarker); err != nil {451					return err452				}453				break454			}455			if err := insert(l.stream, l.data); err != nil {456				return err457			}458			nLines++459			nBytes += len(l.data)460		}461		if _, err := tx.Exec(ctx, `update agent_commands set output_lines = $2, output_bytes = $3, output_truncated = $4 where id = $1`, id, nLines, nBytes, truncated); err != nil {462			return err463		}464		if len(stored) == 0 {465			return nil466		}467		if err := events.Publish(ctx, tx, "action.output", map[string]any{468			"action_id": actionID, "command_id": id, "node_id": a.NodeID, "lines": stored, "truncated": truncated,469		}); err != nil {470			return err471		}472		return notifyUpdate(ctx, tx, id)473	})474	if err != nil {475		s.writeErr(w, r, err)476		return477	}478	w.WriteHeader(http.StatusNoContent)479}480481var resultStatuses = map[string]bool{"succeeded": true, "failed": true, "timed_out": true, "cancelled": true}482483// handleCommandResult closes a command. The orchestrator, woken by the484// notification, is the only writer of the actions table.485func (s *Server) handleCommandResult(w http.ResponseWriter, r *http.Request) {486	a := agentFrom(r.Context())487	id, err := commandID(r)488	if err != nil {489		httpx.WriteError(w, r, err)490		return491	}492	var res protocol.CommandResult493	if err := decodeBody(w, r, 256<<10, &res); err != nil {494		httpx.WriteError(w, r, err)495		return496	}497	if !resultStatuses[res.Status] {498		httpx.WriteError(w, r, httpx.Validation("status must be succeeded, failed, timed_out or cancelled."))499		return500	}501	if !finite(res.Duration) || res.Duration < 0 {502		httpx.WriteError(w, r, httpx.Validation("duration_s is invalid."))503		return504	}505	res.Error = redact.String(clip(res.Error, 2000))506	if res.Result != nil {507		res.Result, _ = redact.Value(res.Result).(map[string]any)508	}509	doc, err := json.Marshal(res)510	if err != nil || len(doc) > 256<<10 {511		httpx.WriteError(w, r, httpx.Validation("result is not serializable or too large."))512		return513	}514	ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)515	defer cancel()516	err = s.inTx(ctx, func(tx pgx.Tx) error {517		state, _, err := lockCommand(ctx, tx, id, a.ID)518		if err != nil {519			return err520		}521		// 'delivered' is accepted too: after a crash the agent reports commands522		// it had started without having been able to acknowledge them.523		if state != "acked" && state != "delivered" {524			return errCommandState525		}526		if _, err := tx.Exec(ctx, `update agent_commands set state = 'finished', finished_at = now(), result_status = $2, result = $3::jsonb,527				acked_at = coalesce(acked_at, now()), ack_accepted = coalesce(ack_accepted, true)528			where id = $1`, id, res.Status, doc); err != nil {529			return err530		}531		return notifyUpdate(ctx, tx, id)532	})533	if err != nil {534		s.writeErr(w, r, err)535		return536	}537	w.WriteHeader(http.StatusNoContent)538}539
Files main
Menu

Theme