SPB Git

Repository Public

commands.go 9.1 KB 325 lines go

1package agent23import (4	"context"5	"encoding/json"6	"errors"7	"time"89	"clustr.spboucher.ai/mcc/internal/agent/client"10	"clustr.spboucher.ai/mcc/internal/agent/executor"11	"clustr.spboucher.ai/mcc/internal/agent/journal"12	"clustr.spboucher.ai/mcc/internal/protocol"13)1415// commandLoop long-polls the ingest service. The transport is not trusted16// for commands: each envelope is verified against the orchestrator's key.17func (a *Agent) commandLoop(ctx context.Context) {18	backoff := time.Second19	for ctx.Err() == nil {20		resp, err := a.cl.Commands(ctx, a.o.PollWait)21		if err != nil {22			if ctx.Err() != nil {23				return24			}25			wait := client.Jitter(backoff)26			if client.Revoked(err) {27				wait = client.Jitter(60 * time.Second) // revoked: stay quiet until re-enrolled28			}29			select {30			case <-ctx.Done():31				return32			case <-time.After(wait):33			}34			if backoff < 30*time.Second {35				backoff *= 236			}37			continue38		}39		backoff = time.Second40		for _, id := range resp.Cancel {41			if a.cancelCommand(id) {42				a.log.Info("command cancellation requested by the server", "command_id", id)43			}44		}45		for _, sc := range resp.Commands {46			a.wg.Add(1)47			go func() {48				defer a.wg.Done()49				a.handleCommand(ctx, sc)50			}()51		}52	}53}5455// cancelCommand cooperatively stops a running command (SIGTERM to its56// process group, SIGKILL after the grace period).57func (a *Agent) cancelCommand(id string) bool {58	a.cmdMu.Lock()59	rc := a.running[id]60	a.cmdMu.Unlock()61	if rc == nil {62		return false63	}64	rc.cancel(executor.CancelCause())65	return true66}6768// peekID reads the command id from an unverified payload. It is used only69// to tell the server which command is being refused; nothing else from an70// unverified payload is ever used.71func peekID(payload []byte) string {72	var p struct {73		ID string `json:"id"`74	}75	json.Unmarshal(payload, &p)76	if len(p.ID) > 64 {77		return ""78	}79	return p.ID80}8182// post retries a call to the server for a bounded time.83func (a *Agent) post(ctx context.Context, total time.Duration, fn func(context.Context) error) error {84	deadline := time.Now().Add(total)85	backoff := 500 * time.Millisecond86	for {87		cctx, cancel := context.WithTimeout(ctx, 10*time.Second)88		err := fn(cctx)89		cancel()90		if err == nil || client.Permanent(err) || client.Revoked(err) {91			return err92		}93		if time.Now().Add(backoff).After(deadline) || ctx.Err() != nil {94			return err95		}96		select {97		case <-ctx.Done():98			return err99		case <-time.After(client.Jitter(backoff)):100		}101		if backoff < 10*time.Second {102			backoff *= 2103		}104	}105}106107func (a *Agent) refuse(ctx context.Context, id, reason string) {108	a.log.Warn("command refused", "command_id", id, "reason", reason)109	if id == "" {110		return111	}112	if err := a.post(ctx, 20*time.Second, func(c context.Context) error {113		return a.cl.Ack(c, id, protocol.CommandAck{Accepted: false, Reason: reason})114	}); err != nil {115		a.log.Warn("could not deliver the refusal", "command_id", id, "err", err)116	}117}118119// handleCommand takes one envelope through: kill switch, signature and120// replay verification, local policy, concurrency limits, durable journal,121// acknowledgement, execution, result.122func (a *Agent) handleCommand(ctx context.Context, sc protocol.SignedCommand) {123	// Refusals and results must reach the server even if the agent is asked to stop.124	bg := context.WithoutCancel(ctx)125	id := peekID(sc.Payload)126127	if a.dir.Frozen() {128		a.refuse(bg, id, "frozen")129		return130	}131	a.mu.Lock()132	keys := a.keys133	a.mu.Unlock()134	cmd, err := protocol.Verify(sc, keys, a.cl.ID.AgentID, time.Now(), a.nonce)135	if err != nil {136		if errors.Is(err, protocol.ErrUnknownKey) {137			a.tryRenew("unknown command key") // the orchestrator key may have rotated138		}139		a.refuse(bg, id, "verification failed: "+err.Error())140		return141	}142	pol := a.Policy()143	pctx, pcancel := context.WithTimeout(bg, 15*time.Second)144	plan, err := a.exec.Prepare(pctx, cmd, pol)145	pcancel()146	if err != nil {147		a.refuse(bg, cmd.ID, err.Error())148		return149	}150151	timeout := time.Duration(cmd.TimeoutS) * time.Second152	if timeout > 24*time.Hour {153		timeout = 24 * time.Hour154	}155	base, cancelTimeout := context.WithTimeout(bg, timeout)156	defer cancelTimeout()157	runCtx, cancel := context.WithCancelCause(base)158	defer cancel(nil)159160	a.cmdMu.Lock()161	switch {162	case len(a.running) >= MaxConcurrentCommands:163		a.cmdMu.Unlock()164		a.refuse(bg, cmd.ID, "busy: the agent already runs its maximum of concurrent commands")165		return166	case a.targets[plan.Target] != "":167		a.cmdMu.Unlock()168		a.refuse(bg, cmd.ID, "busy: another command is running on the same target")169		return170	}171	a.running[cmd.ID] = &runningCmd{cancel: cancel, typ: cmd.Type}172	a.targets[plan.Target] = cmd.ID173	a.cmdMu.Unlock()174	defer func() {175		a.cmdMu.Lock()176		delete(a.running, cmd.ID)177		delete(a.targets, plan.Target)178		a.cmdMu.Unlock()179	}()180181	// Durable before anything runs: if the agent dies from here on, the182	// command is reported "outcome unknown" at restart and never run again.183	if err := a.jrnl.Started(cmd.ID, cmd.Type); err != nil {184		a.refuse(bg, cmd.ID, "the agent cannot journal the command: "+err.Error())185		return186	}187	// No execution without an acknowledged acceptance: if the server cannot188	// be told, or says the command is closed, nothing runs.189	if err := a.post(bg, 15*time.Second, func(c context.Context) error {190		return a.cl.Ack(c, cmd.ID, protocol.CommandAck{Accepted: true})191	}); err != nil {192		a.log.Warn("acceptance not acknowledged by the server: command not executed", "command_id", cmd.ID, "err", err)193		a.jrnl.Finished(cmd.ID)194		return195	}196	a.log.Info("command accepted", "command_id", cmd.ID, "type", cmd.Type, "target", plan.Target)197198	sender := newOutputSender(bg, a, cmd.ID)199	out := executor.NewOutput(sender.add)200	start := time.Now()201	oc := func() (oc executor.Outcome) {202		defer func() {203			if v := recover(); v != nil {204				oc = executor.Outcome{Status: "failed", Error: "executor panic"}205				a.log.Error("executor panic", "command_id", cmd.ID, "value", v)206			}207		}()208		return plan.Run(runCtx, out)209	}()210	sender.close()211212	res := protocol.CommandResult{Status: oc.Status, ExitCode: oc.ExitCode, Error: oc.Error, Result: oc.Result, Duration: time.Since(start).Seconds()}213	if err := a.post(bg, 2*time.Minute, func(c context.Context) error { return a.cl.Result(c, cmd.ID, res) }); err != nil {214		a.log.Warn("result not delivered", "command_id", cmd.ID, "status", oc.Status, "err", err)215		if !client.Permanent(err) {216			// Left open in the journal: reported as unknown outcome after a restart.217			return218		}219	}220	a.jrnl.Finished(cmd.ID)221	a.log.Info("command finished", "command_id", cmd.ID, "type", cmd.Type, "status", oc.Status, "duration_s", res.Duration)222	if oc.RestartAgent && oc.Status == "succeeded" {223		a.exitCode.Store(executor.RestartExitCode)224		a.stop()225	}226}227228// resolveUnknown reports the commands a previous process had started and229// never closed. They are never executed again.230func (a *Agent) resolveUnknown(ctx context.Context) {231	pending := a.unresolved232	for len(pending) > 0 && ctx.Err() == nil {233		var left []journal.Unresolved234		for _, u := range pending {235			res := protocol.CommandResult{236				Status: "failed",237				Error:  "outcome unknown: the agent stopped while this command was in progress; it was not executed again",238				Result: map[string]any{"outcome_unknown": true, "started_at": u.Started},239			}240			cctx, cancel := context.WithTimeout(ctx, 10*time.Second)241			err := a.cl.Result(cctx, u.ID, res)242			cancel()243			switch {244			case err == nil, client.Permanent(err):245				// Delivered, or the server already closed the command.246				a.jrnl.Reported(u.ID)247				a.log.Warn("command interrupted by a previous agent stop reported as outcome unknown", "command_id", u.ID, "type", u.Type)248			case time.Since(u.Started) > 48*time.Hour:249				a.jrnl.Reported(u.ID)250			default:251				left = append(left, u)252			}253		}254		pending = left255		if len(pending) == 0 {256			return257		}258		select {259		case <-ctx.Done():260			return261		case <-time.After(client.Jitter(30 * time.Second)):262		}263	}264}265266// outputSender batches output lines and posts them in order.267type outputSender struct {268	a    *Agent269	id   string270	ch   chan protocol.CommandOutput271	done chan struct{}272}273274func newOutputSender(ctx context.Context, a *Agent, id string) *outputSender {275	s := &outputSender{a: a, id: id, ch: make(chan protocol.CommandOutput, 4096), done: make(chan struct{})}276	go s.loop(ctx)277	return s278}279280// add never blocks the executor: when the server is too slow, lines are281// dropped rather than stalling the command.282func (s *outputSender) add(o protocol.CommandOutput) {283	select {284	case s.ch <- o:285	default:286	}287}288289func (s *outputSender) close() {290	close(s.ch)291	<-s.done292}293294func (s *outputSender) loop(ctx context.Context) {295	defer close(s.done)296	tick := time.NewTicker(300 * time.Millisecond)297	defer tick.Stop()298	var batch []protocol.CommandOutput299	flush := func() {300		if len(batch) == 0 {301			return302		}303		lines := batch304		batch = nil305		if err := s.a.post(ctx, 10*time.Second, func(c context.Context) error { return s.a.cl.Events(c, s.id, lines) }); err != nil {306			s.a.log.Warn("command output not delivered", "command_id", s.id, "lines", len(lines), "err", err)307		}308	}309	for {310		select {311		case o, ok := <-s.ch:312			if !ok {313				flush()314				return315			}316			batch = append(batch, o)317			if len(batch) >= 200 {318				flush()319			}320		case <-tick.C:321			flush()322		}323	}324}325
Files main
Menu

Theme