Repository Public
1package agent23import (4 "context"5 "encoding/json"6 "errors"7 "time"89 "clustr.spboucher.ai/mcc/internal/agent/client"10 "clustr.spboucher.ai/mcc/internal/agent/executor"11 "clustr.spboucher.ai/mcc/internal/agent/journal"12 "clustr.spboucher.ai/mcc/internal/protocol"13)1415// commandLoop long-polls the ingest service. The transport is not trusted16// for commands: each envelope is verified against the orchestrator's key.17func (a *Agent) commandLoop(ctx context.Context) {18 backoff := time.Second19 for ctx.Err() == nil {20 resp, err := a.cl.Commands(ctx, a.o.PollWait)21 if err != nil {22 if ctx.Err() != nil {23 return24 }25 wait := client.Jitter(backoff)26 if client.Revoked(err) {27 wait = client.Jitter(60 * time.Second) // revoked: stay quiet until re-enrolled28 }29 select {30 case <-ctx.Done():31 return32 case <-time.After(wait):33 }34 if backoff < 30*time.Second {35 backoff *= 236 }37 continue38 }39 backoff = time.Second40 for _, id := range resp.Cancel {41 if a.cancelCommand(id) {42 a.log.Info("command cancellation requested by the server", "command_id", id)43 }44 }45 for _, sc := range resp.Commands {46 a.wg.Add(1)47 go func() {48 defer a.wg.Done()49 a.handleCommand(ctx, sc)50 }()51 }52 }53}5455// cancelCommand cooperatively stops a running command (SIGTERM to its56// process group, SIGKILL after the grace period).57func (a *Agent) cancelCommand(id string) bool {58 a.cmdMu.Lock()59 rc := a.running[id]60 a.cmdMu.Unlock()61 if rc == nil {62 return false63 }64 rc.cancel(executor.CancelCause())65 return true66}6768// peekID reads the command id from an unverified payload. It is used only69// to tell the server which command is being refused; nothing else from an70// unverified payload is ever used.71func peekID(payload []byte) string {72 var p struct {73 ID string `json:"id"`74 }75 json.Unmarshal(payload, &p)76 if len(p.ID) > 64 {77 return ""78 }79 return p.ID80}8182// post retries a call to the server for a bounded time.83func (a *Agent) post(ctx context.Context, total time.Duration, fn func(context.Context) error) error {84 deadline := time.Now().Add(total)85 backoff := 500 * time.Millisecond86 for {87 cctx, cancel := context.WithTimeout(ctx, 10*time.Second)88 err := fn(cctx)89 cancel()90 if err == nil || client.Permanent(err) || client.Revoked(err) {91 return err92 }93 if time.Now().Add(backoff).After(deadline) || ctx.Err() != nil {94 return err95 }96 select {97 case <-ctx.Done():98 return err99 case <-time.After(client.Jitter(backoff)):100 }101 if backoff < 10*time.Second {102 backoff *= 2103 }104 }105}106107func (a *Agent) refuse(ctx context.Context, id, reason string) {108 a.log.Warn("command refused", "command_id", id, "reason", reason)109 if id == "" {110 return111 }112 if err := a.post(ctx, 20*time.Second, func(c context.Context) error {113 return a.cl.Ack(c, id, protocol.CommandAck{Accepted: false, Reason: reason})114 }); err != nil {115 a.log.Warn("could not deliver the refusal", "command_id", id, "err", err)116 }117}118119// handleCommand takes one envelope through: kill switch, signature and120// replay verification, local policy, concurrency limits, durable journal,121// acknowledgement, execution, result.122func (a *Agent) handleCommand(ctx context.Context, sc protocol.SignedCommand) {123 // Refusals and results must reach the server even if the agent is asked to stop.124 bg := context.WithoutCancel(ctx)125 id := peekID(sc.Payload)126127 if a.dir.Frozen() {128 a.refuse(bg, id, "frozen")129 return130 }131 a.mu.Lock()132 keys := a.keys133 a.mu.Unlock()134 cmd, err := protocol.Verify(sc, keys, a.cl.ID.AgentID, time.Now(), a.nonce)135 if err != nil {136 if errors.Is(err, protocol.ErrUnknownKey) {137 a.tryRenew("unknown command key") // the orchestrator key may have rotated138 }139 a.refuse(bg, id, "verification failed: "+err.Error())140 return141 }142 pol := a.Policy()143 pctx, pcancel := context.WithTimeout(bg, 15*time.Second)144 plan, err := a.exec.Prepare(pctx, cmd, pol)145 pcancel()146 if err != nil {147 a.refuse(bg, cmd.ID, err.Error())148 return149 }150151 timeout := time.Duration(cmd.TimeoutS) * time.Second152 if timeout > 24*time.Hour {153 timeout = 24 * time.Hour154 }155 base, cancelTimeout := context.WithTimeout(bg, timeout)156 defer cancelTimeout()157 runCtx, cancel := context.WithCancelCause(base)158 defer cancel(nil)159160 a.cmdMu.Lock()161 switch {162 case len(a.running) >= MaxConcurrentCommands:163 a.cmdMu.Unlock()164 a.refuse(bg, cmd.ID, "busy: the agent already runs its maximum of concurrent commands")165 return166 case a.targets[plan.Target] != "":167 a.cmdMu.Unlock()168 a.refuse(bg, cmd.ID, "busy: another command is running on the same target")169 return170 }171 a.running[cmd.ID] = &runningCmd{cancel: cancel, typ: cmd.Type}172 a.targets[plan.Target] = cmd.ID173 a.cmdMu.Unlock()174 defer func() {175 a.cmdMu.Lock()176 delete(a.running, cmd.ID)177 delete(a.targets, plan.Target)178 a.cmdMu.Unlock()179 }()180181 // Durable before anything runs: if the agent dies from here on, the182 // command is reported "outcome unknown" at restart and never run again.183 if err := a.jrnl.Started(cmd.ID, cmd.Type); err != nil {184 a.refuse(bg, cmd.ID, "the agent cannot journal the command: "+err.Error())185 return186 }187 // No execution without an acknowledged acceptance: if the server cannot188 // be told, or says the command is closed, nothing runs.189 if err := a.post(bg, 15*time.Second, func(c context.Context) error {190 return a.cl.Ack(c, cmd.ID, protocol.CommandAck{Accepted: true})191 }); err != nil {192 a.log.Warn("acceptance not acknowledged by the server: command not executed", "command_id", cmd.ID, "err", err)193 a.jrnl.Finished(cmd.ID)194 return195 }196 a.log.Info("command accepted", "command_id", cmd.ID, "type", cmd.Type, "target", plan.Target)197198 sender := newOutputSender(bg, a, cmd.ID)199 out := executor.NewOutput(sender.add)200 start := time.Now()201 oc := func() (oc executor.Outcome) {202 defer func() {203 if v := recover(); v != nil {204 oc = executor.Outcome{Status: "failed", Error: "executor panic"}205 a.log.Error("executor panic", "command_id", cmd.ID, "value", v)206 }207 }()208 return plan.Run(runCtx, out)209 }()210 sender.close()211212 res := protocol.CommandResult{Status: oc.Status, ExitCode: oc.ExitCode, Error: oc.Error, Result: oc.Result, Duration: time.Since(start).Seconds()}213 if err := a.post(bg, 2*time.Minute, func(c context.Context) error { return a.cl.Result(c, cmd.ID, res) }); err != nil {214 a.log.Warn("result not delivered", "command_id", cmd.ID, "status", oc.Status, "err", err)215 if !client.Permanent(err) {216 // Left open in the journal: reported as unknown outcome after a restart.217 return218 }219 }220 a.jrnl.Finished(cmd.ID)221 a.log.Info("command finished", "command_id", cmd.ID, "type", cmd.Type, "status", oc.Status, "duration_s", res.Duration)222 if oc.RestartAgent && oc.Status == "succeeded" {223 a.exitCode.Store(executor.RestartExitCode)224 a.stop()225 }226}227228// resolveUnknown reports the commands a previous process had started and229// never closed. They are never executed again.230func (a *Agent) resolveUnknown(ctx context.Context) {231 pending := a.unresolved232 for len(pending) > 0 && ctx.Err() == nil {233 var left []journal.Unresolved234 for _, u := range pending {235 res := protocol.CommandResult{236 Status: "failed",237 Error: "outcome unknown: the agent stopped while this command was in progress; it was not executed again",238 Result: map[string]any{"outcome_unknown": true, "started_at": u.Started},239 }240 cctx, cancel := context.WithTimeout(ctx, 10*time.Second)241 err := a.cl.Result(cctx, u.ID, res)242 cancel()243 switch {244 case err == nil, client.Permanent(err):245 // Delivered, or the server already closed the command.246 a.jrnl.Reported(u.ID)247 a.log.Warn("command interrupted by a previous agent stop reported as outcome unknown", "command_id", u.ID, "type", u.Type)248 case time.Since(u.Started) > 48*time.Hour:249 a.jrnl.Reported(u.ID)250 default:251 left = append(left, u)252 }253 }254 pending = left255 if len(pending) == 0 {256 return257 }258 select {259 case <-ctx.Done():260 return261 case <-time.After(client.Jitter(30 * time.Second)):262 }263 }264}265266// outputSender batches output lines and posts them in order.267type outputSender struct {268 a *Agent269 id string270 ch chan protocol.CommandOutput271 done chan struct{}272}273274func newOutputSender(ctx context.Context, a *Agent, id string) *outputSender {275 s := &outputSender{a: a, id: id, ch: make(chan protocol.CommandOutput, 4096), done: make(chan struct{})}276 go s.loop(ctx)277 return s278}279280// add never blocks the executor: when the server is too slow, lines are281// dropped rather than stalling the command.282func (s *outputSender) add(o protocol.CommandOutput) {283 select {284 case s.ch <- o:285 default:286 }287}288289func (s *outputSender) close() {290 close(s.ch)291 <-s.done292}293294func (s *outputSender) loop(ctx context.Context) {295 defer close(s.done)296 tick := time.NewTicker(300 * time.Millisecond)297 defer tick.Stop()298 var batch []protocol.CommandOutput299 flush := func() {300 if len(batch) == 0 {301 return302 }303 lines := batch304 batch = nil305 if err := s.a.post(ctx, 10*time.Second, func(c context.Context) error { return s.a.cl.Events(c, s.id, lines) }); err != nil {306 s.a.log.Warn("command output not delivered", "command_id", s.id, "lines", len(lines), "err", err)307 }308 }309 for {310 select {311 case o, ok := <-s.ch:312 if !ok {313 flush()314 return315 }316 batch = append(batch, o)317 if len(batch) >= 200 {318 flush()319 }320 case <-tick.C:321 flush()322 }323 }324}325