Repository Public
1// Package agent is the mcc-agent runtime: it measures the node, reports to2// the ingest service over mTLS, buffers on disk while disconnected, and3// executes the signed, policy-approved commands it receives.4package agent56import (7 "context"8 "crypto/ed25519"9 "errors"10 "log/slog"11 "os"12 "path/filepath"13 "sync"14 "sync/atomic"15 "time"1617 "clustr.spboucher.ai/mcc/internal/agent/client"18 "clustr.spboucher.ai/mcc/internal/agent/collect"19 "clustr.spboucher.ai/mcc/internal/agent/executor"20 "clustr.spboucher.ai/mcc/internal/agent/journal"21 "clustr.spboucher.ai/mcc/internal/agent/nonce"22 "clustr.spboucher.ai/mcc/internal/agent/policy"23 "clustr.spboucher.ai/mcc/internal/agent/services"24 "clustr.spboucher.ai/mcc/internal/agent/spool"25 "clustr.spboucher.ai/mcc/internal/agent/state"26 "clustr.spboucher.ai/mcc/internal/protocol"27)2829// MaxConcurrentCommands bounds parallel command execution on a node.30const MaxConcurrentCommands = 23132// Options configures an Agent.33type Options struct {34 Dir *state.Dir35 Version string36 Log *slog.Logger37 // Interval overrides the report interval (tests). Zero: the server's value.38 Interval time.Duration39 // PollWait is the long-poll duration (default 25 s).40 PollWait time.Duration41 // OwnLabel is the launchd label / systemd unit running this agent.42 OwnLabel string43 // SpoolMaxAge / SpoolMaxBytes override the buffer bounds (tests).44 SpoolMaxAge time.Duration45 SpoolMaxBytes int6446}4748// Agent is one running agent.49type Agent struct {50 o Options51 log *slog.Logger52 dir *state.Dir53 cl *client.Client54 col *collect.Collector55 inf *services.Inference56 spool *spool.Spool57 nonce *nonce.Store58 jrnl *journal.Journal59 exec *executor.Executor6061 started time.Time6263 mu sync.Mutex64 interval time.Duration65 wantInv bool66 lastInv time.Time67 pendingInv *protocol.Inventory68 nasTargets []string69 lastServices time.Time70 lastInfer time.Time71 lastVolumes []protocol.Volume72 online bool73 revoked bool74 lastOK time.Time75 lastSeq int6476 lastErr string77 lastRenewTry time.Time78 selfCPU *float6479 selfRSS *int6480 policyErr string81 keys map[string]ed25519.PublicKey8283 cmdMu sync.Mutex84 running map[string]*runningCmd // command id -> state85 targets map[string]string // target -> command id8687 unresolved []journal.Unresolved88 exitCode atomic.Int3289 stop context.CancelFunc90 wg sync.WaitGroup91}9293type runningCmd struct {94 cancel context.CancelCauseFunc95 typ string96}9798// New opens the state of an enrolled agent.99func New(o Options) (*Agent, error) {100 if o.Log == nil {101 o.Log = slog.Default()102 }103 if o.PollWait <= 0 {104 o.PollWait = 25 * time.Second105 }106 cl, err := client.New(o.Dir)107 if err != nil {108 return nil, err109 }110 keys, err := o.Dir.LoadCommandKeys()111 if err != nil {112 return nil, err113 }114 sp, err := spool.Open(o.Dir.File("spool"), o.SpoolMaxAge, o.SpoolMaxBytes)115 if err != nil {116 return nil, err117 }118 ns, err := nonce.Open(o.Dir.File("nonces.log"))119 if err != nil {120 return nil, err121 }122 jr, unresolved, err := journal.Open(o.Dir.File("commands.journal"))123 if err != nil {124 return nil, err125 }126 a := &Agent{127 o: o, log: o.Log, dir: o.Dir, cl: cl, col: collect.New(), inf: services.NewInference(),128 spool: sp, nonce: ns, jrnl: jr, started: time.Now().UTC(), keys: keys,129 running: map[string]*runningCmd{}, targets: map[string]string{}, unresolved: unresolved,130 interval: time.Duration(cl.ID.IntervalS) * time.Second, wantInv: true,131 }132 if o.Interval > 0 {133 a.interval = o.Interval134 }135 if a.interval <= 0 {136 a.interval = 10 * time.Second137 }138 a.col.IngestHost = cl.Host139 a.exec = executor.New(executor.Deps{140 RefreshInventory: a.refreshInventory,141 Gateway: func(ctx context.Context) string {142 gw, _ := a.col.Gateway(ctx, time.Now())143 return gw144 },145 Cancel: a.cancelCommand,146 OwnLabel: o.OwnLabel,147 })148 return a, nil149}150151// Run blocks until ctx ends or an accepted agent.restart asks the process to152// exit. It returns the process exit code (0, or executor.RestartExitCode).153func (a *Agent) Run(ctx context.Context) int {154 ctx, a.stop = context.WithCancel(ctx)155 defer a.stop()156 a.wg.Add(4)157 go func() { defer a.wg.Done(); a.reportLoop(ctx) }()158 go func() { defer a.wg.Done(); a.replayLoop(ctx) }()159 go func() { defer a.wg.Done(); a.commandLoop(ctx) }()160 go func() { defer a.wg.Done(); a.resolveUnknown(ctx) }()161 <-ctx.Done()162 // Running commands get their cooperative stop; their results are still posted.163 a.cmdMu.Lock()164 for _, rc := range a.running {165 rc.cancel(errors.New("agent stopping"))166 }167 a.cmdMu.Unlock()168 done := make(chan struct{})169 go func() { a.wg.Wait(); close(done) }()170 select {171 case <-done:172 case <-time.After(executor.KillGrace + 10*time.Second):173 }174 a.writeStatus()175 a.cl.Close()176 a.nonce.Close()177 a.jrnl.Close()178 return int(a.exitCode.Load())179}180181// Policy loads the local policy. Errors are remembered for `status` and the182// log; the returned policy is then deny-all.183func (a *Agent) Policy() *policy.Policy {184 pol, err := policy.Load(a.dir.File(state.PolicyFile))185 msg := ""186 if err != nil {187 msg = err.Error()188 }189 a.mu.Lock()190 changed := a.policyErr != msg191 a.policyErr = msg192 a.mu.Unlock()193 if changed && msg != "" {194 a.log.Error("local policy is invalid: every command is refused", "err", msg)195 }196 return pol197}198199func (a *Agent) capabilities() []string {200 if a.dir.Frozen() {201 return []string{}202 }203 return a.Policy().Capabilities()204}205206func (a *Agent) refreshInventory(ctx context.Context) protocol.Inventory {207 inv := a.col.Inventory(ctx)208 a.mu.Lock()209 a.pendingInv, a.lastInv, a.wantInv = &inv, time.Now(), false210 a.mu.Unlock()211 return inv212}213214// ─────────────────────────── reports ───────────────────────────215216func (a *Agent) reportLoop(ctx context.Context) {217 a.reportOnce(ctx)218 for {219 a.mu.Lock()220 every := a.interval221 a.mu.Unlock()222 select {223 case <-ctx.Done():224 return225 case <-time.After(every):226 }227 a.reportOnce(ctx)228 }229}230231// buildReport measures the node and assembles one report.232func (a *Agent) buildReport(ctx context.Context, now time.Time) (*protocol.Report, error) {233 a.mu.Lock()234 needInv := a.pendingInv == nil && (a.wantInv || now.Sub(a.lastInv) > 6*time.Hour)235 needSvc := now.Sub(a.lastServices) >= 30*time.Second-time.Second236 needInf := now.Sub(a.lastInfer) >= 60*time.Second-time.Second237 nas := append([]string(nil), a.nasTargets...)238 a.mu.Unlock()239240 var svc *protocol.ServiceSet241 var wg sync.WaitGroup242 if needSvc {243 wg.Add(1)244 go func() { defer wg.Done(); svc = services.Observe(ctx) }()245 }246 if needInv {247 wg.Add(1)248 go func() { defer wg.Done(); a.refreshInventory(ctx) }()249 }250 round := a.col.Collect(ctx, now)251 wg.Wait()252253 seq, err := a.dir.NextSeq()254 if err != nil {255 return nil, err256 }257 rep := &protocol.Report{258 V: protocol.Version, Seq: seq, TS: now.UTC(), AgentVersion: a.o.Version,259 Samples: round.Samples, Availability: a.col.Availability(), Processes: round.Processes,260 Volumes: round.Volumes, Services: svc, Capabilities: a.capabilities(),261 }262 if rep.Samples == nil {263 rep.Samples = []protocol.Sample{}264 }265 a.mu.Lock()266 if svc != nil {267 a.lastServices = now268 }269 if round.Volumes != nil {270 a.lastVolumes = round.Volumes271 }272 if round.SelfCPU != nil {273 a.selfCPU = round.SelfCPU274 }275 if round.SelfRSS != nil {276 a.selfRSS = round.SelfRSS277 }278 rep.Inventory = a.pendingInv279 vols := a.lastVolumes280 a.mu.Unlock()281282 if needInf {283 if rows, err := collect.ListProcesses(ctx); err == nil {284 rep.Inference = a.inf.Observe(ctx, rows, now)285 a.mu.Lock()286 a.lastInfer = now287 a.mu.Unlock()288 if len(nas) > 0 {289 rep.NAS = services.ProbeNAS(ctx, nas, vols)290 }291 }292 }293 return rep, nil294}295296func (a *Agent) reportOnce(ctx context.Context) {297 now := time.Now()298 rep, err := a.buildReport(ctx, now)299 if err != nil {300 a.log.Error("cannot build report", "err", err)301 a.setError(err)302 return303 }304 enc, err := client.EncodeReport(rep)305 if err != nil {306 a.log.Error("cannot encode report", "err", err)307 return308 }309 sctx, cancel := context.WithTimeout(ctx, 8*time.Second)310 ack, err := a.cl.Report(sctx, enc)311 cancel()312 switch {313 case err == nil:314 a.onAck(rep, ack)315 case client.Permanent(err):316 // The server will never accept this report: dropping it is the only317 // way not to block the queue. Logged with the reason.318 a.log.Error("report rejected by the server and dropped", "seq", rep.Seq, "err", err)319 a.setError(err)320 default:321 // Not delivered: keep it for replay. State carried by a buffered322 // report (services, inventory…) is ignored by the server, so those323 // observations are scheduled again for the next live report.324 rep.Buffered = true325 inv := rep.Inventory326 rep.Inventory, rep.Services, rep.Inference, rep.NAS = nil, nil, nil, nil327 if enc, eerr := client.EncodeReport(rep); eerr == nil {328 if perr := a.spool.Put(rep.Seq, rep.TS, enc, now); perr != nil {329 a.log.Error("cannot buffer report", "err", perr)330 }331 }332 a.mu.Lock()333 wasOnline := a.online334 a.online = false335 a.revoked = client.Revoked(err)336 a.lastErr = err.Error()337 a.lastServices, a.lastInfer = time.Time{}, time.Time{}338 if inv != nil {339 a.pendingInv = inv340 }341 a.mu.Unlock()342 if wasOnline {343 a.log.Warn("ingest unreachable: buffering reports on disk", "err", err)344 }345 }346 a.spool.Enforce(now)347 a.writeStatus()348}349350func (a *Agent) onAck(rep *protocol.Report, ack *protocol.ReportAck) {351 a.mu.Lock()352 wasOnline := a.online353 a.online, a.revoked, a.lastOK, a.lastSeq, a.lastErr = true, false, time.Now().UTC(), rep.Seq, ""354 if rep.Inventory != nil {355 a.pendingInv = nil356 }357 if ack.WantInventory && rep.Inventory == nil {358 a.wantInv = true359 }360 if a.o.Interval == 0 && ack.IntervalS >= 5 && ack.IntervalS <= 300 {361 a.interval = time.Duration(ack.IntervalS) * time.Second362 }363 a.nasTargets = ack.NASTargets364 renew := ack.RenewCert365 a.mu.Unlock()366 if !wasOnline {367 a.log.Info("connected to ingest", "endpoint", a.cl.Endpoint())368 }369 nb, na := a.cl.CertNotAfter()370 if half := nb.Add(na.Sub(nb) / 2); time.Now().After(half) {371 renew = true372 }373 if renew {374 a.tryRenew("certificate past half of its life")375 }376}377378// tryRenew renews the certificate in the background, at most once per 10 minutes.379func (a *Agent) tryRenew(reason string) {380 a.mu.Lock()381 if time.Since(a.lastRenewTry) < 10*time.Minute {382 a.mu.Unlock()383 return384 }385 a.lastRenewTry = time.Now()386 a.mu.Unlock()387 a.wg.Add(1)388 go func() {389 defer a.wg.Done()390 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)391 defer cancel()392 if err := a.cl.Renew(ctx); err != nil {393 a.log.Warn("certificate renewal failed; will retry", "reason", reason, "err", err)394 return395 }396 if keys, err := a.dir.LoadCommandKeys(); err == nil {397 a.mu.Lock()398 a.keys = keys399 a.mu.Unlock()400 }401 _, na := a.cl.CertNotAfter()402 a.log.Info("certificate renewed", "reason", reason, "not_after", na)403 }()404}405406func (a *Agent) setError(err error) {407 a.mu.Lock()408 a.lastErr = err.Error()409 a.mu.Unlock()410}411412// replayLoop drains the spool, oldest first, while the live path works. It413// runs beside the live reports and is paced, so it never starves them.414func (a *Agent) replayLoop(ctx context.Context) {415 for {416 a.mu.Lock()417 online := a.online418 a.mu.Unlock()419 pause := 2 * time.Second420 if online {421 e, data, err := a.spool.Oldest()422 if err == nil {423 sctx, cancel := context.WithTimeout(ctx, 8*time.Second)424 _, rerr := a.cl.Report(sctx, data)425 cancel()426 switch {427 case rerr == nil:428 a.spool.Remove(e.Seq)429 pause = 100 * time.Millisecond430 case client.Permanent(rerr):431 a.log.Warn("buffered report rejected by the server and dropped", "seq", e.Seq, "err", rerr)432 a.spool.Remove(e.Seq)433 pause = 100 * time.Millisecond434 default:435 pause = 5 * time.Second // the live loop decides when we are online again436 }437 }438 }439 select {440 case <-ctx.Done():441 return442 case <-time.After(pause):443 }444 }445}446447func (a *Agent) writeStatus() {448 n, bytes, dropped := a.spool.Stats()449 a.mu.Lock()450 st := &state.Status{451 PID: os.Getpid(), Version: a.o.Version, StartedAt: a.started, UpdatedAt: time.Now().UTC(),452 Connection: "offline", Endpoint: a.cl.Endpoint(), LastReportOK: a.lastOK, LastSeq: a.lastSeq, LastError: a.lastErr,453 SpoolReports: n, SpoolBytes: bytes, Dropped: dropped, SelfCPUPercent: a.selfCPU, SelfRSSBytes: a.selfRSS,454 Frozen: a.dir.Frozen(), PolicyError: a.policyErr,455 }456 switch {457 case a.revoked:458 st.Connection = "revoked"459 case a.online:460 st.Connection = "online"461 }462 a.mu.Unlock()463 st.Capabilities = a.capabilities()464 if err := a.dir.SaveStatus(st); err != nil {465 a.log.Warn("cannot write status file", "err", err)466 }467}468469// SpoolDir is the spool location inside a state directory.470func SpoolDir(d *state.Dir) string { return filepath.Join(d.Path, "spool") }471