SPB Git

Repository Public

agent.go 12.5 KB 471 lines go

1// Package agent is the mcc-agent runtime: it measures the node, reports to2// the ingest service over mTLS, buffers on disk while disconnected, and3// executes the signed, policy-approved commands it receives.4package agent56import (7	"context"8	"crypto/ed25519"9	"errors"10	"log/slog"11	"os"12	"path/filepath"13	"sync"14	"sync/atomic"15	"time"1617	"clustr.spboucher.ai/mcc/internal/agent/client"18	"clustr.spboucher.ai/mcc/internal/agent/collect"19	"clustr.spboucher.ai/mcc/internal/agent/executor"20	"clustr.spboucher.ai/mcc/internal/agent/journal"21	"clustr.spboucher.ai/mcc/internal/agent/nonce"22	"clustr.spboucher.ai/mcc/internal/agent/policy"23	"clustr.spboucher.ai/mcc/internal/agent/services"24	"clustr.spboucher.ai/mcc/internal/agent/spool"25	"clustr.spboucher.ai/mcc/internal/agent/state"26	"clustr.spboucher.ai/mcc/internal/protocol"27)2829// MaxConcurrentCommands bounds parallel command execution on a node.30const MaxConcurrentCommands = 23132// Options configures an Agent.33type Options struct {34	Dir     *state.Dir35	Version string36	Log     *slog.Logger37	// Interval overrides the report interval (tests). Zero: the server's value.38	Interval time.Duration39	// PollWait is the long-poll duration (default 25 s).40	PollWait time.Duration41	// OwnLabel is the launchd label / systemd unit running this agent.42	OwnLabel string43	// SpoolMaxAge / SpoolMaxBytes override the buffer bounds (tests).44	SpoolMaxAge   time.Duration45	SpoolMaxBytes int6446}4748// Agent is one running agent.49type Agent struct {50	o     Options51	log   *slog.Logger52	dir   *state.Dir53	cl    *client.Client54	col   *collect.Collector55	inf   *services.Inference56	spool *spool.Spool57	nonce *nonce.Store58	jrnl  *journal.Journal59	exec  *executor.Executor6061	started time.Time6263	mu           sync.Mutex64	interval     time.Duration65	wantInv      bool66	lastInv      time.Time67	pendingInv   *protocol.Inventory68	nasTargets   []string69	lastServices time.Time70	lastInfer    time.Time71	lastVolumes  []protocol.Volume72	online       bool73	revoked      bool74	lastOK       time.Time75	lastSeq      int6476	lastErr      string77	lastRenewTry time.Time78	selfCPU      *float6479	selfRSS      *int6480	policyErr    string81	keys         map[string]ed25519.PublicKey8283	cmdMu   sync.Mutex84	running map[string]*runningCmd // command id -> state85	targets map[string]string      // target -> command id8687	unresolved []journal.Unresolved88	exitCode   atomic.Int3289	stop       context.CancelFunc90	wg         sync.WaitGroup91}9293type runningCmd struct {94	cancel context.CancelCauseFunc95	typ    string96}9798// New opens the state of an enrolled agent.99func New(o Options) (*Agent, error) {100	if o.Log == nil {101		o.Log = slog.Default()102	}103	if o.PollWait <= 0 {104		o.PollWait = 25 * time.Second105	}106	cl, err := client.New(o.Dir)107	if err != nil {108		return nil, err109	}110	keys, err := o.Dir.LoadCommandKeys()111	if err != nil {112		return nil, err113	}114	sp, err := spool.Open(o.Dir.File("spool"), o.SpoolMaxAge, o.SpoolMaxBytes)115	if err != nil {116		return nil, err117	}118	ns, err := nonce.Open(o.Dir.File("nonces.log"))119	if err != nil {120		return nil, err121	}122	jr, unresolved, err := journal.Open(o.Dir.File("commands.journal"))123	if err != nil {124		return nil, err125	}126	a := &Agent{127		o: o, log: o.Log, dir: o.Dir, cl: cl, col: collect.New(), inf: services.NewInference(),128		spool: sp, nonce: ns, jrnl: jr, started: time.Now().UTC(), keys: keys,129		running: map[string]*runningCmd{}, targets: map[string]string{}, unresolved: unresolved,130		interval: time.Duration(cl.ID.IntervalS) * time.Second, wantInv: true,131	}132	if o.Interval > 0 {133		a.interval = o.Interval134	}135	if a.interval <= 0 {136		a.interval = 10 * time.Second137	}138	a.col.IngestHost = cl.Host139	a.exec = executor.New(executor.Deps{140		RefreshInventory: a.refreshInventory,141		Gateway: func(ctx context.Context) string {142			gw, _ := a.col.Gateway(ctx, time.Now())143			return gw144		},145		Cancel:   a.cancelCommand,146		OwnLabel: o.OwnLabel,147	})148	return a, nil149}150151// Run blocks until ctx ends or an accepted agent.restart asks the process to152// exit. It returns the process exit code (0, or executor.RestartExitCode).153func (a *Agent) Run(ctx context.Context) int {154	ctx, a.stop = context.WithCancel(ctx)155	defer a.stop()156	a.wg.Add(4)157	go func() { defer a.wg.Done(); a.reportLoop(ctx) }()158	go func() { defer a.wg.Done(); a.replayLoop(ctx) }()159	go func() { defer a.wg.Done(); a.commandLoop(ctx) }()160	go func() { defer a.wg.Done(); a.resolveUnknown(ctx) }()161	<-ctx.Done()162	// Running commands get their cooperative stop; their results are still posted.163	a.cmdMu.Lock()164	for _, rc := range a.running {165		rc.cancel(errors.New("agent stopping"))166	}167	a.cmdMu.Unlock()168	done := make(chan struct{})169	go func() { a.wg.Wait(); close(done) }()170	select {171	case <-done:172	case <-time.After(executor.KillGrace + 10*time.Second):173	}174	a.writeStatus()175	a.cl.Close()176	a.nonce.Close()177	a.jrnl.Close()178	return int(a.exitCode.Load())179}180181// Policy loads the local policy. Errors are remembered for `status` and the182// log; the returned policy is then deny-all.183func (a *Agent) Policy() *policy.Policy {184	pol, err := policy.Load(a.dir.File(state.PolicyFile))185	msg := ""186	if err != nil {187		msg = err.Error()188	}189	a.mu.Lock()190	changed := a.policyErr != msg191	a.policyErr = msg192	a.mu.Unlock()193	if changed && msg != "" {194		a.log.Error("local policy is invalid: every command is refused", "err", msg)195	}196	return pol197}198199func (a *Agent) capabilities() []string {200	if a.dir.Frozen() {201		return []string{}202	}203	return a.Policy().Capabilities()204}205206func (a *Agent) refreshInventory(ctx context.Context) protocol.Inventory {207	inv := a.col.Inventory(ctx)208	a.mu.Lock()209	a.pendingInv, a.lastInv, a.wantInv = &inv, time.Now(), false210	a.mu.Unlock()211	return inv212}213214// ─────────────────────────── reports ───────────────────────────215216func (a *Agent) reportLoop(ctx context.Context) {217	a.reportOnce(ctx)218	for {219		a.mu.Lock()220		every := a.interval221		a.mu.Unlock()222		select {223		case <-ctx.Done():224			return225		case <-time.After(every):226		}227		a.reportOnce(ctx)228	}229}230231// buildReport measures the node and assembles one report.232func (a *Agent) buildReport(ctx context.Context, now time.Time) (*protocol.Report, error) {233	a.mu.Lock()234	needInv := a.pendingInv == nil && (a.wantInv || now.Sub(a.lastInv) > 6*time.Hour)235	needSvc := now.Sub(a.lastServices) >= 30*time.Second-time.Second236	needInf := now.Sub(a.lastInfer) >= 60*time.Second-time.Second237	nas := append([]string(nil), a.nasTargets...)238	a.mu.Unlock()239240	var svc *protocol.ServiceSet241	var wg sync.WaitGroup242	if needSvc {243		wg.Add(1)244		go func() { defer wg.Done(); svc = services.Observe(ctx) }()245	}246	if needInv {247		wg.Add(1)248		go func() { defer wg.Done(); a.refreshInventory(ctx) }()249	}250	round := a.col.Collect(ctx, now)251	wg.Wait()252253	seq, err := a.dir.NextSeq()254	if err != nil {255		return nil, err256	}257	rep := &protocol.Report{258		V: protocol.Version, Seq: seq, TS: now.UTC(), AgentVersion: a.o.Version,259		Samples: round.Samples, Availability: a.col.Availability(), Processes: round.Processes,260		Volumes: round.Volumes, Services: svc, Capabilities: a.capabilities(),261	}262	if rep.Samples == nil {263		rep.Samples = []protocol.Sample{}264	}265	a.mu.Lock()266	if svc != nil {267		a.lastServices = now268	}269	if round.Volumes != nil {270		a.lastVolumes = round.Volumes271	}272	if round.SelfCPU != nil {273		a.selfCPU = round.SelfCPU274	}275	if round.SelfRSS != nil {276		a.selfRSS = round.SelfRSS277	}278	rep.Inventory = a.pendingInv279	vols := a.lastVolumes280	a.mu.Unlock()281282	if needInf {283		if rows, err := collect.ListProcesses(ctx); err == nil {284			rep.Inference = a.inf.Observe(ctx, rows, now)285			a.mu.Lock()286			a.lastInfer = now287			a.mu.Unlock()288			if len(nas) > 0 {289				rep.NAS = services.ProbeNAS(ctx, nas, vols)290			}291		}292	}293	return rep, nil294}295296func (a *Agent) reportOnce(ctx context.Context) {297	now := time.Now()298	rep, err := a.buildReport(ctx, now)299	if err != nil {300		a.log.Error("cannot build report", "err", err)301		a.setError(err)302		return303	}304	enc, err := client.EncodeReport(rep)305	if err != nil {306		a.log.Error("cannot encode report", "err", err)307		return308	}309	sctx, cancel := context.WithTimeout(ctx, 8*time.Second)310	ack, err := a.cl.Report(sctx, enc)311	cancel()312	switch {313	case err == nil:314		a.onAck(rep, ack)315	case client.Permanent(err):316		// The server will never accept this report: dropping it is the only317		// way not to block the queue. Logged with the reason.318		a.log.Error("report rejected by the server and dropped", "seq", rep.Seq, "err", err)319		a.setError(err)320	default:321		// Not delivered: keep it for replay. State carried by a buffered322		// report (services, inventory…) is ignored by the server, so those323		// observations are scheduled again for the next live report.324		rep.Buffered = true325		inv := rep.Inventory326		rep.Inventory, rep.Services, rep.Inference, rep.NAS = nil, nil, nil, nil327		if enc, eerr := client.EncodeReport(rep); eerr == nil {328			if perr := a.spool.Put(rep.Seq, rep.TS, enc, now); perr != nil {329				a.log.Error("cannot buffer report", "err", perr)330			}331		}332		a.mu.Lock()333		wasOnline := a.online334		a.online = false335		a.revoked = client.Revoked(err)336		a.lastErr = err.Error()337		a.lastServices, a.lastInfer = time.Time{}, time.Time{}338		if inv != nil {339			a.pendingInv = inv340		}341		a.mu.Unlock()342		if wasOnline {343			a.log.Warn("ingest unreachable: buffering reports on disk", "err", err)344		}345	}346	a.spool.Enforce(now)347	a.writeStatus()348}349350func (a *Agent) onAck(rep *protocol.Report, ack *protocol.ReportAck) {351	a.mu.Lock()352	wasOnline := a.online353	a.online, a.revoked, a.lastOK, a.lastSeq, a.lastErr = true, false, time.Now().UTC(), rep.Seq, ""354	if rep.Inventory != nil {355		a.pendingInv = nil356	}357	if ack.WantInventory && rep.Inventory == nil {358		a.wantInv = true359	}360	if a.o.Interval == 0 && ack.IntervalS >= 5 && ack.IntervalS <= 300 {361		a.interval = time.Duration(ack.IntervalS) * time.Second362	}363	a.nasTargets = ack.NASTargets364	renew := ack.RenewCert365	a.mu.Unlock()366	if !wasOnline {367		a.log.Info("connected to ingest", "endpoint", a.cl.Endpoint())368	}369	nb, na := a.cl.CertNotAfter()370	if half := nb.Add(na.Sub(nb) / 2); time.Now().After(half) {371		renew = true372	}373	if renew {374		a.tryRenew("certificate past half of its life")375	}376}377378// tryRenew renews the certificate in the background, at most once per 10 minutes.379func (a *Agent) tryRenew(reason string) {380	a.mu.Lock()381	if time.Since(a.lastRenewTry) < 10*time.Minute {382		a.mu.Unlock()383		return384	}385	a.lastRenewTry = time.Now()386	a.mu.Unlock()387	a.wg.Add(1)388	go func() {389		defer a.wg.Done()390		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)391		defer cancel()392		if err := a.cl.Renew(ctx); err != nil {393			a.log.Warn("certificate renewal failed; will retry", "reason", reason, "err", err)394			return395		}396		if keys, err := a.dir.LoadCommandKeys(); err == nil {397			a.mu.Lock()398			a.keys = keys399			a.mu.Unlock()400		}401		_, na := a.cl.CertNotAfter()402		a.log.Info("certificate renewed", "reason", reason, "not_after", na)403	}()404}405406func (a *Agent) setError(err error) {407	a.mu.Lock()408	a.lastErr = err.Error()409	a.mu.Unlock()410}411412// replayLoop drains the spool, oldest first, while the live path works. It413// runs beside the live reports and is paced, so it never starves them.414func (a *Agent) replayLoop(ctx context.Context) {415	for {416		a.mu.Lock()417		online := a.online418		a.mu.Unlock()419		pause := 2 * time.Second420		if online {421			e, data, err := a.spool.Oldest()422			if err == nil {423				sctx, cancel := context.WithTimeout(ctx, 8*time.Second)424				_, rerr := a.cl.Report(sctx, data)425				cancel()426				switch {427				case rerr == nil:428					a.spool.Remove(e.Seq)429					pause = 100 * time.Millisecond430				case client.Permanent(rerr):431					a.log.Warn("buffered report rejected by the server and dropped", "seq", e.Seq, "err", rerr)432					a.spool.Remove(e.Seq)433					pause = 100 * time.Millisecond434				default:435					pause = 5 * time.Second // the live loop decides when we are online again436				}437			}438		}439		select {440		case <-ctx.Done():441			return442		case <-time.After(pause):443		}444	}445}446447func (a *Agent) writeStatus() {448	n, bytes, dropped := a.spool.Stats()449	a.mu.Lock()450	st := &state.Status{451		PID: os.Getpid(), Version: a.o.Version, StartedAt: a.started, UpdatedAt: time.Now().UTC(),452		Connection: "offline", Endpoint: a.cl.Endpoint(), LastReportOK: a.lastOK, LastSeq: a.lastSeq, LastError: a.lastErr,453		SpoolReports: n, SpoolBytes: bytes, Dropped: dropped, SelfCPUPercent: a.selfCPU, SelfRSSBytes: a.selfRSS,454		Frozen: a.dir.Frozen(), PolicyError: a.policyErr,455	}456	switch {457	case a.revoked:458		st.Connection = "revoked"459	case a.online:460		st.Connection = "online"461	}462	a.mu.Unlock()463	st.Capabilities = a.capabilities()464	if err := a.dir.SaveStatus(st); err != nil {465		a.log.Warn("cannot write status file", "err", err)466	}467}468469// SpoolDir is the spool location inside a state directory.470func SpoolDir(d *state.Dir) string { return filepath.Join(d.Path, "spool") }471
Files main
Menu

Theme