SPB Git

Repository Public

collect.go 19.7 KB 714 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/agent/collect/collect.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Package collect measures the node. Every figure comes from the operating43// system; a metric that cannot be measured is declared in the availability44// map (unsupported / permission_denied / unavailable) and produces no sample.45// Nothing is ever estimated or defaulted to zero.46package collect4748import (49	"context"50	"errors"51	"net"52	"os"53	"sort"54	"strings"55	"sync"56	"syscall"57	"time"5859	"github.com/shirou/gopsutil/v4/cpu"60	"github.com/shirou/gopsutil/v4/disk"61	"github.com/shirou/gopsutil/v4/host"62	"github.com/shirou/gopsutil/v4/load"63	gnet "github.com/shirou/gopsutil/v4/net"6465	"clustr.spboucher.ai/mcc/internal/agent/runcmd"66	"clustr.spboucher.ai/mcc/internal/protocol"67)6869var errUnsupported = errors.New("unsupported on this platform")7071// OSKind is "mac" or "linux".72const OSKind = osKind7374type memStats struct {75	Total, Used                  uint6476	Wired, Compressed            uint6477	HasWired, HasCompressed      bool78	PressureLevel, AvailablePct  float6479	HasPressure, HasAvailablePct bool80}8182// Round is the outcome of one collection pass.83type Round struct {84	Samples   []protocol.Sample85	Processes []protocol.Process // nil when the process table was not sampled this round86	Volumes   []protocol.Volume  // nil when volumes were not sampled this round87	SelfCPU   *float6488	SelfRSS   *int6489}9091type netCounters struct{ rx, tx, errs uint64 }9293// Collector keeps the state needed to turn counters into rates. It is used94// from one goroutine (the report loop); its sub-collectors run concurrently95// inside Collect and synchronise on mu.96type Collector struct {97	// IngestHost returns the host currently used to reach the ingest service98	// (ICMP target "ingest"); empty to skip.99	IngestHost func() string100101	mu    sync.Mutex102	round *Round103	avail map[string]string104	last  map[string]time.Time // group -> last collection105106	prevCPU   *cpu.TimesStat107	prevDiskR uint64108	prevDiskW uint64109	prevDisk  time.Time110	prevNet   map[string]netCounters111	prevNetAt time.Time112	prevPS    map[int32]float64113	prevPSAt  time.Time114	selfCPU   time.Duration115	selfAt    time.Time116117	gateway   string118	gatewayAt time.Time119	defIface  string120}121122// New returns a collector. thermal.temp and power.draw are declared123// unsupported from the start: they need root (powermetrics / SMC) and the124// agent never runs as root.125func New() *Collector {126	c := &Collector{avail: map[string]string{}, last: map[string]time.Time{}}127	c.avail["thermal.temp"] = protocol.AvailUnsupported128	c.avail["power.draw"] = protocol.AvailUnsupported129	return c130}131132// Availability returns a copy of the current availability map: every metric133// that is not being measured, with the reason.134func (c *Collector) Availability() map[string]string {135	c.mu.Lock()136	defer c.mu.Unlock()137	out := make(map[string]string, len(c.avail))138	for k, v := range c.avail {139		out[k] = v140	}141	return out142}143144func (c *Collector) put(name, label string, v float64) {145	c.mu.Lock()146	c.round.Samples = append(c.round.Samples, protocol.Sample{M: name, L: label, V: v})147	delete(c.avail, name)148	c.mu.Unlock()149}150151func (c *Collector) missing(state string, names ...string) {152	c.mu.Lock()153	for _, n := range names {154		c.avail[n] = state155	}156	c.mu.Unlock()157}158159// fail classifies a collector error.160func (c *Collector) fail(err error, names ...string) {161	switch {162	case errors.Is(err, errUnsupported):163		c.missing(protocol.AvailUnsupported, names...)164	case errors.Is(err, os.ErrPermission), errors.Is(err, syscall.EPERM), errors.Is(err, syscall.EACCES):165		c.missing(protocol.AvailPermissionDenied, names...)166	default:167		c.missing(protocol.AvailUnavailable, names...)168	}169}170171// due reports whether a group with the given interval must run now, and172// marks it as run. A small tolerance absorbs timer jitter.173func (c *Collector) due(group string, every time.Duration, now time.Time) bool {174	if last, ok := c.last[group]; ok && now.Sub(last) < every-1500*time.Millisecond {175		return false176	}177	c.last[group] = now178	return true179}180181// Collect runs every collector that is due and returns the samples. It182// never blocks longer than a few seconds: external probes have their own183// deadlines and run concurrently.184func (c *Collector) Collect(ctx context.Context, now time.Time) *Round {185	ctx, cancel := context.WithTimeout(ctx, 7*time.Second)186	defer cancel()187	r := &Round{}188	c.mu.Lock()189	c.round = r190	c.mu.Unlock()191192	due30 := c.due("30s", 30*time.Second, now)193	due60 := c.due("60s", 60*time.Second, now)194195	var wg sync.WaitGroup196	goSafe := func(names []string, fn func()) {197		wg.Add(1)198		go func() {199			defer wg.Done()200			defer func() {201				if recover() != nil {202					c.missing(protocol.AvailUnavailable, names...)203				}204			}()205			fn()206		}()207	}208	// 10 s group.209	goSafe([]string{"cpu.usage", "cpu.user", "cpu.system"}, c.collectCPU)210	goSafe([]string{"load.1", "load.5", "load.15"}, c.collectLoad)211	goSafe([]string{"mem.used", "mem.wired", "mem.compressed", "mem.pressure.level", "mem.available.percent"}, func() { c.collectMemory(due60) })212	goSafe([]string{"swap.used", "swap.total"}, func() { c.collectSwap(ctx, due60) })213	goSafe([]string{"disk.read", "disk.write"}, func() { c.collectDiskIO(now) })214	goSafe([]string{"net.rx", "net.tx", "net.errors"}, func() { c.collectNet(now) })215	goSafe([]string{"gpu.usage", "gpu.mem.used"}, func() { c.collectGPU(ctx) })216	if due30 {217		goSafe([]string{"net.latency", "net.loss"}, func() { c.collectPing(ctx, now) })218		goSafe([]string{"thermal.pressure"}, c.collectThermal)219		goSafe([]string{"procs.count", "agent.cpu", "agent.rss"}, func() { c.collectProcesses(ctx, now) })220	}221	if due60 {222		goSafe([]string{"disk.used", "disk.total"}, func() { c.collectVolumes(ctx) })223		goSafe([]string{"battery.charge"}, func() { c.collectBattery(ctx) })224		goSafe([]string{"uptime"}, func() { c.collectUptime(now) })225	}226	wg.Wait()227	c.mu.Lock()228	c.round = nil229	c.mu.Unlock()230	sort.SliceStable(r.Samples, func(i, j int) bool {231		if r.Samples[i].M != r.Samples[j].M {232			return r.Samples[i].M < r.Samples[j].M233		}234		return r.Samples[i].L < r.Samples[j].L235	})236	return r237}238239func (c *Collector) collectCPU() {240	names := []string{"cpu.usage", "cpu.user", "cpu.system"}241	ts, err := cpu.Times(false)242	if err != nil || len(ts) == 0 {243		c.fail(err, names...)244		return245	}246	cur := ts[0]247	prev := c.prevCPU248	c.prevCPU = &cur249	if prev == nil {250		return // first round: a share needs two readings251	}252	total := func(t *cpu.TimesStat) float64 {253		return t.User + t.System + t.Idle + t.Nice + t.Iowait + t.Irq + t.Softirq + t.Steal254	}255	dTotal := total(&cur) - total(prev)256	dIdle := (cur.Idle + cur.Iowait) - (prev.Idle + prev.Iowait)257	dUser := (cur.User + cur.Nice) - (prev.User + prev.Nice)258	dSys := (cur.System + cur.Irq + cur.Softirq) - (prev.System + prev.Irq + prev.Softirq)259	if dTotal <= 0 || dIdle < 0 || dUser < 0 || dSys < 0 {260		return // counters went backwards (sleep/wake): skip this round261	}262	clamp := func(v float64) float64 { return max(0, min(100, v)) }263	c.put("cpu.usage", "", clamp(100*(dTotal-dIdle)/dTotal))264	c.put("cpu.user", "", clamp(100*dUser/dTotal))265	c.put("cpu.system", "", clamp(100*dSys/dTotal))266}267268func (c *Collector) collectLoad() {269	l, err := load.Avg()270	if err != nil {271		c.fail(err, "load.1", "load.5", "load.15")272		return273	}274	c.put("load.1", "", l.Load1)275	c.put("load.5", "", l.Load5)276	c.put("load.15", "", l.Load15)277}278279func (c *Collector) collectMemory(withTotal bool) {280	m, err := platformMemory()281	if err != nil {282		c.fail(err, "mem.used", "mem.total", "mem.wired", "mem.compressed", "mem.pressure.level", "mem.available.percent")283		return284	}285	c.put("mem.used", "", float64(m.Used))286	if withTotal {287		c.put("mem.total", "", float64(m.Total))288	}289	opt := func(name string, has bool, v float64) {290		switch {291		case has:292			c.put(name, "", v)293		case OSKind == "linux":294			c.missing(protocol.AvailUnsupported, name) // macOS-only notion295		default:296			c.missing(protocol.AvailUnavailable, name)297		}298	}299	opt("mem.wired", m.HasWired, float64(m.Wired))300	opt("mem.compressed", m.HasCompressed, float64(m.Compressed))301	opt("mem.pressure.level", m.HasPressure, m.PressureLevel)302	if m.HasAvailablePct {303		c.put("mem.available.percent", "", m.AvailablePct)304	} else {305		c.missing(protocol.AvailUnavailable, "mem.available.percent")306	}307}308309func (c *Collector) collectSwap(ctx context.Context, withTotal bool) {310	used, total, err := platformSwap(ctx)311	if err != nil {312		c.fail(err, "swap.used", "swap.total")313		return314	}315	c.put("swap.used", "", float64(used))316	if withTotal {317		c.put("swap.total", "", float64(total))318	}319}320321func (c *Collector) collectDiskIO(now time.Time) {322	names := []string{"disk.read", "disk.write"}323	counters, err := disk.IOCounters()324	if err != nil {325		c.fail(err, names...)326		return327	}328	var r, w uint64329	n := 0330	for name, st := range counters {331		if !physicalDisk(name) {332			continue333		}334		r += st.ReadBytes335		w += st.WriteBytes336		n++337	}338	if n == 0 {339		c.missing(protocol.AvailUnavailable, names...)340		return341	}342	prevR, prevW, prevAt := c.prevDiskR, c.prevDiskW, c.prevDisk343	c.prevDiskR, c.prevDiskW, c.prevDisk = r, w, now344	dt := now.Sub(prevAt).Seconds()345	if prevAt.IsZero() || dt <= 0 || dt > 300 || r < prevR || w < prevW {346		return // first round, long pause or counter reset: re-baseline, no rate347	}348	c.put("disk.read", "", float64(r-prevR)/dt)349	c.put("disk.write", "", float64(w-prevW)/dt)350}351352// usableAddr reports whether an interface address is worth reporting a rate353// for: not loopback, not link-local only.354func usableAddr(a net.Addr) bool {355	ipn, ok := a.(*net.IPNet)356	if !ok {357		return false358	}359	ip := ipn.IP360	return !ip.IsLoopback() && !ip.IsLinkLocalUnicast() && !ip.IsLinkLocalMulticast() && !ip.IsUnspecified()361}362363// activeInterfaces returns up, non-loopback interfaces carrying a routable364// address, the default-route interface first, at most MaxInterfaces.365func (c *Collector) activeInterfaces() []net.Interface {366	ifs, err := net.Interfaces()367	if err != nil {368		return nil369	}370	var out []net.Interface371	for _, ifc := range ifs {372		if ifc.Flags&net.FlagUp == 0 || ifc.Flags&net.FlagLoopback != 0 || skipInterface(ifc) {373			continue374		}375		addrs, err := ifc.Addrs()376		if err != nil {377			continue378		}379		ok := false380		for _, a := range addrs {381			if usableAddr(a) {382				ok = true383				break384			}385		}386		if ok {387			out = append(out, ifc)388		}389	}390	def := c.defIface391	sort.SliceStable(out, func(i, j int) bool {392		if (out[i].Name == def) != (out[j].Name == def) {393			return out[i].Name == def394		}395		return out[i].Name < out[j].Name396	})397	if len(out) > protocol.MaxInterfaces {398		out = out[:protocol.MaxInterfaces]399	}400	return out401}402403func (c *Collector) collectNet(now time.Time) {404	names := []string{"net.rx", "net.tx", "net.errors"}405	stats, err := gnet.IOCounters(true)406	if err != nil {407		c.fail(err, names...)408		return409	}410	byName := map[string]gnet.IOCountersStat{}411	for _, s := range stats {412		byName[s.Name] = s413	}414	cur := map[string]netCounters{}415	for _, ifc := range c.activeInterfaces() {416		if s, ok := byName[ifc.Name]; ok {417			cur[ifc.Name] = netCounters{s.BytesRecv, s.BytesSent, s.Errin + s.Errout}418		}419	}420	prev, prevAt := c.prevNet, c.prevNetAt421	c.prevNet, c.prevNetAt = cur, now422	if len(cur) == 0 {423		c.missing(protocol.AvailUnavailable, names...)424		return425	}426	dt := now.Sub(prevAt).Seconds()427	if prev == nil || dt <= 0 || dt > 300 {428		return429	}430	emitted := false431	for name, n := range cur {432		p, ok := prev[name]433		if !ok || n.rx < p.rx || n.tx < p.tx || n.errs < p.errs {434			continue // new interface or counter reset: baseline only435		}436		label := name437		if len(label) > protocol.MaxLabelLen {438			continue439		}440		c.put("net.rx", label, float64(n.rx-p.rx)/dt)441		c.put("net.tx", label, float64(n.tx-p.tx)/dt)442		c.put("net.errors", label, float64(n.errs-p.errs)/dt)443		emitted = true444	}445	_ = emitted446}447448func (c *Collector) collectGPU(ctx context.Context) {449	g, err := platformGPU(ctx)450	if err != nil {451		c.fail(err, "gpu.usage", "gpu.mem.used")452		return453	}454	if g.HasUtilization {455		c.put("gpu.usage", "", g.UtilizationPct)456	} else {457		c.missing(protocol.AvailUnavailable, "gpu.usage")458	}459	if g.HasMem {460		c.put("gpu.mem.used", "", float64(g.MemUsed))461	} else {462		c.missing(protocol.AvailUnavailable, "gpu.mem.used")463	}464}465466func (c *Collector) collectThermal() {467	v, err := platformThermal()468	if err != nil {469		c.fail(err, "thermal.pressure")470		return471	}472	c.put("thermal.pressure", "", v)473}474475func (c *Collector) collectBattery(ctx context.Context) {476	pct, present, err := platformBattery(ctx)477	switch {478	case err != nil:479		c.fail(err, "battery.charge")480	case !present:481		c.missing(protocol.AvailUnsupported, "battery.charge") // no battery in this machine482	default:483		c.put("battery.charge", "", pct)484	}485}486487func (c *Collector) collectUptime(now time.Time) {488	boot, err := host.BootTime()489	if err != nil || boot == 0 {490		c.fail(err, "uptime")491		return492	}493	up := now.Unix() - int64(boot)494	if up < 0 {495		c.missing(protocol.AvailUnavailable, "uptime")496		return497	}498	c.put("uptime", "", float64(up))499}500501func (c *Collector) collectVolumes(ctx context.Context) {502	vols, err := platformVolumes(ctx)503	if err != nil {504		c.fail(err, "disk.used", "disk.total")505		return506	}507	sort.SliceStable(vols, func(i, j int) bool {508		if (vols[i].Mount == "/") != (vols[j].Mount == "/") {509			return vols[i].Mount == "/"510		}511		return vols[i].Mount < vols[j].Mount512	})513	kept := vols[:0]514	for _, v := range vols {515		if len(v.Mount) > protocol.MaxLabelLen || len(kept) >= protocol.MaxVolumes {516			continue517		}518		kept = append(kept, v)519	}520	if len(kept) == 0 {521		c.missing(protocol.AvailUnavailable, "disk.used", "disk.total")522		return523	}524	for _, v := range kept {525		c.put("disk.used", v.Mount, float64(v.Used))526		c.put("disk.total", v.Mount, float64(v.Total))527	}528	c.mu.Lock()529	c.round.Volumes = kept530	c.mu.Unlock()531}532533// Gateway returns the default gateway and interface (cached one minute).534func (c *Collector) Gateway(ctx context.Context, now time.Time) (string, string) {535	c.mu.Lock()536	gw, at, ifc := c.gateway, c.gatewayAt, c.defIface537	c.mu.Unlock()538	if now.Sub(at) < time.Minute {539		return gw, ifc540	}541	gw, ifc, err := platformGateway(ctx)542	if err != nil {543		gw = ""544	}545	c.mu.Lock()546	c.gateway, c.defIface, c.gatewayAt = gw, ifc, now547	c.mu.Unlock()548	return gw, ifc549}550551// Ping runs one unprivileged ICMP probe (3 echoes).552func Ping(ctx context.Context, target string) (PingResult, error) {553	if ip := net.ParseIP(target); ip == nil {554		return PingResult{}, errors.New("ping target must be an IP address")555	}556	res, err := runcmd.Run(ctx, 6*time.Second, 16<<10, nil, pingCommand(target)...)557	if err != nil {558		return PingResult{}, err559	}560	// ping exits non-zero on loss; the summary is still printed.561	return ParsePing(string(res.Stdout))562}563564func (c *Collector) collectPing(ctx context.Context, now time.Time) {565	targets := map[string]string{}566	if gw, _ := c.Gateway(ctx, now); gw != "" && net.ParseIP(gw) != nil {567		targets["gateway"] = gw568	}569	if c.IngestHost != nil {570		if h := c.IngestHost(); h != "" {571			if ip := net.ParseIP(h); ip != nil {572				targets["ingest"] = h573			} else if addrs, err := net.DefaultResolver.LookupHost(ctx, h); err == nil && len(addrs) > 0 {574				targets["ingest"] = addrs[0]575			}576		}577	}578	if len(targets) == 0 {579		c.missing(protocol.AvailUnavailable, "net.latency", "net.loss")580		return581	}582	type res struct {583		label string584		r     PingResult585		err   error586	}587	ch := make(chan res, len(targets))588	for label, t := range targets {589		go func() {590			r, err := Ping(ctx, t)591			ch <- res{label, r, err}592		}()593	}594	anyLoss, anyRTT := false, false595	for range targets {596		x := <-ch597		if x.err != nil {598			continue599		}600		c.put("net.loss", x.label, x.r.LossPct)601		anyLoss = true602		if x.r.HasRTT {603			// A target that answered nothing has no latency: no sample, never 0.604			c.put("net.latency", x.label, x.r.AvgMS)605			anyRTT = true606		}607	}608	if !anyLoss {609		c.missing(protocol.AvailUnavailable, "net.loss")610	}611	if !anyRTT {612		c.missing(protocol.AvailUnavailable, "net.latency")613	}614}615616// ListProcesses returns the process table (pid, user, rss, cpu time, name).617func ListProcesses(ctx context.Context) ([]PSRow, error) {618	out, err := runcmd.Output(ctx, 5*time.Second, "/bin/ps", "-axo", "pid=,user=,rss=,time=,comm=")619	if err != nil {620		return nil, err621	}622	rows := ParsePS(string(out))623	if len(rows) == 0 {624		return nil, errors.New("ps returned no process")625	}626	return rows, nil627}628629func (c *Collector) collectProcesses(ctx context.Context, now time.Time) {630	// The agent's own CPU: getrusage delta, independent of ps.631	var ru syscall.Rusage632	if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err == nil {633		cur := time.Duration(ru.Utime.Nano() + ru.Stime.Nano())634		if !c.selfAt.IsZero() {635			if dt := now.Sub(c.selfAt); dt > 0 {636				v := 100 * float64(cur-c.selfCPU) / float64(dt)637				c.put("agent.cpu", "", v)638				c.mu.Lock()639				c.round.SelfCPU = &v640				c.mu.Unlock()641			}642		}643		c.selfCPU, c.selfAt = cur, now644	} else {645		c.fail(err, "agent.cpu")646	}647648	rows, err := ListProcesses(ctx)649	if err != nil {650		c.fail(err, "procs.count", "agent.rss")651		return652	}653	c.put("procs.count", "", float64(len(rows)))654	self := int32(os.Getpid())655	prev, prevAt := c.prevPS, c.prevPSAt656	cur := make(map[int32]float64, len(rows))657	dt := now.Sub(prevAt).Seconds()658	haveCPU := prev != nil && dt > 1 && dt < 300659	procs := make([]protocol.Process, 0, len(rows))660	foundSelf := false661	for _, r := range rows {662		cur[r.PID] = r.CPUTime663		if r.PID == self {664			foundSelf = true665			rss := r.RSS666			c.put("agent.rss", "", float64(rss))667			c.mu.Lock()668			c.round.SelfRSS = &rss669			c.mu.Unlock()670		}671		if !haveCPU {672			continue673		}674		p, known := prev[r.PID]675		var pct float64676		switch {677		case known && r.CPUTime >= p:678			pct = 100 * (r.CPUTime - p) / dt679		case !known:680			pct = 100 * r.CPUTime / dt // started during the window681		default:682			continue // pid reused: no meaningful delta683		}684		procs = append(procs, protocol.Process{PID: r.PID, Name: clipName(r.Name), User: r.User, CPU: pct, RSS: r.RSS})685	}686	if !foundSelf {687		c.missing(protocol.AvailUnavailable, "agent.rss")688	}689	c.prevPS, c.prevPSAt = cur, now690	if !haveCPU {691		return // first sampling: CPU shares need two readings, so no table yet692	}693	sort.Slice(procs, func(i, j int) bool {694		if procs[i].CPU != procs[j].CPU {695			return procs[i].CPU > procs[j].CPU696		}697		return procs[i].RSS > procs[j].RSS698	})699	if len(procs) > protocol.MaxProcesses {700		procs = procs[:protocol.MaxProcesses]701	}702	c.mu.Lock()703	c.round.Processes = procs704	c.mu.Unlock()705}706707func clipName(s string) string {708	s = strings.ToValidUTF8(s, "")709	if len(s) > 64 {710		s = strings.ToValidUTF8(s[:64], "")711	}712	return s713}714
Files main
Menu

Theme