Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/agent/collect/collect.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Package collect measures the node. Every figure comes from the operating43// system; a metric that cannot be measured is declared in the availability44// map (unsupported / permission_denied / unavailable) and produces no sample.45// Nothing is ever estimated or defaulted to zero.46package collect4748import (49 "context"50 "errors"51 "net"52 "os"53 "sort"54 "strings"55 "sync"56 "syscall"57 "time"5859 "github.com/shirou/gopsutil/v4/cpu"60 "github.com/shirou/gopsutil/v4/disk"61 "github.com/shirou/gopsutil/v4/host"62 "github.com/shirou/gopsutil/v4/load"63 gnet "github.com/shirou/gopsutil/v4/net"6465 "clustr.spboucher.ai/mcc/internal/agent/runcmd"66 "clustr.spboucher.ai/mcc/internal/protocol"67)6869var errUnsupported = errors.New("unsupported on this platform")7071// OSKind is "mac" or "linux".72const OSKind = osKind7374type memStats struct {75 Total, Used uint6476 Wired, Compressed uint6477 HasWired, HasCompressed bool78 PressureLevel, AvailablePct float6479 HasPressure, HasAvailablePct bool80}8182// Round is the outcome of one collection pass.83type Round struct {84 Samples []protocol.Sample85 Processes []protocol.Process // nil when the process table was not sampled this round86 Volumes []protocol.Volume // nil when volumes were not sampled this round87 SelfCPU *float6488 SelfRSS *int6489}9091type netCounters struct{ rx, tx, errs uint64 }9293// Collector keeps the state needed to turn counters into rates. It is used94// from one goroutine (the report loop); its sub-collectors run concurrently95// inside Collect and synchronise on mu.96type Collector struct {97 // IngestHost returns the host currently used to reach the ingest service98 // (ICMP target "ingest"); empty to skip.99 IngestHost func() string100101 mu sync.Mutex102 round *Round103 avail map[string]string104 last map[string]time.Time // group -> last collection105106 prevCPU *cpu.TimesStat107 prevDiskR uint64108 prevDiskW uint64109 prevDisk time.Time110 prevNet map[string]netCounters111 prevNetAt time.Time112 prevPS map[int32]float64113 prevPSAt time.Time114 selfCPU time.Duration115 selfAt time.Time116117 gateway string118 gatewayAt time.Time119 defIface string120}121122// New returns a collector. thermal.temp and power.draw are declared123// unsupported from the start: they need root (powermetrics / SMC) and the124// agent never runs as root.125func New() *Collector {126 c := &Collector{avail: map[string]string{}, last: map[string]time.Time{}}127 c.avail["thermal.temp"] = protocol.AvailUnsupported128 c.avail["power.draw"] = protocol.AvailUnsupported129 return c130}131132// Availability returns a copy of the current availability map: every metric133// that is not being measured, with the reason.134func (c *Collector) Availability() map[string]string {135 c.mu.Lock()136 defer c.mu.Unlock()137 out := make(map[string]string, len(c.avail))138 for k, v := range c.avail {139 out[k] = v140 }141 return out142}143144func (c *Collector) put(name, label string, v float64) {145 c.mu.Lock()146 c.round.Samples = append(c.round.Samples, protocol.Sample{M: name, L: label, V: v})147 delete(c.avail, name)148 c.mu.Unlock()149}150151func (c *Collector) missing(state string, names ...string) {152 c.mu.Lock()153 for _, n := range names {154 c.avail[n] = state155 }156 c.mu.Unlock()157}158159// fail classifies a collector error.160func (c *Collector) fail(err error, names ...string) {161 switch {162 case errors.Is(err, errUnsupported):163 c.missing(protocol.AvailUnsupported, names...)164 case errors.Is(err, os.ErrPermission), errors.Is(err, syscall.EPERM), errors.Is(err, syscall.EACCES):165 c.missing(protocol.AvailPermissionDenied, names...)166 default:167 c.missing(protocol.AvailUnavailable, names...)168 }169}170171// due reports whether a group with the given interval must run now, and172// marks it as run. A small tolerance absorbs timer jitter.173func (c *Collector) due(group string, every time.Duration, now time.Time) bool {174 if last, ok := c.last[group]; ok && now.Sub(last) < every-1500*time.Millisecond {175 return false176 }177 c.last[group] = now178 return true179}180181// Collect runs every collector that is due and returns the samples. It182// never blocks longer than a few seconds: external probes have their own183// deadlines and run concurrently.184func (c *Collector) Collect(ctx context.Context, now time.Time) *Round {185 ctx, cancel := context.WithTimeout(ctx, 7*time.Second)186 defer cancel()187 r := &Round{}188 c.mu.Lock()189 c.round = r190 c.mu.Unlock()191192 due30 := c.due("30s", 30*time.Second, now)193 due60 := c.due("60s", 60*time.Second, now)194195 var wg sync.WaitGroup196 goSafe := func(names []string, fn func()) {197 wg.Add(1)198 go func() {199 defer wg.Done()200 defer func() {201 if recover() != nil {202 c.missing(protocol.AvailUnavailable, names...)203 }204 }()205 fn()206 }()207 }208 // 10 s group.209 goSafe([]string{"cpu.usage", "cpu.user", "cpu.system"}, c.collectCPU)210 goSafe([]string{"load.1", "load.5", "load.15"}, c.collectLoad)211 goSafe([]string{"mem.used", "mem.wired", "mem.compressed", "mem.pressure.level", "mem.available.percent"}, func() { c.collectMemory(due60) })212 goSafe([]string{"swap.used", "swap.total"}, func() { c.collectSwap(ctx, due60) })213 goSafe([]string{"disk.read", "disk.write"}, func() { c.collectDiskIO(now) })214 goSafe([]string{"net.rx", "net.tx", "net.errors"}, func() { c.collectNet(now) })215 goSafe([]string{"gpu.usage", "gpu.mem.used"}, func() { c.collectGPU(ctx) })216 if due30 {217 goSafe([]string{"net.latency", "net.loss"}, func() { c.collectPing(ctx, now) })218 goSafe([]string{"thermal.pressure"}, c.collectThermal)219 goSafe([]string{"procs.count", "agent.cpu", "agent.rss"}, func() { c.collectProcesses(ctx, now) })220 }221 if due60 {222 goSafe([]string{"disk.used", "disk.total"}, func() { c.collectVolumes(ctx) })223 goSafe([]string{"battery.charge"}, func() { c.collectBattery(ctx) })224 goSafe([]string{"uptime"}, func() { c.collectUptime(now) })225 }226 wg.Wait()227 c.mu.Lock()228 c.round = nil229 c.mu.Unlock()230 sort.SliceStable(r.Samples, func(i, j int) bool {231 if r.Samples[i].M != r.Samples[j].M {232 return r.Samples[i].M < r.Samples[j].M233 }234 return r.Samples[i].L < r.Samples[j].L235 })236 return r237}238239func (c *Collector) collectCPU() {240 names := []string{"cpu.usage", "cpu.user", "cpu.system"}241 ts, err := cpu.Times(false)242 if err != nil || len(ts) == 0 {243 c.fail(err, names...)244 return245 }246 cur := ts[0]247 prev := c.prevCPU248 c.prevCPU = &cur249 if prev == nil {250 return // first round: a share needs two readings251 }252 total := func(t *cpu.TimesStat) float64 {253 return t.User + t.System + t.Idle + t.Nice + t.Iowait + t.Irq + t.Softirq + t.Steal254 }255 dTotal := total(&cur) - total(prev)256 dIdle := (cur.Idle + cur.Iowait) - (prev.Idle + prev.Iowait)257 dUser := (cur.User + cur.Nice) - (prev.User + prev.Nice)258 dSys := (cur.System + cur.Irq + cur.Softirq) - (prev.System + prev.Irq + prev.Softirq)259 if dTotal <= 0 || dIdle < 0 || dUser < 0 || dSys < 0 {260 return // counters went backwards (sleep/wake): skip this round261 }262 clamp := func(v float64) float64 { return max(0, min(100, v)) }263 c.put("cpu.usage", "", clamp(100*(dTotal-dIdle)/dTotal))264 c.put("cpu.user", "", clamp(100*dUser/dTotal))265 c.put("cpu.system", "", clamp(100*dSys/dTotal))266}267268func (c *Collector) collectLoad() {269 l, err := load.Avg()270 if err != nil {271 c.fail(err, "load.1", "load.5", "load.15")272 return273 }274 c.put("load.1", "", l.Load1)275 c.put("load.5", "", l.Load5)276 c.put("load.15", "", l.Load15)277}278279func (c *Collector) collectMemory(withTotal bool) {280 m, err := platformMemory()281 if err != nil {282 c.fail(err, "mem.used", "mem.total", "mem.wired", "mem.compressed", "mem.pressure.level", "mem.available.percent")283 return284 }285 c.put("mem.used", "", float64(m.Used))286 if withTotal {287 c.put("mem.total", "", float64(m.Total))288 }289 opt := func(name string, has bool, v float64) {290 switch {291 case has:292 c.put(name, "", v)293 case OSKind == "linux":294 c.missing(protocol.AvailUnsupported, name) // macOS-only notion295 default:296 c.missing(protocol.AvailUnavailable, name)297 }298 }299 opt("mem.wired", m.HasWired, float64(m.Wired))300 opt("mem.compressed", m.HasCompressed, float64(m.Compressed))301 opt("mem.pressure.level", m.HasPressure, m.PressureLevel)302 if m.HasAvailablePct {303 c.put("mem.available.percent", "", m.AvailablePct)304 } else {305 c.missing(protocol.AvailUnavailable, "mem.available.percent")306 }307}308309func (c *Collector) collectSwap(ctx context.Context, withTotal bool) {310 used, total, err := platformSwap(ctx)311 if err != nil {312 c.fail(err, "swap.used", "swap.total")313 return314 }315 c.put("swap.used", "", float64(used))316 if withTotal {317 c.put("swap.total", "", float64(total))318 }319}320321func (c *Collector) collectDiskIO(now time.Time) {322 names := []string{"disk.read", "disk.write"}323 counters, err := disk.IOCounters()324 if err != nil {325 c.fail(err, names...)326 return327 }328 var r, w uint64329 n := 0330 for name, st := range counters {331 if !physicalDisk(name) {332 continue333 }334 r += st.ReadBytes335 w += st.WriteBytes336 n++337 }338 if n == 0 {339 c.missing(protocol.AvailUnavailable, names...)340 return341 }342 prevR, prevW, prevAt := c.prevDiskR, c.prevDiskW, c.prevDisk343 c.prevDiskR, c.prevDiskW, c.prevDisk = r, w, now344 dt := now.Sub(prevAt).Seconds()345 if prevAt.IsZero() || dt <= 0 || dt > 300 || r < prevR || w < prevW {346 return // first round, long pause or counter reset: re-baseline, no rate347 }348 c.put("disk.read", "", float64(r-prevR)/dt)349 c.put("disk.write", "", float64(w-prevW)/dt)350}351352// usableAddr reports whether an interface address is worth reporting a rate353// for: not loopback, not link-local only.354func usableAddr(a net.Addr) bool {355 ipn, ok := a.(*net.IPNet)356 if !ok {357 return false358 }359 ip := ipn.IP360 return !ip.IsLoopback() && !ip.IsLinkLocalUnicast() && !ip.IsLinkLocalMulticast() && !ip.IsUnspecified()361}362363// activeInterfaces returns up, non-loopback interfaces carrying a routable364// address, the default-route interface first, at most MaxInterfaces.365func (c *Collector) activeInterfaces() []net.Interface {366 ifs, err := net.Interfaces()367 if err != nil {368 return nil369 }370 var out []net.Interface371 for _, ifc := range ifs {372 if ifc.Flags&net.FlagUp == 0 || ifc.Flags&net.FlagLoopback != 0 || skipInterface(ifc) {373 continue374 }375 addrs, err := ifc.Addrs()376 if err != nil {377 continue378 }379 ok := false380 for _, a := range addrs {381 if usableAddr(a) {382 ok = true383 break384 }385 }386 if ok {387 out = append(out, ifc)388 }389 }390 def := c.defIface391 sort.SliceStable(out, func(i, j int) bool {392 if (out[i].Name == def) != (out[j].Name == def) {393 return out[i].Name == def394 }395 return out[i].Name < out[j].Name396 })397 if len(out) > protocol.MaxInterfaces {398 out = out[:protocol.MaxInterfaces]399 }400 return out401}402403func (c *Collector) collectNet(now time.Time) {404 names := []string{"net.rx", "net.tx", "net.errors"}405 stats, err := gnet.IOCounters(true)406 if err != nil {407 c.fail(err, names...)408 return409 }410 byName := map[string]gnet.IOCountersStat{}411 for _, s := range stats {412 byName[s.Name] = s413 }414 cur := map[string]netCounters{}415 for _, ifc := range c.activeInterfaces() {416 if s, ok := byName[ifc.Name]; ok {417 cur[ifc.Name] = netCounters{s.BytesRecv, s.BytesSent, s.Errin + s.Errout}418 }419 }420 prev, prevAt := c.prevNet, c.prevNetAt421 c.prevNet, c.prevNetAt = cur, now422 if len(cur) == 0 {423 c.missing(protocol.AvailUnavailable, names...)424 return425 }426 dt := now.Sub(prevAt).Seconds()427 if prev == nil || dt <= 0 || dt > 300 {428 return429 }430 emitted := false431 for name, n := range cur {432 p, ok := prev[name]433 if !ok || n.rx < p.rx || n.tx < p.tx || n.errs < p.errs {434 continue // new interface or counter reset: baseline only435 }436 label := name437 if len(label) > protocol.MaxLabelLen {438 continue439 }440 c.put("net.rx", label, float64(n.rx-p.rx)/dt)441 c.put("net.tx", label, float64(n.tx-p.tx)/dt)442 c.put("net.errors", label, float64(n.errs-p.errs)/dt)443 emitted = true444 }445 _ = emitted446}447448func (c *Collector) collectGPU(ctx context.Context) {449 g, err := platformGPU(ctx)450 if err != nil {451 c.fail(err, "gpu.usage", "gpu.mem.used")452 return453 }454 if g.HasUtilization {455 c.put("gpu.usage", "", g.UtilizationPct)456 } else {457 c.missing(protocol.AvailUnavailable, "gpu.usage")458 }459 if g.HasMem {460 c.put("gpu.mem.used", "", float64(g.MemUsed))461 } else {462 c.missing(protocol.AvailUnavailable, "gpu.mem.used")463 }464}465466func (c *Collector) collectThermal() {467 v, err := platformThermal()468 if err != nil {469 c.fail(err, "thermal.pressure")470 return471 }472 c.put("thermal.pressure", "", v)473}474475func (c *Collector) collectBattery(ctx context.Context) {476 pct, present, err := platformBattery(ctx)477 switch {478 case err != nil:479 c.fail(err, "battery.charge")480 case !present:481 c.missing(protocol.AvailUnsupported, "battery.charge") // no battery in this machine482 default:483 c.put("battery.charge", "", pct)484 }485}486487func (c *Collector) collectUptime(now time.Time) {488 boot, err := host.BootTime()489 if err != nil || boot == 0 {490 c.fail(err, "uptime")491 return492 }493 up := now.Unix() - int64(boot)494 if up < 0 {495 c.missing(protocol.AvailUnavailable, "uptime")496 return497 }498 c.put("uptime", "", float64(up))499}500501func (c *Collector) collectVolumes(ctx context.Context) {502 vols, err := platformVolumes(ctx)503 if err != nil {504 c.fail(err, "disk.used", "disk.total")505 return506 }507 sort.SliceStable(vols, func(i, j int) bool {508 if (vols[i].Mount == "/") != (vols[j].Mount == "/") {509 return vols[i].Mount == "/"510 }511 return vols[i].Mount < vols[j].Mount512 })513 kept := vols[:0]514 for _, v := range vols {515 if len(v.Mount) > protocol.MaxLabelLen || len(kept) >= protocol.MaxVolumes {516 continue517 }518 kept = append(kept, v)519 }520 if len(kept) == 0 {521 c.missing(protocol.AvailUnavailable, "disk.used", "disk.total")522 return523 }524 for _, v := range kept {525 c.put("disk.used", v.Mount, float64(v.Used))526 c.put("disk.total", v.Mount, float64(v.Total))527 }528 c.mu.Lock()529 c.round.Volumes = kept530 c.mu.Unlock()531}532533// Gateway returns the default gateway and interface (cached one minute).534func (c *Collector) Gateway(ctx context.Context, now time.Time) (string, string) {535 c.mu.Lock()536 gw, at, ifc := c.gateway, c.gatewayAt, c.defIface537 c.mu.Unlock()538 if now.Sub(at) < time.Minute {539 return gw, ifc540 }541 gw, ifc, err := platformGateway(ctx)542 if err != nil {543 gw = ""544 }545 c.mu.Lock()546 c.gateway, c.defIface, c.gatewayAt = gw, ifc, now547 c.mu.Unlock()548 return gw, ifc549}550551// Ping runs one unprivileged ICMP probe (3 echoes).552func Ping(ctx context.Context, target string) (PingResult, error) {553 if ip := net.ParseIP(target); ip == nil {554 return PingResult{}, errors.New("ping target must be an IP address")555 }556 res, err := runcmd.Run(ctx, 6*time.Second, 16<<10, nil, pingCommand(target)...)557 if err != nil {558 return PingResult{}, err559 }560 // ping exits non-zero on loss; the summary is still printed.561 return ParsePing(string(res.Stdout))562}563564func (c *Collector) collectPing(ctx context.Context, now time.Time) {565 targets := map[string]string{}566 if gw, _ := c.Gateway(ctx, now); gw != "" && net.ParseIP(gw) != nil {567 targets["gateway"] = gw568 }569 if c.IngestHost != nil {570 if h := c.IngestHost(); h != "" {571 if ip := net.ParseIP(h); ip != nil {572 targets["ingest"] = h573 } else if addrs, err := net.DefaultResolver.LookupHost(ctx, h); err == nil && len(addrs) > 0 {574 targets["ingest"] = addrs[0]575 }576 }577 }578 if len(targets) == 0 {579 c.missing(protocol.AvailUnavailable, "net.latency", "net.loss")580 return581 }582 type res struct {583 label string584 r PingResult585 err error586 }587 ch := make(chan res, len(targets))588 for label, t := range targets {589 go func() {590 r, err := Ping(ctx, t)591 ch <- res{label, r, err}592 }()593 }594 anyLoss, anyRTT := false, false595 for range targets {596 x := <-ch597 if x.err != nil {598 continue599 }600 c.put("net.loss", x.label, x.r.LossPct)601 anyLoss = true602 if x.r.HasRTT {603 // A target that answered nothing has no latency: no sample, never 0.604 c.put("net.latency", x.label, x.r.AvgMS)605 anyRTT = true606 }607 }608 if !anyLoss {609 c.missing(protocol.AvailUnavailable, "net.loss")610 }611 if !anyRTT {612 c.missing(protocol.AvailUnavailable, "net.latency")613 }614}615616// ListProcesses returns the process table (pid, user, rss, cpu time, name).617func ListProcesses(ctx context.Context) ([]PSRow, error) {618 out, err := runcmd.Output(ctx, 5*time.Second, "/bin/ps", "-axo", "pid=,user=,rss=,time=,comm=")619 if err != nil {620 return nil, err621 }622 rows := ParsePS(string(out))623 if len(rows) == 0 {624 return nil, errors.New("ps returned no process")625 }626 return rows, nil627}628629func (c *Collector) collectProcesses(ctx context.Context, now time.Time) {630 // The agent's own CPU: getrusage delta, independent of ps.631 var ru syscall.Rusage632 if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err == nil {633 cur := time.Duration(ru.Utime.Nano() + ru.Stime.Nano())634 if !c.selfAt.IsZero() {635 if dt := now.Sub(c.selfAt); dt > 0 {636 v := 100 * float64(cur-c.selfCPU) / float64(dt)637 c.put("agent.cpu", "", v)638 c.mu.Lock()639 c.round.SelfCPU = &v640 c.mu.Unlock()641 }642 }643 c.selfCPU, c.selfAt = cur, now644 } else {645 c.fail(err, "agent.cpu")646 }647648 rows, err := ListProcesses(ctx)649 if err != nil {650 c.fail(err, "procs.count", "agent.rss")651 return652 }653 c.put("procs.count", "", float64(len(rows)))654 self := int32(os.Getpid())655 prev, prevAt := c.prevPS, c.prevPSAt656 cur := make(map[int32]float64, len(rows))657 dt := now.Sub(prevAt).Seconds()658 haveCPU := prev != nil && dt > 1 && dt < 300659 procs := make([]protocol.Process, 0, len(rows))660 foundSelf := false661 for _, r := range rows {662 cur[r.PID] = r.CPUTime663 if r.PID == self {664 foundSelf = true665 rss := r.RSS666 c.put("agent.rss", "", float64(rss))667 c.mu.Lock()668 c.round.SelfRSS = &rss669 c.mu.Unlock()670 }671 if !haveCPU {672 continue673 }674 p, known := prev[r.PID]675 var pct float64676 switch {677 case known && r.CPUTime >= p:678 pct = 100 * (r.CPUTime - p) / dt679 case !known:680 pct = 100 * r.CPUTime / dt // started during the window681 default:682 continue // pid reused: no meaningful delta683 }684 procs = append(procs, protocol.Process{PID: r.PID, Name: clipName(r.Name), User: r.User, CPU: pct, RSS: r.RSS})685 }686 if !foundSelf {687 c.missing(protocol.AvailUnavailable, "agent.rss")688 }689 c.prevPS, c.prevPSAt = cur, now690 if !haveCPU {691 return // first sampling: CPU shares need two readings, so no table yet692 }693 sort.Slice(procs, func(i, j int) bool {694 if procs[i].CPU != procs[j].CPU {695 return procs[i].CPU > procs[j].CPU696 }697 return procs[i].RSS > procs[j].RSS698 })699 if len(procs) > protocol.MaxProcesses {700 procs = procs[:protocol.MaxProcesses]701 }702 c.mu.Lock()703 c.round.Processes = procs704 c.mu.Unlock()705}706707func clipName(s string) string {708 s = strings.ToValidUTF8(s, "")709 if len(s) > 64 {710 s = strings.ToValidUTF8(s[:64], "")711 }712 return s713}714