Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/ingest/report.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package ingest4344import (45 "compress/gzip"46 "context"47 "encoding/json"48 "errors"49 "fmt"50 "io"51 "math"52 "net/http"53 "net/netip"54 "strings"55 "time"5657 "github.com/google/uuid"58 "github.com/jackc/pgx/v5"59 "github.com/jackc/pgx/v5/pgconn"6061 "clustr.spboucher.ai/mcc/internal/events"62 "clustr.spboucher.ai/mcc/internal/httpx"63 "clustr.spboucher.ai/mcc/internal/ids"64 "clustr.spboucher.ai/mcc/internal/protocol"65)6667const (68 maxReportBody = 1 << 20 // bytes on the wire69 maxReportPlain = 4 << 20 // bytes after gzip70 maxFutureSkew = 5 * time.Minute71 maxInference = 1672 maxNASProbes = 873)7475var errTooLarge = httpx.Err(http.StatusRequestEntityTooLarge, "payload_too_large", "The request body is too large.")7677// decodeBody reads a bounded JSON body, transparently gunzipping it. Unknown78// fields are ignored: within protocol v1, additions are backward compatible.79func decodeBody(w http.ResponseWriter, r *http.Request, max int64, dst any) error {80 if r.ContentLength > max {81 return errTooLarge82 }83 var src io.Reader = http.MaxBytesReader(w, r.Body, max)84 plainMax := max85 if strings.EqualFold(r.Header.Get("Content-Encoding"), "gzip") {86 zr, err := gzip.NewReader(src)87 if err != nil {88 return bodyErr(err)89 }90 defer zr.Close()91 plainMax = max * 492 src = zr93 } else if ce := r.Header.Get("Content-Encoding"); ce != "" && !strings.EqualFold(ce, "identity") {94 return httpx.Err(http.StatusUnsupportedMediaType, "unsupported_encoding", "Content-Encoding must be gzip or identity.")95 }96 body, err := io.ReadAll(io.LimitReader(src, plainMax+1))97 if err != nil {98 return bodyErr(err)99 }100 if int64(len(body)) > plainMax {101 return errTooLarge102 }103 if err := json.Unmarshal(body, dst); err != nil {104 return httpx.Validation("Malformed JSON body.")105 }106 return nil107}108109func bodyErr(err error) error {110 var mbe *http.MaxBytesError111 if errors.As(err, &mbe) {112 return errTooLarge113 }114 return httpx.Validation("Unreadable request body.")115}116117func invalid(format string, a ...any) *httpx.Error {118 return httpx.Err(http.StatusBadRequest, "invalid_report", fmt.Sprintf(format, a...))119}120121var availStates = map[string]bool{122 protocol.AvailUnsupported: true, protocol.AvailPermissionDenied: true, protocol.AvailUnavailable: true,123}124var serviceManagers = map[string]bool{"pm2": true, "launchd": true, "systemd": true, "docker": true}125var serviceStates = map[string]bool{"online": true, "stopped": true, "errored": true, "unknown": true}126127func finite(v float64) bool { return !math.IsNaN(v) && !math.IsInf(v, 0) }128129func validLabel(l string) bool {130 if len(l) > protocol.MaxLabelLen {131 return false132 }133 for _, r := range l {134 if r < 0x20 || r == 0x7f || r == 0xfffd {135 return false136 }137 }138 return true139}140141// validateReport enforces the protocol strictly. A report that fails here is142// rejected as a whole with 400; the agent drops it (it would never succeed).143func (s *Server) validateReport(rep *protocol.Report, now time.Time) *httpx.Error {144 if rep.V != protocol.Version {145 return httpx.Err(http.StatusBadRequest, "unsupported_version", fmt.Sprintf("Protocol version %d is not supported (expected %d).", rep.V, protocol.Version))146 }147 if rep.Seq <= 0 {148 return invalid("seq must be positive")149 }150 if rep.TS.IsZero() {151 return invalid("ts is required")152 }153 if rep.TS.After(now.Add(maxFutureSkew)) {154 return httpx.Err(http.StatusBadRequest, "clock_skew", "Report timestamp is more than 5 minutes in the future; check the node clock.")155 }156 if rep.TS.Before(now.Add(-s.cfg.RetainRaw)) {157 return httpx.Err(http.StatusBadRequest, "too_old", "Report is older than the raw retention.")158 }159 if len(rep.AgentVersion) > 64 {160 return invalid("agent_version too long")161 }162 if len(rep.Samples) > protocol.MaxSamples {163 return invalid("too many samples (%d > %d)", len(rep.Samples), protocol.MaxSamples)164 }165 type key struct {166 id int16167 l string168 }169 seen := make(map[key]struct{}, len(rep.Samples))170 perMetric := map[string]int{}171 for _, sm := range rep.Samples {172 m, ok := protocol.MetricByName(sm.M)173 if !ok {174 return httpx.Err(http.StatusBadRequest, "unknown_metric", fmt.Sprintf("Unknown metric %q.", clip(sm.M, 64)))175 }176 if !finite(sm.V) {177 return invalid("metric %s: value is not finite", m.Name)178 }179 if !m.Labeled && sm.L != "" {180 return invalid("metric %s takes no label", m.Name)181 }182 if m.Labeled && sm.L == "" {183 return invalid("metric %s requires a label", m.Name)184 }185 if !validLabel(sm.L) {186 return invalid("metric %s: invalid label", m.Name)187 }188 k := key{m.ID, sm.L}189 if _, dup := seen[k]; dup {190 return invalid("metric %s: duplicate sample", m.Name)191 }192 seen[k] = struct{}{}193 perMetric[m.Name]++194 }195 for name, n := range perMetric {196 limit := 1197 switch {198 case strings.HasPrefix(name, "disk."):199 limit = protocol.MaxVolumes200 case strings.HasPrefix(name, "net."):201 limit = protocol.MaxInterfaces202 }203 if n > limit {204 return invalid("metric %s: too many labels (%d > %d)", name, n, limit)205 }206 }207 if len(rep.Availability) > len(protocol.Catalog) {208 return invalid("availability has too many entries")209 }210 for name, st := range rep.Availability {211 if _, ok := protocol.MetricByName(name); !ok {212 return httpx.Err(http.StatusBadRequest, "unknown_metric", fmt.Sprintf("Unknown metric %q in availability.", clip(name, 64)))213 }214 if !availStates[st] {215 return invalid("availability of %s: unknown state", name)216 }217 }218 if len(rep.Processes) > protocol.MaxProcesses {219 return invalid("too many processes (%d > %d)", len(rep.Processes), protocol.MaxProcesses)220 }221 for i := range rep.Processes {222 p := &rep.Processes[i]223 if !finite(p.CPU) || p.CPU < 0 || p.RSS < 0 || p.PID < 0 {224 return invalid("process %d: invalid figures", i)225 }226 p.Name, p.User = clip(p.Name, 64), clip(p.User, 64)227 }228 if sv := rep.Services; sv != nil {229 if len(sv.Items) > protocol.MaxServices {230 return invalid("too many services (%d > %d)", len(sv.Items), protocol.MaxServices)231 }232 if len(sv.Managers) > len(serviceManagers) {233 return invalid("too many service managers")234 }235 observed := map[string]bool{}236 for _, m := range sv.Managers {237 if !serviceManagers[m] {238 return invalid("unknown service manager")239 }240 observed[m] = true241 }242 names := map[string]bool{}243 for i := range sv.Items {244 it := &sv.Items[i]245 if !observed[it.Manager] {246 return invalid("service %d: manager not listed in managers", i)247 }248 if it.Name == "" || len(it.Name) > 200 || !validText(it.Name) {249 return invalid("service %d: invalid name", i)250 }251 if !serviceStates[it.State] {252 return invalid("service %d: unknown state", i)253 }254 if !finite(it.CPU) || it.MemBytes < 0 || it.UptimeS < 0 || it.Restarts < 0 || it.PID < 0 {255 return invalid("service %d: invalid figures", i)256 }257 if names[it.Manager+"\x00"+it.Name] {258 return invalid("service %d: duplicate", i)259 }260 names[it.Manager+"\x00"+it.Name] = true261 it.RawState, it.Version, it.Scope = clip(it.RawState, 64), clip(it.Version, 64), clip(it.Scope, 16)262 if len(it.Ports) > 16 {263 return invalid("service %d: too many ports", i)264 }265 for _, p := range it.Ports {266 if p <= 0 || p > 65535 {267 return invalid("service %d: invalid port", i)268 }269 }270 }271 }272 if len(rep.Volumes) > protocol.MaxVolumes {273 return invalid("too many volumes (%d > %d)", len(rep.Volumes), protocol.MaxVolumes)274 }275 mounts := map[string]bool{}276 for i := range rep.Volumes {277 v := &rep.Volumes[i]278 if v.Mount == "" || len(v.Mount) > 255 || !validText(v.Mount) || mounts[v.Mount] {279 return invalid("volume %d: invalid mount", i)280 }281 mounts[v.Mount] = true282 if v.Total < 0 || v.Used < 0 {283 return invalid("volume %d: invalid sizes", i)284 }285 if v.Kind != "internal" && v.Kind != "external" && v.Kind != "network" {286 return invalid("volume %d: unknown kind", i)287 }288 if v.SMART != "" && v.SMART != "verified" && v.SMART != "failing" {289 return invalid("volume %d: unknown smart status", i)290 }291 if strings.Contains(v.Remote, "@") {292 return invalid("volume %d: remote must not carry credentials", i)293 }294 v.Device, v.FS, v.Remote = clip(v.Device, 255), clip(v.FS, 32), clip(v.Remote, 255)295 }296 if inf := rep.Inference; inf != nil {297 if len(inf.Items) > maxInference {298 return invalid("too many inference servers")299 }300 ports := map[int]bool{}301 for i := range inf.Items {302 it := &inf.Items[i]303 if it.Port <= 0 || it.Port > 65535 || ports[it.Port] {304 return invalid("inference %d: invalid port", i)305 }306 ports[it.Port] = true307 if it.Health != "ok" && it.Health != "failing" && it.Health != "unknown" {308 return invalid("inference %d: unknown health", i)309 }310 if !finite(it.HealthMS) || it.RSS < 0 || it.UptimeS < 0 || it.DiskBytes < 0 {311 return invalid("inference %d: invalid figures", i)312 }313 it.Kind, it.Model = clip(it.Kind, 32), clip(it.Model, 200)314 if it.Kind == "" {315 return invalid("inference %d: kind required", i)316 }317 if len(it.ListedModels) > 64 {318 it.ListedModels = it.ListedModels[:64]319 }320 for j := range it.ListedModels {321 it.ListedModels[j] = clip(it.ListedModels[j], 200)322 }323 }324 }325 if len(rep.NAS) > maxNASProbes {326 return invalid("too many NAS probes")327 }328 for i := range rep.NAS {329 n := &rep.NAS[i]330 if _, err := netip.ParseAddr(n.Host); err != nil {331 return invalid("nas %d: host must be an IP address", i)332 }333 if !finite(n.LatencyMS) || n.LatencyMS < 0 || n.Total < 0 || n.Used < 0 {334 return invalid("nas %d: invalid figures", i)335 }336 }337 if len(rep.Capabilities) > 64 {338 return invalid("too many capabilities")339 }340 for i := range rep.Capabilities {341 if len(rep.Capabilities[i]) > 64 || !validText(rep.Capabilities[i]) {342 return invalid("capability %d invalid", i)343 }344 }345 if rep.Inventory != nil {346 if err := sanitizeInventory(rep.Inventory); err != nil {347 return invalid("inventory: %s", err)348 }349 }350 return nil351}352353func validText(s string) bool {354 for _, r := range s {355 if r < 0x20 || r == 0x7f || r == 0xfffd {356 return false357 }358 }359 return true360}361362// handleReport ingests one telemetry batch in a single transaction.363func (s *Server) handleReport(w http.ResponseWriter, r *http.Request) {364 a := agentFrom(r.Context())365 var rep protocol.Report366 if err := decodeBody(w, r, maxReportBody, &rep); err != nil {367 httpx.WriteError(w, r, err)368 return369 }370 now := time.Now().UTC()371 if verr := s.validateReport(&rep, now); verr != nil {372 s.log.Warn("ingest: report rejected", "agent_id", a.ID, "code", verr.Code, "reason", verr.Message)373 httpx.WriteError(w, r, verr)374 return375 }376 rep.TS = rep.TS.UTC().Truncate(time.Millisecond)377378 ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)379 defer cancel()380 ack, err := s.applyReport(ctx, a, &rep, now)381 if err != nil && missingPartition(err) {382 // The day of a buffered report has no partition yet: create and retry once.383 if perr := s.EnsurePartitions(ctx, rep.TS); perr == nil {384 ack, err = s.applyReport(ctx, a, &rep, now)385 }386 }387 if err != nil {388 s.writeErr(w, r, err)389 return390 }391 s.dbOK.Store(true)392 writeJSON(w, http.StatusOK, ack)393}394395func missingPartition(err error) bool {396 var pg *pgconn.PgError397 return errors.As(err, &pg) && pg.Code == "23514" && strings.Contains(pg.Message, "no partition")398}399400// applyReport is the report transaction. Idempotent on (agent, seq).401func (s *Server) applyReport(ctx context.Context, a *agentIdentity, rep *protocol.Report, now time.Time) (*protocol.ReportAck, error) {402 ack := &protocol.ReportAck{Seq: rep.Seq, ServerTime: now, IntervalS: s.intervalS}403 ack.RenewCert = a.NotAfter.Sub(now) < 15*24*time.Hour404 ack.NASTargets = s.nasHosts(ctx)405406 err := s.inTx(ctx, func(tx pgx.Tx) error {407 // Serialize the reports of one agent (a live report and a replayed one may overlap).408 var lastSeq int64409 var agentRevoked bool410 if err := tx.QueryRow(ctx, `select last_seq, revoked_at is not null from agents where id = $1 for update`, a.ID).Scan(&lastSeq, &agentRevoked); err != nil {411 return err412 }413 if agentRevoked {414 return errAgentRevoked415 }416 tag, err := tx.Exec(ctx, `insert into agent_report_seqs(agent_id, seq, ts) values ($1,$2,$3) on conflict do nothing`, a.ID, rep.Seq, rep.TS)417 if err != nil {418 return err419 }420 duplicate := tag.RowsAffected() == 0421422 if _, err := tx.Exec(ctx, `update agents set last_seen_at = now(), last_seq = greatest(last_seq, $2) where id = $1`, a.ID, rep.Seq); err != nil {423 return err424 }425 var invAt *time.Time426 if err := tx.QueryRow(ctx, `update nodes set last_seen_at = now() where id = $1 returning inventory_at`, a.NodeID).Scan(&invAt); err != nil {427 return err428 }429 ack.WantInventory = invAt == nil || now.Sub(*invAt) > 12*time.Hour430 if duplicate {431 return nil // already applied: acknowledge so the agent drops it from its buffer432 }433434 if err := insertSamples(ctx, tx, a.NodeID, rep); err != nil {435 return err436 }437 if err := markLate(ctx, tx, rep.TS); err != nil {438 return err439 }440441 // Live state only moves forward: a buffered or out-of-order report442 // contributes samples but never overwrites newer state.443 var stateTS *time.Time444 if err := tx.QueryRow(ctx, `select ts from node_state where node_id = $1 for update`, a.NodeID).Scan(&stateTS); err != nil && !errors.Is(err, pgx.ErrNoRows) {445 return err446 }447 live := !rep.Buffered && (stateTS == nil || rep.TS.After(*stateTS))448 if !live {449 return nil450 }451 values, err := upsertNodeState(ctx, tx, a.NodeID, rep, now)452 if err != nil {453 return err454 }455 if err := updateAgentMeta(ctx, tx, a.ID, rep, values); err != nil {456 return err457 }458 if rep.Services != nil {459 if err := applyServices(ctx, tx, a.NodeID, rep.Services); err != nil {460 return err461 }462 }463 if rep.Inference != nil {464 if err := applyInference(ctx, tx, a.NodeID, rep.Inference); err != nil {465 return err466 }467 }468 if len(rep.Volumes) > 0 {469 if err := applyVolumes(ctx, tx, a.NodeID, rep.Volumes); err != nil {470 return err471 }472 }473 if rep.Inventory != nil {474 var hw *string475 if err := tx.QueryRow(ctx, `select hardware_uuid from nodes where id = $1`, a.NodeID).Scan(&hw); err != nil {476 return err477 }478 if hw != nil && rep.Inventory.HardwareUUID != "" && *hw != rep.Inventory.HardwareUUID {479 // The identity moved to other hardware (disk clone, restored backup): do not rewrite the node.480 s.log.Warn("ingest: inventory hardware uuid differs from the enrolled node; inventory ignored", "agent_id", a.ID, "node_id", a.NodeID)481 } else {482 changed, err := applyInventory(ctx, tx, a.NodeID, rep.Inventory)483 if err != nil {484 return err485 }486 ack.WantInventory = false487 if changed {488 if err := events.Publish(ctx, tx, "node.updated", map[string]any{"node_id": a.NodeID, "change": "inventory", "reason": "inventory"}); err != nil {489 return err490 }491 }492 }493 }494 if len(rep.NAS) > 0 {495 if err := applyNAS(ctx, tx, a.NodeID, rep.NAS); err != nil {496 return err497 }498 }499 ev := nodeStateEvent(a.NodeID, rep, values)500 // Contract fields (NodeStateEvent): a report has just been stored, so the node is fresh by definition.501 var alias string502 var maintenance bool503 if err := tx.QueryRow(ctx, `select alias::text, maintenance from nodes where id=$1`, a.NodeID).Scan(&alias, &maintenance); err != nil {504 return err505 }506 ev["alias"], ev["maintenance"], ev["freshness"], ev["observed_at"] = alias, maintenance, "fresh", time.Now().UTC()507 return events.Publish(ctx, tx, "node.state", ev)508 })509 if err != nil {510 return nil, err511 }512 return ack, nil513}514515// insertSamples writes the batch with one statement (arrays unnested server side).516func insertSamples(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, rep *protocol.Report) error {517 if len(rep.Samples) == 0 {518 return nil519 }520 idsArr := make([]int16, len(rep.Samples))521 labels := make([]string, len(rep.Samples))522 vals := make([]float64, len(rep.Samples))523 for i, sm := range rep.Samples {524 m, _ := protocol.MetricByName(sm.M)525 idsArr[i], labels[i], vals[i] = m.ID, sm.L, sm.V526 }527 _, err := tx.Exec(ctx, `insert into samples_raw(node_id, metric_id, label, ts, value)528 select $1, m, l, $2, v from unnest($3::smallint[], $4::text[], $5::float8[]) as t(m, l, v)529 on conflict do nothing`, nodeID, rep.TS, idsArr, labels, vals)530 return err531}532533// markLate flags the rollup buckets of a report that arrived after the534// rollup watermark passed its minute, so they are recomputed.535func markLate(ctx context.Context, tx pgx.Tx, ts time.Time) error {536 _, err := tx.Exec(ctx, `insert into rollup_dirty(tier, bucket)537 select '1m', date_trunc('minute', $1::timestamptz)538 where exists (select 1 from rollup_state where tier = '1m' and watermark > date_trunc('minute', $1::timestamptz))539 on conflict do nothing`, ts)540 return err541}542543// reportValues folds samples into the node_state document:544// unlabeled metric -> number, labeled metric -> {label: number}.545func reportValues(rep *protocol.Report) map[string]any {546 out := map[string]any{}547 for _, sm := range rep.Samples {548 if sm.L == "" {549 out[sm.M] = sm.V550 continue551 }552 m, _ := out[sm.M].(map[string]float64)553 if m == nil {554 m = map[string]float64{}555 out[sm.M] = m556 }557 m[sm.L] = sm.V558 }559 return out560}561562// upsertNodeState merges this report into the live snapshot. Metrics have563// different intervals, so a report carries a subset: values present replace564// the stored ones, metrics declared unavailable are removed, others are kept.565func upsertNodeState(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, rep *protocol.Report, now time.Time) (map[string]any, error) {566 values := reportValues(rep)567 avail := rep.Availability568 if avail == nil {569 avail = map[string]string{}570 }571 gone := make([]string, 0, len(avail))572 for k := range avail {573 gone = append(gone, k)574 }575 var procs any576 var procsAt *time.Time577 if len(rep.Processes) > 0 {578 procs, procsAt = mustJSON(rep.Processes), &rep.TS579 }580 var merged map[string]any581 err := tx.QueryRow(ctx, `insert into node_state(node_id, ts, received_at, "values", availability, processes, processes_at, schema_version)582 values ($1,$2,$3,$4::jsonb,$5::jsonb,$6::jsonb,$7,1)583 on conflict (node_id) do update set584 ts = excluded.ts, received_at = excluded.received_at,585 "values" = (node_state."values" - $8::text[]) || excluded."values",586 availability = excluded.availability,587 processes = coalesce(excluded.processes, node_state.processes),588 processes_at = coalesce(excluded.processes_at, node_state.processes_at)589 returning "values"`,590 nodeID, rep.TS, now, mustJSON(values), mustJSON(avail), procs, procsAt, gone).Scan(&merged)591 return merged, err592}593594func updateAgentMeta(ctx context.Context, tx pgx.Tx, agentID uuid.UUID, rep *protocol.Report, values map[string]any) error {595 overhead := map[string]any{}596 if v, ok := values["agent.cpu"].(float64); ok {597 overhead["cpu_percent"] = v598 }599 if v, ok := values["agent.rss"].(float64); ok {600 overhead["rss_bytes"] = v601 }602 if len(overhead) > 0 {603 overhead["ts"] = rep.TS604 }605 // A live report is authoritative for capabilities. The field is omitted606 // from the JSON when empty (frozen agent, deny-all policy): absent = none.607 caps := rep.Capabilities608 if caps == nil {609 caps = []string{}610 }611 _, err := tx.Exec(ctx, `update agents set612 version = case when $2 <> '' then $2 else version end,613 capabilities = $3::text[],614 overhead = case when $4::jsonb = '{}'::jsonb then overhead else $4::jsonb end615 where id = $1`, agentID, rep.AgentVersion, caps, mustJSON(overhead))616 return err617}618619// headline metrics carried by the compact node.state event.620var headline = []string{621 "cpu.usage", "load.1", "mem.used", "mem.total", "mem.pressure.level", "mem.available.percent",622 "swap.used", "gpu.usage", "gpu.mem.used", "thermal.pressure", "battery.charge", "uptime", "procs.count",623 "disk.used", "disk.total", "net.rx", "net.tx", "net.latency", "net.loss",624}625626func nodeStateEvent(nodeID uuid.UUID, rep *protocol.Report, merged map[string]any) map[string]any {627 vals := map[string]any{}628 for _, k := range headline {629 if v, ok := merged[k]; ok {630 vals[k] = v631 }632 }633 av := rep.Availability634 if av == nil {635 av = map[string]string{}636 }637 return map[string]any{"node_id": nodeID, "ts": rep.TS, "values": vals, "availability": av}638}639640// ─────────────────────────── services ───────────────────────────641642type svcRow struct {643 id uuid.UUID644 state string645 gone bool646}647648// applyServices applies a complete observation by the managers listed in649// set.Managers: listed services are upserted, services of an observed manager650// that are no longer listed are marked gone, and services of a manager that651// was not observed are left untouched.652func applyServices(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, set *protocol.ServiceSet) error {653 if len(set.Managers) == 0 {654 return nil655 }656 rows, err := tx.Query(ctx, `select id, manager::text, name, state, gone_at is not null from services657 where node_id = $1 and manager::text = any($2) for update`, nodeID, set.Managers)658 if err != nil {659 return err660 }661 existing := map[string]svcRow{}662 for rows.Next() {663 var r svcRow664 var mgr, name string665 if err := rows.Scan(&r.id, &mgr, &name, &r.state, &r.gone); err != nil {666 rows.Close()667 return err668 }669 existing[mgr+"\x00"+name] = r670 }671 rows.Close()672 if err := rows.Err(); err != nil {673 return err674 }675676 type change struct {677 id uuid.UUID678 manager, name, state, pv string679 gone bool680 }681 var changes []change682 batch := &pgx.Batch{}683 listed := map[string]bool{}684 for _, it := range set.Items {685 k := it.Manager + "\x00" + it.Name686 listed[k] = true687 prev, known := existing[k]688 id := prev.id689 if !known {690 id = ids.New()691 }692 if !known || prev.gone || prev.state != it.State {693 pv := prev.state694 if !known || prev.gone {695 pv = ""696 }697 changes = append(changes, change{id: id, manager: it.Manager, name: it.Name, state: it.State, pv: pv})698 }699 ports := it.Ports700 if ports == nil {701 ports = []int{}702 }703 meta := map[string]any{"raw_state": it.RawState}704 if it.Scope != "" {705 meta["scope"] = it.Scope706 }707 batch.Queue(`insert into services(id, node_id, manager, name, state, pid, restarts, mem_bytes, cpu_pct, uptime_s, version, ports, meta, last_seen_at)708 values ($1,$2,$3::service_manager,$4,$5,nullif($6,0),$7,nullif($8::bigint,0),$9,nullif($10::bigint,0),nullif($11,''),$12,$13::jsonb, now())709 on conflict (node_id, manager, name) do update set710 state = excluded.state, pid = excluded.pid, restarts = excluded.restarts, mem_bytes = excluded.mem_bytes,711 cpu_pct = excluded.cpu_pct, uptime_s = excluded.uptime_s, version = excluded.version, ports = excluded.ports,712 meta = services.meta || excluded.meta, last_seen_at = now(), gone_at = null`,713 id, nodeID, it.Manager, it.Name, it.State, it.PID, it.Restarts, it.MemBytes, it.CPU, it.UptimeS, it.Version, ports, mustJSON(meta))714 }715 var goneIDs []uuid.UUID716 for k, prev := range existing {717 if listed[k] || prev.gone {718 continue719 }720 goneIDs = append(goneIDs, prev.id)721 parts := strings.SplitN(k, "\x00", 2)722 changes = append(changes, change{id: prev.id, manager: parts[0], name: parts[1], state: "gone", pv: prev.state, gone: true})723 }724 if len(goneIDs) > 0 {725 batch.Queue(`update services set gone_at = now() where id = any($1) and gone_at is null`, goneIDs)726 }727 if batch.Len() > 0 {728 if err := tx.SendBatch(ctx, batch).Close(); err != nil {729 return err730 }731 }732 for _, c := range changes {733 if err := events.Publish(ctx, tx, "service.changed", map[string]any{734 "node_id": nodeID, "service_id": c.id, "manager": c.manager, "name": c.name,735 "state": c.state, "previous_state": c.pv, "gone": c.gone,736 }); err != nil {737 return err738 }739 }740 return nil741}742743func applyInference(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, set *protocol.InferenceSet) error {744 ports := make([]int, 0, len(set.Items))745 for _, it := range set.Items {746 ports = append(ports, it.Port)747 var listed any748 if it.ListedModels != nil {749 listed = mustJSON(it.ListedModels)750 }751 if _, err := tx.Exec(ctx, `insert into inference_servers(id, node_id, kind, port, model, pid, rss_bytes, uptime_s, disk_bytes, health, health_at, health_latency_ms, listed_models, last_seen_at)752 values ($1,$2,$3,$4,nullif($5,''),nullif($6,0),nullif($7::bigint,0),$8,nullif($9::bigint,0),$10, now(), nullif($11::real,0),$12::jsonb, now())753 on conflict (node_id, port) do update set754 kind = excluded.kind, model = excluded.model, pid = excluded.pid, rss_bytes = excluded.rss_bytes,755 uptime_s = excluded.uptime_s, disk_bytes = coalesce(excluded.disk_bytes, inference_servers.disk_bytes),756 health = excluded.health,757 health_at = case when excluded.health = 'unknown' then inference_servers.health_at else excluded.health_at end,758 health_latency_ms = excluded.health_latency_ms,759 listed_models = coalesce(excluded.listed_models, inference_servers.listed_models),760 last_seen_at = now(), gone_at = null`,761 ids.New(), nodeID, it.Kind, it.Port, it.Model, it.PID, it.RSS, it.UptimeS, it.DiskBytes, it.Health, it.HealthMS, listed); err != nil {762 return err763 }764 }765 _, err := tx.Exec(ctx, `update inference_servers set gone_at = now() where node_id = $1 and gone_at is null and not (port = any($2::int[]))`, nodeID, ports)766 return err767}768769func applyVolumes(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, vols []protocol.Volume) error {770 mounts := make([]string, 0, len(vols))771 batch := &pgx.Batch{}772 for _, v := range vols {773 mounts = append(mounts, v.Mount)774 batch.Queue(`insert into volumes(id, node_id, mount, device, fs, kind, total_bytes, used_bytes, smart_status, remote, last_seen_at)775 values ($1,$2,$3,nullif($4,''),nullif($5,''),$6,$7,$8,nullif($9,''),nullif($10,''), now())776 on conflict (node_id, mount) do update set777 device = excluded.device, fs = excluded.fs, kind = excluded.kind, total_bytes = excluded.total_bytes,778 used_bytes = excluded.used_bytes, smart_status = excluded.smart_status, remote = excluded.remote,779 last_seen_at = now(), gone_at = null`,780 ids.New(), nodeID, v.Mount, v.Device, v.FS, v.Kind, v.Total, v.Used, v.SMART, v.Remote)781 }782 batch.Queue(`update volumes set gone_at = now() where node_id = $1 and gone_at is null and not (mount = any($2::text[]))`, nodeID, mounts)783 return tx.SendBatch(ctx, batch).Close()784}785786// ─────────────────────────── NAS ───────────────────────────787788// nasHosts returns the registered NAS addresses agents should probe (cached 30 s).789func (s *Server) nasHosts(ctx context.Context) []string {790 s.nasMu.Lock()791 defer s.nasMu.Unlock()792 if time.Since(s.nasAt) < 30*time.Second {793 return s.nasTargets794 }795 rows, err := s.pool.Query(ctx, `select host(host) from nas_targets order by name limit $1`, maxNASProbes)796 if err != nil {797 return s.nasTargets798 }799 hosts, err := pgx.CollectRows(rows, pgx.RowTo[string])800 if err != nil {801 return s.nasTargets802 }803 s.nasTargets, s.nasAt = hosts, time.Now()804 return hosts805}806807// applyNAS records this node's view of each registered NAS and recomputes808// the aggregate state from every recent observation: online if any node809// reaches it, offline if none does.810func applyNAS(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, probes []protocol.NASProbe) error {811 for _, p := range probes {812 var nasID uuid.UUID813 err := tx.QueryRow(ctx, `select id from nas_targets where host = $1::inet for update`, p.Host).Scan(&nasID)814 if errors.Is(err, pgx.ErrNoRows) {815 continue // not a registered target: ignored816 }817 if err != nil {818 return err819 }820 if _, err := tx.Exec(ctx, `insert into nas_observations(nas_id, node_id, reachable, mounted, latency_ms, total_bytes, used_bytes, observed_at)821 values ($1,$2,$3,$4,case when $3 then $5::real end,nullif($6::bigint,0),nullif($7::bigint,0), now())822 on conflict (nas_id, node_id) do update set reachable = excluded.reachable, mounted = excluded.mounted,823 latency_ms = excluded.latency_ms, total_bytes = excluded.total_bytes, used_bytes = excluded.used_bytes, observed_at = excluded.observed_at`,824 nasID, nodeID, p.Reachable, p.Mounted, p.LatencyMS, p.Total, p.Used); err != nil {825 return err826 }827 if _, err := tx.Exec(ctx, `with o as (828 select * from nas_observations where nas_id = $1 and observed_at > now() - interval '5 minutes'829 ), agg as (830 select bool_or(reachable) as any_up, count(*) filter (where mounted) as mounted,831 min(latency_ms) filter (where reachable) as latency,832 max(total_bytes) as total, max(used_bytes) filter (where total_bytes is not null) as used,833 (select node_id from o where reachable order by observed_at desc limit 1) as up_by,834 (select node_id from o order by observed_at desc limit 1) as last_by835 from o836 )837 update nas_targets t set838 state = case when agg.any_up then 'online' else 'offline' end, state_at = now(),839 observed_by = coalesce(agg.up_by, agg.last_by), latency_ms = agg.latency, mounted_on = agg.mounted,840 total_bytes = coalesce(agg.total, t.total_bytes), used_bytes = coalesce(agg.used, t.used_bytes)841 from agg where t.id = $1`, nasID); err != nil {842 return err843 }844 }845 return nil846}847