SPB Git

Repository Public

report.go 30.1 KB 847 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/ingest/report.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package ingest4344import (45	"compress/gzip"46	"context"47	"encoding/json"48	"errors"49	"fmt"50	"io"51	"math"52	"net/http"53	"net/netip"54	"strings"55	"time"5657	"github.com/google/uuid"58	"github.com/jackc/pgx/v5"59	"github.com/jackc/pgx/v5/pgconn"6061	"clustr.spboucher.ai/mcc/internal/events"62	"clustr.spboucher.ai/mcc/internal/httpx"63	"clustr.spboucher.ai/mcc/internal/ids"64	"clustr.spboucher.ai/mcc/internal/protocol"65)6667const (68	maxReportBody  = 1 << 20 // bytes on the wire69	maxReportPlain = 4 << 20 // bytes after gzip70	maxFutureSkew  = 5 * time.Minute71	maxInference   = 1672	maxNASProbes   = 873)7475var errTooLarge = httpx.Err(http.StatusRequestEntityTooLarge, "payload_too_large", "The request body is too large.")7677// decodeBody reads a bounded JSON body, transparently gunzipping it. Unknown78// fields are ignored: within protocol v1, additions are backward compatible.79func decodeBody(w http.ResponseWriter, r *http.Request, max int64, dst any) error {80	if r.ContentLength > max {81		return errTooLarge82	}83	var src io.Reader = http.MaxBytesReader(w, r.Body, max)84	plainMax := max85	if strings.EqualFold(r.Header.Get("Content-Encoding"), "gzip") {86		zr, err := gzip.NewReader(src)87		if err != nil {88			return bodyErr(err)89		}90		defer zr.Close()91		plainMax = max * 492		src = zr93	} else if ce := r.Header.Get("Content-Encoding"); ce != "" && !strings.EqualFold(ce, "identity") {94		return httpx.Err(http.StatusUnsupportedMediaType, "unsupported_encoding", "Content-Encoding must be gzip or identity.")95	}96	body, err := io.ReadAll(io.LimitReader(src, plainMax+1))97	if err != nil {98		return bodyErr(err)99	}100	if int64(len(body)) > plainMax {101		return errTooLarge102	}103	if err := json.Unmarshal(body, dst); err != nil {104		return httpx.Validation("Malformed JSON body.")105	}106	return nil107}108109func bodyErr(err error) error {110	var mbe *http.MaxBytesError111	if errors.As(err, &mbe) {112		return errTooLarge113	}114	return httpx.Validation("Unreadable request body.")115}116117func invalid(format string, a ...any) *httpx.Error {118	return httpx.Err(http.StatusBadRequest, "invalid_report", fmt.Sprintf(format, a...))119}120121var availStates = map[string]bool{122	protocol.AvailUnsupported: true, protocol.AvailPermissionDenied: true, protocol.AvailUnavailable: true,123}124var serviceManagers = map[string]bool{"pm2": true, "launchd": true, "systemd": true, "docker": true}125var serviceStates = map[string]bool{"online": true, "stopped": true, "errored": true, "unknown": true}126127func finite(v float64) bool { return !math.IsNaN(v) && !math.IsInf(v, 0) }128129func validLabel(l string) bool {130	if len(l) > protocol.MaxLabelLen {131		return false132	}133	for _, r := range l {134		if r < 0x20 || r == 0x7f || r == 0xfffd {135			return false136		}137	}138	return true139}140141// validateReport enforces the protocol strictly. A report that fails here is142// rejected as a whole with 400; the agent drops it (it would never succeed).143func (s *Server) validateReport(rep *protocol.Report, now time.Time) *httpx.Error {144	if rep.V != protocol.Version {145		return httpx.Err(http.StatusBadRequest, "unsupported_version", fmt.Sprintf("Protocol version %d is not supported (expected %d).", rep.V, protocol.Version))146	}147	if rep.Seq <= 0 {148		return invalid("seq must be positive")149	}150	if rep.TS.IsZero() {151		return invalid("ts is required")152	}153	if rep.TS.After(now.Add(maxFutureSkew)) {154		return httpx.Err(http.StatusBadRequest, "clock_skew", "Report timestamp is more than 5 minutes in the future; check the node clock.")155	}156	if rep.TS.Before(now.Add(-s.cfg.RetainRaw)) {157		return httpx.Err(http.StatusBadRequest, "too_old", "Report is older than the raw retention.")158	}159	if len(rep.AgentVersion) > 64 {160		return invalid("agent_version too long")161	}162	if len(rep.Samples) > protocol.MaxSamples {163		return invalid("too many samples (%d > %d)", len(rep.Samples), protocol.MaxSamples)164	}165	type key struct {166		id int16167		l  string168	}169	seen := make(map[key]struct{}, len(rep.Samples))170	perMetric := map[string]int{}171	for _, sm := range rep.Samples {172		m, ok := protocol.MetricByName(sm.M)173		if !ok {174			return httpx.Err(http.StatusBadRequest, "unknown_metric", fmt.Sprintf("Unknown metric %q.", clip(sm.M, 64)))175		}176		if !finite(sm.V) {177			return invalid("metric %s: value is not finite", m.Name)178		}179		if !m.Labeled && sm.L != "" {180			return invalid("metric %s takes no label", m.Name)181		}182		if m.Labeled && sm.L == "" {183			return invalid("metric %s requires a label", m.Name)184		}185		if !validLabel(sm.L) {186			return invalid("metric %s: invalid label", m.Name)187		}188		k := key{m.ID, sm.L}189		if _, dup := seen[k]; dup {190			return invalid("metric %s: duplicate sample", m.Name)191		}192		seen[k] = struct{}{}193		perMetric[m.Name]++194	}195	for name, n := range perMetric {196		limit := 1197		switch {198		case strings.HasPrefix(name, "disk."):199			limit = protocol.MaxVolumes200		case strings.HasPrefix(name, "net."):201			limit = protocol.MaxInterfaces202		}203		if n > limit {204			return invalid("metric %s: too many labels (%d > %d)", name, n, limit)205		}206	}207	if len(rep.Availability) > len(protocol.Catalog) {208		return invalid("availability has too many entries")209	}210	for name, st := range rep.Availability {211		if _, ok := protocol.MetricByName(name); !ok {212			return httpx.Err(http.StatusBadRequest, "unknown_metric", fmt.Sprintf("Unknown metric %q in availability.", clip(name, 64)))213		}214		if !availStates[st] {215			return invalid("availability of %s: unknown state", name)216		}217	}218	if len(rep.Processes) > protocol.MaxProcesses {219		return invalid("too many processes (%d > %d)", len(rep.Processes), protocol.MaxProcesses)220	}221	for i := range rep.Processes {222		p := &rep.Processes[i]223		if !finite(p.CPU) || p.CPU < 0 || p.RSS < 0 || p.PID < 0 {224			return invalid("process %d: invalid figures", i)225		}226		p.Name, p.User = clip(p.Name, 64), clip(p.User, 64)227	}228	if sv := rep.Services; sv != nil {229		if len(sv.Items) > protocol.MaxServices {230			return invalid("too many services (%d > %d)", len(sv.Items), protocol.MaxServices)231		}232		if len(sv.Managers) > len(serviceManagers) {233			return invalid("too many service managers")234		}235		observed := map[string]bool{}236		for _, m := range sv.Managers {237			if !serviceManagers[m] {238				return invalid("unknown service manager")239			}240			observed[m] = true241		}242		names := map[string]bool{}243		for i := range sv.Items {244			it := &sv.Items[i]245			if !observed[it.Manager] {246				return invalid("service %d: manager not listed in managers", i)247			}248			if it.Name == "" || len(it.Name) > 200 || !validText(it.Name) {249				return invalid("service %d: invalid name", i)250			}251			if !serviceStates[it.State] {252				return invalid("service %d: unknown state", i)253			}254			if !finite(it.CPU) || it.MemBytes < 0 || it.UptimeS < 0 || it.Restarts < 0 || it.PID < 0 {255				return invalid("service %d: invalid figures", i)256			}257			if names[it.Manager+"\x00"+it.Name] {258				return invalid("service %d: duplicate", i)259			}260			names[it.Manager+"\x00"+it.Name] = true261			it.RawState, it.Version, it.Scope = clip(it.RawState, 64), clip(it.Version, 64), clip(it.Scope, 16)262			if len(it.Ports) > 16 {263				return invalid("service %d: too many ports", i)264			}265			for _, p := range it.Ports {266				if p <= 0 || p > 65535 {267					return invalid("service %d: invalid port", i)268				}269			}270		}271	}272	if len(rep.Volumes) > protocol.MaxVolumes {273		return invalid("too many volumes (%d > %d)", len(rep.Volumes), protocol.MaxVolumes)274	}275	mounts := map[string]bool{}276	for i := range rep.Volumes {277		v := &rep.Volumes[i]278		if v.Mount == "" || len(v.Mount) > 255 || !validText(v.Mount) || mounts[v.Mount] {279			return invalid("volume %d: invalid mount", i)280		}281		mounts[v.Mount] = true282		if v.Total < 0 || v.Used < 0 {283			return invalid("volume %d: invalid sizes", i)284		}285		if v.Kind != "internal" && v.Kind != "external" && v.Kind != "network" {286			return invalid("volume %d: unknown kind", i)287		}288		if v.SMART != "" && v.SMART != "verified" && v.SMART != "failing" {289			return invalid("volume %d: unknown smart status", i)290		}291		if strings.Contains(v.Remote, "@") {292			return invalid("volume %d: remote must not carry credentials", i)293		}294		v.Device, v.FS, v.Remote = clip(v.Device, 255), clip(v.FS, 32), clip(v.Remote, 255)295	}296	if inf := rep.Inference; inf != nil {297		if len(inf.Items) > maxInference {298			return invalid("too many inference servers")299		}300		ports := map[int]bool{}301		for i := range inf.Items {302			it := &inf.Items[i]303			if it.Port <= 0 || it.Port > 65535 || ports[it.Port] {304				return invalid("inference %d: invalid port", i)305			}306			ports[it.Port] = true307			if it.Health != "ok" && it.Health != "failing" && it.Health != "unknown" {308				return invalid("inference %d: unknown health", i)309			}310			if !finite(it.HealthMS) || it.RSS < 0 || it.UptimeS < 0 || it.DiskBytes < 0 {311				return invalid("inference %d: invalid figures", i)312			}313			it.Kind, it.Model = clip(it.Kind, 32), clip(it.Model, 200)314			if it.Kind == "" {315				return invalid("inference %d: kind required", i)316			}317			if len(it.ListedModels) > 64 {318				it.ListedModels = it.ListedModels[:64]319			}320			for j := range it.ListedModels {321				it.ListedModels[j] = clip(it.ListedModels[j], 200)322			}323		}324	}325	if len(rep.NAS) > maxNASProbes {326		return invalid("too many NAS probes")327	}328	for i := range rep.NAS {329		n := &rep.NAS[i]330		if _, err := netip.ParseAddr(n.Host); err != nil {331			return invalid("nas %d: host must be an IP address", i)332		}333		if !finite(n.LatencyMS) || n.LatencyMS < 0 || n.Total < 0 || n.Used < 0 {334			return invalid("nas %d: invalid figures", i)335		}336	}337	if len(rep.Capabilities) > 64 {338		return invalid("too many capabilities")339	}340	for i := range rep.Capabilities {341		if len(rep.Capabilities[i]) > 64 || !validText(rep.Capabilities[i]) {342			return invalid("capability %d invalid", i)343		}344	}345	if rep.Inventory != nil {346		if err := sanitizeInventory(rep.Inventory); err != nil {347			return invalid("inventory: %s", err)348		}349	}350	return nil351}352353func validText(s string) bool {354	for _, r := range s {355		if r < 0x20 || r == 0x7f || r == 0xfffd {356			return false357		}358	}359	return true360}361362// handleReport ingests one telemetry batch in a single transaction.363func (s *Server) handleReport(w http.ResponseWriter, r *http.Request) {364	a := agentFrom(r.Context())365	var rep protocol.Report366	if err := decodeBody(w, r, maxReportBody, &rep); err != nil {367		httpx.WriteError(w, r, err)368		return369	}370	now := time.Now().UTC()371	if verr := s.validateReport(&rep, now); verr != nil {372		s.log.Warn("ingest: report rejected", "agent_id", a.ID, "code", verr.Code, "reason", verr.Message)373		httpx.WriteError(w, r, verr)374		return375	}376	rep.TS = rep.TS.UTC().Truncate(time.Millisecond)377378	ctx, cancel := context.WithTimeout(r.Context(), s.reqTimeout)379	defer cancel()380	ack, err := s.applyReport(ctx, a, &rep, now)381	if err != nil && missingPartition(err) {382		// The day of a buffered report has no partition yet: create and retry once.383		if perr := s.EnsurePartitions(ctx, rep.TS); perr == nil {384			ack, err = s.applyReport(ctx, a, &rep, now)385		}386	}387	if err != nil {388		s.writeErr(w, r, err)389		return390	}391	s.dbOK.Store(true)392	writeJSON(w, http.StatusOK, ack)393}394395func missingPartition(err error) bool {396	var pg *pgconn.PgError397	return errors.As(err, &pg) && pg.Code == "23514" && strings.Contains(pg.Message, "no partition")398}399400// applyReport is the report transaction. Idempotent on (agent, seq).401func (s *Server) applyReport(ctx context.Context, a *agentIdentity, rep *protocol.Report, now time.Time) (*protocol.ReportAck, error) {402	ack := &protocol.ReportAck{Seq: rep.Seq, ServerTime: now, IntervalS: s.intervalS}403	ack.RenewCert = a.NotAfter.Sub(now) < 15*24*time.Hour404	ack.NASTargets = s.nasHosts(ctx)405406	err := s.inTx(ctx, func(tx pgx.Tx) error {407		// Serialize the reports of one agent (a live report and a replayed one may overlap).408		var lastSeq int64409		var agentRevoked bool410		if err := tx.QueryRow(ctx, `select last_seq, revoked_at is not null from agents where id = $1 for update`, a.ID).Scan(&lastSeq, &agentRevoked); err != nil {411			return err412		}413		if agentRevoked {414			return errAgentRevoked415		}416		tag, err := tx.Exec(ctx, `insert into agent_report_seqs(agent_id, seq, ts) values ($1,$2,$3) on conflict do nothing`, a.ID, rep.Seq, rep.TS)417		if err != nil {418			return err419		}420		duplicate := tag.RowsAffected() == 0421422		if _, err := tx.Exec(ctx, `update agents set last_seen_at = now(), last_seq = greatest(last_seq, $2) where id = $1`, a.ID, rep.Seq); err != nil {423			return err424		}425		var invAt *time.Time426		if err := tx.QueryRow(ctx, `update nodes set last_seen_at = now() where id = $1 returning inventory_at`, a.NodeID).Scan(&invAt); err != nil {427			return err428		}429		ack.WantInventory = invAt == nil || now.Sub(*invAt) > 12*time.Hour430		if duplicate {431			return nil // already applied: acknowledge so the agent drops it from its buffer432		}433434		if err := insertSamples(ctx, tx, a.NodeID, rep); err != nil {435			return err436		}437		if err := markLate(ctx, tx, rep.TS); err != nil {438			return err439		}440441		// Live state only moves forward: a buffered or out-of-order report442		// contributes samples but never overwrites newer state.443		var stateTS *time.Time444		if err := tx.QueryRow(ctx, `select ts from node_state where node_id = $1 for update`, a.NodeID).Scan(&stateTS); err != nil && !errors.Is(err, pgx.ErrNoRows) {445			return err446		}447		live := !rep.Buffered && (stateTS == nil || rep.TS.After(*stateTS))448		if !live {449			return nil450		}451		values, err := upsertNodeState(ctx, tx, a.NodeID, rep, now)452		if err != nil {453			return err454		}455		if err := updateAgentMeta(ctx, tx, a.ID, rep, values); err != nil {456			return err457		}458		if rep.Services != nil {459			if err := applyServices(ctx, tx, a.NodeID, rep.Services); err != nil {460				return err461			}462		}463		if rep.Inference != nil {464			if err := applyInference(ctx, tx, a.NodeID, rep.Inference); err != nil {465				return err466			}467		}468		if len(rep.Volumes) > 0 {469			if err := applyVolumes(ctx, tx, a.NodeID, rep.Volumes); err != nil {470				return err471			}472		}473		if rep.Inventory != nil {474			var hw *string475			if err := tx.QueryRow(ctx, `select hardware_uuid from nodes where id = $1`, a.NodeID).Scan(&hw); err != nil {476				return err477			}478			if hw != nil && rep.Inventory.HardwareUUID != "" && *hw != rep.Inventory.HardwareUUID {479				// The identity moved to other hardware (disk clone, restored backup): do not rewrite the node.480				s.log.Warn("ingest: inventory hardware uuid differs from the enrolled node; inventory ignored", "agent_id", a.ID, "node_id", a.NodeID)481			} else {482				changed, err := applyInventory(ctx, tx, a.NodeID, rep.Inventory)483				if err != nil {484					return err485				}486				ack.WantInventory = false487				if changed {488					if err := events.Publish(ctx, tx, "node.updated", map[string]any{"node_id": a.NodeID, "change": "inventory", "reason": "inventory"}); err != nil {489						return err490					}491				}492			}493		}494		if len(rep.NAS) > 0 {495			if err := applyNAS(ctx, tx, a.NodeID, rep.NAS); err != nil {496				return err497			}498		}499		ev := nodeStateEvent(a.NodeID, rep, values)500		// Contract fields (NodeStateEvent): a report has just been stored, so the node is fresh by definition.501		var alias string502		var maintenance bool503		if err := tx.QueryRow(ctx, `select alias::text, maintenance from nodes where id=$1`, a.NodeID).Scan(&alias, &maintenance); err != nil {504			return err505		}506		ev["alias"], ev["maintenance"], ev["freshness"], ev["observed_at"] = alias, maintenance, "fresh", time.Now().UTC()507		return events.Publish(ctx, tx, "node.state", ev)508	})509	if err != nil {510		return nil, err511	}512	return ack, nil513}514515// insertSamples writes the batch with one statement (arrays unnested server side).516func insertSamples(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, rep *protocol.Report) error {517	if len(rep.Samples) == 0 {518		return nil519	}520	idsArr := make([]int16, len(rep.Samples))521	labels := make([]string, len(rep.Samples))522	vals := make([]float64, len(rep.Samples))523	for i, sm := range rep.Samples {524		m, _ := protocol.MetricByName(sm.M)525		idsArr[i], labels[i], vals[i] = m.ID, sm.L, sm.V526	}527	_, err := tx.Exec(ctx, `insert into samples_raw(node_id, metric_id, label, ts, value)528		select $1, m, l, $2, v from unnest($3::smallint[], $4::text[], $5::float8[]) as t(m, l, v)529		on conflict do nothing`, nodeID, rep.TS, idsArr, labels, vals)530	return err531}532533// markLate flags the rollup buckets of a report that arrived after the534// rollup watermark passed its minute, so they are recomputed.535func markLate(ctx context.Context, tx pgx.Tx, ts time.Time) error {536	_, err := tx.Exec(ctx, `insert into rollup_dirty(tier, bucket)537		select '1m', date_trunc('minute', $1::timestamptz)538		where exists (select 1 from rollup_state where tier = '1m' and watermark > date_trunc('minute', $1::timestamptz))539		on conflict do nothing`, ts)540	return err541}542543// reportValues folds samples into the node_state document:544// unlabeled metric -> number, labeled metric -> {label: number}.545func reportValues(rep *protocol.Report) map[string]any {546	out := map[string]any{}547	for _, sm := range rep.Samples {548		if sm.L == "" {549			out[sm.M] = sm.V550			continue551		}552		m, _ := out[sm.M].(map[string]float64)553		if m == nil {554			m = map[string]float64{}555			out[sm.M] = m556		}557		m[sm.L] = sm.V558	}559	return out560}561562// upsertNodeState merges this report into the live snapshot. Metrics have563// different intervals, so a report carries a subset: values present replace564// the stored ones, metrics declared unavailable are removed, others are kept.565func upsertNodeState(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, rep *protocol.Report, now time.Time) (map[string]any, error) {566	values := reportValues(rep)567	avail := rep.Availability568	if avail == nil {569		avail = map[string]string{}570	}571	gone := make([]string, 0, len(avail))572	for k := range avail {573		gone = append(gone, k)574	}575	var procs any576	var procsAt *time.Time577	if len(rep.Processes) > 0 {578		procs, procsAt = mustJSON(rep.Processes), &rep.TS579	}580	var merged map[string]any581	err := tx.QueryRow(ctx, `insert into node_state(node_id, ts, received_at, "values", availability, processes, processes_at, schema_version)582		values ($1,$2,$3,$4::jsonb,$5::jsonb,$6::jsonb,$7,1)583		on conflict (node_id) do update set584			ts = excluded.ts, received_at = excluded.received_at,585			"values" = (node_state."values" - $8::text[]) || excluded."values",586			availability = excluded.availability,587			processes = coalesce(excluded.processes, node_state.processes),588			processes_at = coalesce(excluded.processes_at, node_state.processes_at)589		returning "values"`,590		nodeID, rep.TS, now, mustJSON(values), mustJSON(avail), procs, procsAt, gone).Scan(&merged)591	return merged, err592}593594func updateAgentMeta(ctx context.Context, tx pgx.Tx, agentID uuid.UUID, rep *protocol.Report, values map[string]any) error {595	overhead := map[string]any{}596	if v, ok := values["agent.cpu"].(float64); ok {597		overhead["cpu_percent"] = v598	}599	if v, ok := values["agent.rss"].(float64); ok {600		overhead["rss_bytes"] = v601	}602	if len(overhead) > 0 {603		overhead["ts"] = rep.TS604	}605	// A live report is authoritative for capabilities. The field is omitted606	// from the JSON when empty (frozen agent, deny-all policy): absent = none.607	caps := rep.Capabilities608	if caps == nil {609		caps = []string{}610	}611	_, err := tx.Exec(ctx, `update agents set612			version = case when $2 <> '' then $2 else version end,613			capabilities = $3::text[],614			overhead = case when $4::jsonb = '{}'::jsonb then overhead else $4::jsonb end615		where id = $1`, agentID, rep.AgentVersion, caps, mustJSON(overhead))616	return err617}618619// headline metrics carried by the compact node.state event.620var headline = []string{621	"cpu.usage", "load.1", "mem.used", "mem.total", "mem.pressure.level", "mem.available.percent",622	"swap.used", "gpu.usage", "gpu.mem.used", "thermal.pressure", "battery.charge", "uptime", "procs.count",623	"disk.used", "disk.total", "net.rx", "net.tx", "net.latency", "net.loss",624}625626func nodeStateEvent(nodeID uuid.UUID, rep *protocol.Report, merged map[string]any) map[string]any {627	vals := map[string]any{}628	for _, k := range headline {629		if v, ok := merged[k]; ok {630			vals[k] = v631		}632	}633	av := rep.Availability634	if av == nil {635		av = map[string]string{}636	}637	return map[string]any{"node_id": nodeID, "ts": rep.TS, "values": vals, "availability": av}638}639640// ─────────────────────────── services ───────────────────────────641642type svcRow struct {643	id    uuid.UUID644	state string645	gone  bool646}647648// applyServices applies a complete observation by the managers listed in649// set.Managers: listed services are upserted, services of an observed manager650// that are no longer listed are marked gone, and services of a manager that651// was not observed are left untouched.652func applyServices(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, set *protocol.ServiceSet) error {653	if len(set.Managers) == 0 {654		return nil655	}656	rows, err := tx.Query(ctx, `select id, manager::text, name, state, gone_at is not null from services657		where node_id = $1 and manager::text = any($2) for update`, nodeID, set.Managers)658	if err != nil {659		return err660	}661	existing := map[string]svcRow{}662	for rows.Next() {663		var r svcRow664		var mgr, name string665		if err := rows.Scan(&r.id, &mgr, &name, &r.state, &r.gone); err != nil {666			rows.Close()667			return err668		}669		existing[mgr+"\x00"+name] = r670	}671	rows.Close()672	if err := rows.Err(); err != nil {673		return err674	}675676	type change struct {677		id                       uuid.UUID678		manager, name, state, pv string679		gone                     bool680	}681	var changes []change682	batch := &pgx.Batch{}683	listed := map[string]bool{}684	for _, it := range set.Items {685		k := it.Manager + "\x00" + it.Name686		listed[k] = true687		prev, known := existing[k]688		id := prev.id689		if !known {690			id = ids.New()691		}692		if !known || prev.gone || prev.state != it.State {693			pv := prev.state694			if !known || prev.gone {695				pv = ""696			}697			changes = append(changes, change{id: id, manager: it.Manager, name: it.Name, state: it.State, pv: pv})698		}699		ports := it.Ports700		if ports == nil {701			ports = []int{}702		}703		meta := map[string]any{"raw_state": it.RawState}704		if it.Scope != "" {705			meta["scope"] = it.Scope706		}707		batch.Queue(`insert into services(id, node_id, manager, name, state, pid, restarts, mem_bytes, cpu_pct, uptime_s, version, ports, meta, last_seen_at)708			values ($1,$2,$3::service_manager,$4,$5,nullif($6,0),$7,nullif($8::bigint,0),$9,nullif($10::bigint,0),nullif($11,''),$12,$13::jsonb, now())709			on conflict (node_id, manager, name) do update set710				state = excluded.state, pid = excluded.pid, restarts = excluded.restarts, mem_bytes = excluded.mem_bytes,711				cpu_pct = excluded.cpu_pct, uptime_s = excluded.uptime_s, version = excluded.version, ports = excluded.ports,712				meta = services.meta || excluded.meta, last_seen_at = now(), gone_at = null`,713			id, nodeID, it.Manager, it.Name, it.State, it.PID, it.Restarts, it.MemBytes, it.CPU, it.UptimeS, it.Version, ports, mustJSON(meta))714	}715	var goneIDs []uuid.UUID716	for k, prev := range existing {717		if listed[k] || prev.gone {718			continue719		}720		goneIDs = append(goneIDs, prev.id)721		parts := strings.SplitN(k, "\x00", 2)722		changes = append(changes, change{id: prev.id, manager: parts[0], name: parts[1], state: "gone", pv: prev.state, gone: true})723	}724	if len(goneIDs) > 0 {725		batch.Queue(`update services set gone_at = now() where id = any($1) and gone_at is null`, goneIDs)726	}727	if batch.Len() > 0 {728		if err := tx.SendBatch(ctx, batch).Close(); err != nil {729			return err730		}731	}732	for _, c := range changes {733		if err := events.Publish(ctx, tx, "service.changed", map[string]any{734			"node_id": nodeID, "service_id": c.id, "manager": c.manager, "name": c.name,735			"state": c.state, "previous_state": c.pv, "gone": c.gone,736		}); err != nil {737			return err738		}739	}740	return nil741}742743func applyInference(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, set *protocol.InferenceSet) error {744	ports := make([]int, 0, len(set.Items))745	for _, it := range set.Items {746		ports = append(ports, it.Port)747		var listed any748		if it.ListedModels != nil {749			listed = mustJSON(it.ListedModels)750		}751		if _, err := tx.Exec(ctx, `insert into inference_servers(id, node_id, kind, port, model, pid, rss_bytes, uptime_s, disk_bytes, health, health_at, health_latency_ms, listed_models, last_seen_at)752			values ($1,$2,$3,$4,nullif($5,''),nullif($6,0),nullif($7::bigint,0),$8,nullif($9::bigint,0),$10, now(), nullif($11::real,0),$12::jsonb, now())753			on conflict (node_id, port) do update set754				kind = excluded.kind, model = excluded.model, pid = excluded.pid, rss_bytes = excluded.rss_bytes,755				uptime_s = excluded.uptime_s, disk_bytes = coalesce(excluded.disk_bytes, inference_servers.disk_bytes),756				health = excluded.health,757				health_at = case when excluded.health = 'unknown' then inference_servers.health_at else excluded.health_at end,758				health_latency_ms = excluded.health_latency_ms,759				listed_models = coalesce(excluded.listed_models, inference_servers.listed_models),760				last_seen_at = now(), gone_at = null`,761			ids.New(), nodeID, it.Kind, it.Port, it.Model, it.PID, it.RSS, it.UptimeS, it.DiskBytes, it.Health, it.HealthMS, listed); err != nil {762			return err763		}764	}765	_, err := tx.Exec(ctx, `update inference_servers set gone_at = now() where node_id = $1 and gone_at is null and not (port = any($2::int[]))`, nodeID, ports)766	return err767}768769func applyVolumes(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, vols []protocol.Volume) error {770	mounts := make([]string, 0, len(vols))771	batch := &pgx.Batch{}772	for _, v := range vols {773		mounts = append(mounts, v.Mount)774		batch.Queue(`insert into volumes(id, node_id, mount, device, fs, kind, total_bytes, used_bytes, smart_status, remote, last_seen_at)775			values ($1,$2,$3,nullif($4,''),nullif($5,''),$6,$7,$8,nullif($9,''),nullif($10,''), now())776			on conflict (node_id, mount) do update set777				device = excluded.device, fs = excluded.fs, kind = excluded.kind, total_bytes = excluded.total_bytes,778				used_bytes = excluded.used_bytes, smart_status = excluded.smart_status, remote = excluded.remote,779				last_seen_at = now(), gone_at = null`,780			ids.New(), nodeID, v.Mount, v.Device, v.FS, v.Kind, v.Total, v.Used, v.SMART, v.Remote)781	}782	batch.Queue(`update volumes set gone_at = now() where node_id = $1 and gone_at is null and not (mount = any($2::text[]))`, nodeID, mounts)783	return tx.SendBatch(ctx, batch).Close()784}785786// ─────────────────────────── NAS ───────────────────────────787788// nasHosts returns the registered NAS addresses agents should probe (cached 30 s).789func (s *Server) nasHosts(ctx context.Context) []string {790	s.nasMu.Lock()791	defer s.nasMu.Unlock()792	if time.Since(s.nasAt) < 30*time.Second {793		return s.nasTargets794	}795	rows, err := s.pool.Query(ctx, `select host(host) from nas_targets order by name limit $1`, maxNASProbes)796	if err != nil {797		return s.nasTargets798	}799	hosts, err := pgx.CollectRows(rows, pgx.RowTo[string])800	if err != nil {801		return s.nasTargets802	}803	s.nasTargets, s.nasAt = hosts, time.Now()804	return hosts805}806807// applyNAS records this node's view of each registered NAS and recomputes808// the aggregate state from every recent observation: online if any node809// reaches it, offline if none does.810func applyNAS(ctx context.Context, tx pgx.Tx, nodeID uuid.UUID, probes []protocol.NASProbe) error {811	for _, p := range probes {812		var nasID uuid.UUID813		err := tx.QueryRow(ctx, `select id from nas_targets where host = $1::inet for update`, p.Host).Scan(&nasID)814		if errors.Is(err, pgx.ErrNoRows) {815			continue // not a registered target: ignored816		}817		if err != nil {818			return err819		}820		if _, err := tx.Exec(ctx, `insert into nas_observations(nas_id, node_id, reachable, mounted, latency_ms, total_bytes, used_bytes, observed_at)821			values ($1,$2,$3,$4,case when $3 then $5::real end,nullif($6::bigint,0),nullif($7::bigint,0), now())822			on conflict (nas_id, node_id) do update set reachable = excluded.reachable, mounted = excluded.mounted,823				latency_ms = excluded.latency_ms, total_bytes = excluded.total_bytes, used_bytes = excluded.used_bytes, observed_at = excluded.observed_at`,824			nasID, nodeID, p.Reachable, p.Mounted, p.LatencyMS, p.Total, p.Used); err != nil {825			return err826		}827		if _, err := tx.Exec(ctx, `with o as (828				select * from nas_observations where nas_id = $1 and observed_at > now() - interval '5 minutes'829			), agg as (830				select bool_or(reachable) as any_up, count(*) filter (where mounted) as mounted,831				       min(latency_ms) filter (where reachable) as latency,832				       max(total_bytes) as total, max(used_bytes) filter (where total_bytes is not null) as used,833				       (select node_id from o where reachable order by observed_at desc limit 1) as up_by,834				       (select node_id from o order by observed_at desc limit 1) as last_by835				from o836			)837			update nas_targets t set838				state = case when agg.any_up then 'online' else 'offline' end, state_at = now(),839				observed_by = coalesce(agg.up_by, agg.last_by), latency_ms = agg.latency, mounted_on = agg.mounted,840				total_bytes = coalesce(agg.total, t.total_bytes), used_bytes = coalesce(agg.used, t.used_bytes)841			from agg where t.id = $1`, nasID); err != nil {842			return err843		}844	}845	return nil846}847
Files main
Menu

Theme