Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/ingest/maintenance.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package ingest4344import (45 "context"46 "errors"47 "fmt"48 "net"49 "net/http"50 "regexp"51 "strings"52 "sync"53 "time"5455 "github.com/jackc/pgx/v5"5657 "clustr.spboucher.ai/mcc/internal/events"58 "clustr.spboucher.ai/mcc/internal/ids"59 "clustr.spboucher.ai/mcc/internal/protocol"60 "clustr.spboucher.ai/mcc/internal/secret"61)6263// Advisory lock keys of this service (namespace 0x6d6363 = "mcc").64const (65 lockRollup = 0x6d6363_1066 lockPartitions = 0x6d6363_1167)6869// Rollup tuning.70const (71 rollupEvery = 20 * time.Second72 minuteGrace = 20 * time.Second // a minute is rolled once it is this old73 trailingRaw = 3 * time.Minute // always recomputed behind the 1m watermark74 trailingHours = 2 * time.Hour // always recomputed behind the 1h watermark75 partitionsAhead = 376)7778// RollupStatus is the outcome of the last rollup pass.79type RollupStatus struct {80 At time.Time `json:"at"`81 Watermark1m time.Time `json:"watermark_1m"`82 Watermark1h time.Time `json:"watermark_1h"`83 Rows1m int64 `json:"rows_1m"`84 Rows1h int64 `json:"rows_1h"`85 Error string `json:"error,omitempty"`86}8788// SeedCatalog makes metric_catalog match protocol.Catalog.89func (s *Server) SeedCatalog(ctx context.Context) error {90 batch := &pgx.Batch{}91 for _, m := range protocol.Catalog {92 batch.Queue(`insert into metric_catalog(id, name, unit, kind, interval_s, labeled, source, description)93 values ($1,$2,$3,$4,$5,$6,$7,$8)94 on conflict (id) do update set name = excluded.name, unit = excluded.unit, kind = excluded.kind,95 interval_s = excluded.interval_s, labeled = excluded.labeled, source = excluded.source, description = excluded.description`,96 m.ID, m.Name, m.Unit, m.Kind, m.IntervalS, m.Labeled, m.Source, m.Description)97 }98 return s.pool.SendBatch(ctx, batch).Close()99}100101// ─────────────────────────── partitions ───────────────────────────102103func dayStart(t time.Time) time.Time {104 t = t.UTC()105 return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC)106}107108func monthStart(t time.Time) time.Time {109 t = t.UTC()110 return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC)111}112113func rawPartition(day time.Time) string { return "samples_raw_p" + day.UTC().Format("20060102") }114func monthPartition(table string, m time.Time) string {115 return table + "_p" + m.UTC().Format("200601")116}117118const tsLit = "2006-01-02 15:04:05+00"119120func createPartition(ctx context.Context, q interface {121 Exec(context.Context, string, ...any) (pgconnTag, error)122}, parent, name string, from, to time.Time) error {123 // Identifiers are built from fixed prefixes and formatted dates only.124 _, err := q.Exec(ctx, fmt.Sprintf(`create table if not exists %s partition of %s for values from ('%s') to ('%s')`,125 name, parent, from.UTC().Format(tsLit), to.UTC().Format(tsLit)))126 return err127}128129// EnsurePartitions creates the partitions needed to store data at the given130// instants plus the standing window: every day from now-RetainRaw to131// now+3 days for samples_raw, and the matching months for the rollups.132func (s *Server) EnsurePartitions(ctx context.Context, extra ...time.Time) error {133 now := time.Now().UTC()134 conn, err := s.pool.Acquire(ctx)135 if err != nil {136 return err137 }138 defer conn.Release()139 // Serialize DDL between instances and with concurrent on-demand creation.140 if _, err := conn.Exec(ctx, `select pg_advisory_lock($1)`, lockPartitions); err != nil {141 return err142 }143 defer conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock($1)`, lockPartitions)144145 days := map[time.Time]bool{}146 for d := dayStart(now.Add(-s.cfg.RetainRaw)); !d.After(dayStart(now).AddDate(0, 0, partitionsAhead)); d = d.AddDate(0, 0, 1) {147 days[d] = true148 }149 for _, t := range extra {150 if t.After(now.Add(-s.cfg.RetainRaw-24*time.Hour)) && t.Before(now.Add(24*time.Hour)) {151 days[dayStart(t)] = true152 }153 }154 months := map[time.Time]bool{monthStart(now).AddDate(0, 1, 0): true}155 for d := range days {156 if err := createPartition(ctx, conn, "samples_raw", rawPartition(d), d, d.AddDate(0, 0, 1)); err != nil {157 return err158 }159 months[monthStart(d)] = true160 }161 for m := range months {162 for _, table := range []string{"samples_1m", "samples_1h"} {163 if err := createPartition(ctx, conn, table, monthPartition(table, m), m, m.AddDate(0, 1, 0)); err != nil {164 return err165 }166 }167 }168 return nil169}170171var partitionName = regexp.MustCompile(`^(samples_raw|samples_1m|samples_1h)_p(\d{6}|\d{8})$`)172173// DropExpiredPartitions drops partitions lying entirely before the retention174// of their tier and returns their names.175func (s *Server) DropExpiredPartitions(ctx context.Context) ([]string, error) {176 now := time.Now().UTC()177 rows, err := s.pool.Query(ctx, `select c.relname from pg_inherits i178 join pg_class c on c.oid = i.inhrelid join pg_class p on p.oid = i.inhparent179 where p.relname in ('samples_raw','samples_1m','samples_1h') and c.relnamespace = p.relnamespace`)180 if err != nil {181 return nil, err182 }183 names, err := pgx.CollectRows(rows, pgx.RowTo[string])184 if err != nil {185 return nil, err186 }187 var dropped []string188 for _, n := range names {189 m := partitionName.FindStringSubmatch(n)190 if m == nil {191 continue // not one of ours: never touched192 }193 var end, cutoff time.Time194 switch m[1] {195 case "samples_raw":196 d, err := time.Parse("20060102", m[2])197 if err != nil {198 continue199 }200 end, cutoff = d.AddDate(0, 0, 1), now.Add(-s.cfg.RetainRaw)201 default:202 d, err := time.Parse("200601", m[2])203 if err != nil {204 continue205 }206 end = d.AddDate(0, 1, 0)207 cutoff = now.Add(-s.cfg.Retain1m)208 if m[1] == "samples_1h" {209 cutoff = now.Add(-s.cfg.Retain1h)210 }211 }212 if !end.After(cutoff) {213 if _, err := s.pool.Exec(ctx, `drop table if exists `+n); err != nil {214 return dropped, err215 }216 dropped = append(dropped, n)217 }218 }219 return dropped, nil220}221222// Retention removes what partition drops cannot (rollup partitions are223// monthly, their retention is not) and the bookkeeping tables.224func (s *Server) Retention(ctx context.Context) error {225 stmts := []struct {226 sql string227 arg any228 }{229 {`delete from samples_1m where ts < now() - $1::interval`, s.cfg.Retain1m.String()},230 {`delete from samples_1h where ts < now() - $1::interval`, s.cfg.Retain1h.String()},231 {`delete from agent_report_seqs where received_at < now() - $1::interval`, (s.cfg.RetainRaw + 24*time.Hour).String()},232 {`delete from rollup_dirty where bucket < now() - $1::interval`, (s.cfg.RetainRaw + 24*time.Hour).String()},233 {`delete from nas_observations where observed_at < now() - $1::interval`, (24 * time.Hour).String()},234 }235 for _, st := range stmts {236 if _, err := s.pool.Exec(ctx, st.sql, st.arg); err != nil {237 return err238 }239 }240 return nil241}242243// ─────────────────────────── rollups ───────────────────────────244245const roll1mSQL = `insert into samples_1m(node_id, metric_id, label, ts, avg, min, max, n)246 select node_id, metric_id, label, date_trunc('minute', ts), avg(value), min(value), max(value), count(*)247 from samples_raw where ts >= $1 and ts < $2248 group by 1,2,3,4249 on conflict (node_id, metric_id, label, ts) do update set avg = excluded.avg, min = excluded.min, max = excluded.max, n = excluded.n`250251const roll1hSQL = `insert into samples_1h(node_id, metric_id, label, ts, avg, min, max, n)252 select node_id, metric_id, label, date_trunc('hour', ts), sum(avg * n) / sum(n), min(min), max(max), sum(n)253 from samples_1m where ts >= $1 and ts < $2254 group by 1,2,3,4255 on conflict (node_id, metric_id, label, ts) do update set avg = excluded.avg, min = excluded.min, max = excluded.max, n = excluded.n`256257// Rollup runs one raw→1m→1h pass. It is idempotent (buckets are recomputed258// from their source and upserted), tolerant of late data (a trailing window259// behind each watermark and the buckets flagged in rollup_dirty are260// recomputed) and exclusive (transaction-level advisory lock; a concurrent261// pass returns immediately with ran=false).262func (s *Server) Rollup(ctx context.Context, now time.Time) (st RollupStatus, ran bool, err error) {263 now = now.UTC()264 st.At = now265 if err := s.EnsurePartitions(ctx); err != nil {266 return st, false, err267 }268 err = s.inTx(ctx, func(tx pgx.Tx) error {269 var got bool270 if err := tx.QueryRow(ctx, `select pg_try_advisory_xact_lock($1)`, lockRollup).Scan(&got); err != nil {271 return err272 }273 if !got {274 return nil275 }276 ran = true277278 // ── raw → 1m ──279 until1m := now.Add(-minuteGrace).Truncate(time.Minute) // exclusive end of complete minutes280 var wm1m time.Time281 err := tx.QueryRow(ctx, `select watermark from rollup_state where tier = '1m' for update`).Scan(&wm1m)282 if errors.Is(err, pgx.ErrNoRows) {283 var first *time.Time284 if err := tx.QueryRow(ctx, `select min(ts) from samples_raw`).Scan(&first); err != nil {285 return err286 }287 wm1m = until1m288 if first != nil && first.Before(wm1m) {289 wm1m = first.UTC().Truncate(time.Minute)290 }291 } else if err != nil {292 return err293 }294 // Late buckets first (each is one minute), then the regular window.295 late, err := takeDirty(ctx, tx, "1m", until1m)296 if err != nil {297 return err298 }299 hours := map[time.Time]bool{}300 for _, b := range late {301 tag, err := tx.Exec(ctx, roll1mSQL, b, b.Add(time.Minute))302 if err != nil {303 return err304 }305 st.Rows1m += tag.RowsAffected()306 hours[b.Truncate(time.Hour)] = true307 }308 from := wm1m.Add(-trailingRaw)309 if until1m.After(from) {310 tag, err := tx.Exec(ctx, roll1mSQL, from, until1m)311 if err != nil {312 return err313 }314 st.Rows1m += tag.RowsAffected()315 }316 if until1m.After(wm1m) {317 wm1m = until1m318 }319 if _, err := tx.Exec(ctx, `insert into rollup_state(tier, watermark) values ('1m', $1)320 on conflict (tier) do update set watermark = excluded.watermark`, wm1m); err != nil {321 return err322 }323 st.Watermark1m = wm1m324325 // ── 1m → 1h ──326 until1h := wm1m.Truncate(time.Hour) // complete hours only327 var wm1h time.Time328 err = tx.QueryRow(ctx, `select watermark from rollup_state where tier = '1h' for update`).Scan(&wm1h)329 if errors.Is(err, pgx.ErrNoRows) {330 var first *time.Time331 if err := tx.QueryRow(ctx, `select min(ts) from samples_1m`).Scan(&first); err != nil {332 return err333 }334 wm1h = until1h335 if first != nil && first.Before(wm1h) {336 wm1h = first.UTC().Truncate(time.Hour)337 }338 } else if err != nil {339 return err340 }341 // Hours touched by late minutes, when already rolled.342 for h := range hours {343 if h.Before(wm1h) {344 if _, err := tx.Exec(ctx, `insert into rollup_dirty(tier, bucket) values ('1h', $1) on conflict do nothing`, h); err != nil {345 return err346 }347 }348 }349 lateH, err := takeDirty(ctx, tx, "1h", until1h)350 if err != nil {351 return err352 }353 for _, b := range lateH {354 tag, err := tx.Exec(ctx, roll1hSQL, b, b.Add(time.Hour))355 if err != nil {356 return err357 }358 st.Rows1h += tag.RowsAffected()359 }360 // The regular window runs when an hour completes; late minutes reach361 // already rolled hours through rollup_dirty above.362 fromH := wm1h.Add(-trailingHours)363 if until1h.After(wm1h) {364 tag, err := tx.Exec(ctx, roll1hSQL, fromH, until1h)365 if err != nil {366 return err367 }368 st.Rows1h += tag.RowsAffected()369 }370 if until1h.After(wm1h) {371 wm1h = until1h372 }373 if _, err := tx.Exec(ctx, `insert into rollup_state(tier, watermark) values ('1h', $1)374 on conflict (tier) do update set watermark = excluded.watermark`, wm1h); err != nil {375 return err376 }377 st.Watermark1h = wm1h378 return nil379 })380 return st, ran, err381}382383// takeDirty removes and returns the flagged buckets that are complete.384func takeDirty(ctx context.Context, tx pgx.Tx, tier string, before time.Time) ([]time.Time, error) {385 rows, err := tx.Query(ctx, `delete from rollup_dirty where tier = $1 and bucket < $2 returning bucket`, tier, before)386 if err != nil {387 return nil, err388 }389 return pgx.CollectRows(rows, pgx.RowTo[time.Time])390}391392// ─────────────────────────── background ───────────────────────────393394// Background runs the periodic jobs until ctx ends. Every job tolerates a395// database outage: it logs, and runs again at its next tick.396func (s *Server) Background(ctx context.Context) {397 var wg sync.WaitGroup398 run := func(name string, every time.Duration, immediate bool, fn func(context.Context) error) {399 wg.Add(1)400 go func() {401 defer wg.Done()402 t := time.NewTicker(every)403 defer t.Stop()404 for first := true; ; first = false {405 if !first || immediate {406 jctx, cancel := context.WithTimeout(ctx, 2*time.Minute)407 err := func() (err error) {408 defer func() {409 if v := recover(); v != nil {410 err = fmt.Errorf("panic: %v", v)411 }412 }()413 return fn(jctx)414 }()415 cancel()416 if err != nil && ctx.Err() == nil {417 s.log.Warn("ingest: background job failed", "job", name, "err", err)418 }419 }420 select {421 case <-ctx.Done():422 return423 case <-t.C:424 }425 }426 }()427 }428 wg.Add(1)429 go func() { defer wg.Done(); s.listenCommands(ctx) }()430431 run("startup", time.Minute, true, s.startupOnce())432 run("heartbeat", 10*time.Second, true, s.Heartbeat)433 run("rollup", rollupEvery, true, func(ctx context.Context) error {434 st, ran, err := s.Rollup(ctx, time.Now())435 if err != nil {436 st.Error = err.Error()437 if prev := s.lastRollup.Load(); prev != nil {438 st.Watermark1m, st.Watermark1h = prev.Watermark1m, prev.Watermark1h439 }440 s.lastRollup.Store(&st)441 return err442 }443 if ran {444 s.lastRollup.Store(&st)445 }446 return nil447 })448 run("partitions", time.Hour, false, func(ctx context.Context) error {449 if err := s.EnsurePartitions(ctx); err != nil {450 return err451 }452 dropped, err := s.DropExpiredPartitions(ctx)453 if len(dropped) > 0 {454 s.log.Info("ingest: dropped expired partitions", "partitions", dropped)455 }456 if err != nil {457 return err458 }459 return s.Retention(ctx)460 })461 run("events-prune", 5*time.Minute, false, func(ctx context.Context) error { return events.Prune(ctx, s.pool) })462 wg.Wait()463}464465// startupOnce retries the start-of-day work until it has succeeded once466// (the database may be down when the service starts).467func (s *Server) startupOnce() func(context.Context) error {468 done := false469 return func(ctx context.Context) error {470 if done {471 return nil472 }473 if err := s.SeedCatalog(ctx); err != nil {474 return err475 }476 if err := s.EnsurePartitions(ctx); err != nil {477 return err478 }479 if err := s.ca.Publish(ctx, s.pool, s.cfg.IngestServerName, s.cfg.IngestAdvertise); err != nil {480 return err481 }482 if _, err := s.DropExpiredPartitions(ctx); err != nil {483 return err484 }485 done = true486 s.log.Info("ingest: startup tasks done", "ca_fingerprint", s.ca.Fingerprint())487 return nil488 }489}490491// Heartbeat records this instance in service_heartbeats.492func (s *Server) Heartbeat(ctx context.Context) error {493 detail := map[string]any{"connected_agents": s.connectedAgents(), "listen": s.cfg.IngestListen}494 if r := s.lastRollup.Load(); r != nil {495 detail["last_rollup"] = r496 }497 _, err := s.pool.Exec(ctx, `insert into service_heartbeats(service, instance, version, started_at, beat_at, detail)498 values ('telemetry-ingest', $1, $2, $3, now(), $4::jsonb)499 on conflict (service) do update set instance = excluded.instance, version = excluded.version,500 started_at = excluded.started_at, beat_at = excluded.beat_at, detail = excluded.detail`,501 s.inst, s.version, s.started, mustJSON(detail))502 s.dbOK.Store(err == nil)503 return err504}505506// ─────────────────────────── health ───────────────────────────507508// HealthHandler serves GET /healthz for the loopback-only plain HTTP port.509func (s *Server) HealthHandler() http.Handler {510 mux := http.NewServeMux()511 mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {512 ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)513 defer cancel()514 dbErr := s.pool.Ping(ctx)515 s.dbOK.Store(dbErr == nil)516 body := map[string]any{517 "service": "telemetry-ingest", "version": s.version, "instance": s.inst,518 "started_at": s.started, "uptime_s": int64(time.Since(s.started).Seconds()),519 "db": map[string]any{"ok": dbErr == nil}, "connected_agents": s.connectedAgents(), "long_polls": s.ActiveLongPolls(),520 "last_rollup": s.lastRollup.Load(), "ca_fingerprint": s.ca.Fingerprint(),521 }522 status := http.StatusOK523 body["status"] = "ok"524 if dbErr != nil {525 status, body["status"] = http.StatusServiceUnavailable, "degraded"526 }527 writeJSON(w, status, body)528 })529 return mux530}531532// ServeHealth serves the health endpoint on addr, which must be loopback.533func (s *Server) ServeHealth(ctx context.Context, addr string) error {534 host, _, err := net.SplitHostPort(addr)535 if err != nil {536 return fmt.Errorf("health listen address: %w", err)537 }538 if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) {539 return fmt.Errorf("health endpoint must listen on a loopback address, got %q", addr)540 }541 ln, err := net.Listen("tcp", addr)542 if err != nil {543 return err544 }545 srv := &http.Server{Handler: s.HealthHandler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second}546 go func() {547 <-ctx.Done()548 sctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)549 defer cancel()550 srv.Shutdown(sctx)551 }()552 if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {553 return err554 }555 return nil556}557558// ─────────────────────────── dev helper ───────────────────────────559560// CreateEnrollmentToken inserts a one-time enrollment token and returns its561// clear value. Production tokens are created by the control API; this exists562// for the dev-token subcommand and for tests.563func CreateEnrollmentToken(ctx context.Context, db interface {564 Exec(context.Context, string, ...any) (pgconnTag, error)565}, alias, kind string, ttl time.Duration) (string, error) {566 alias = strings.TrimSpace(alias)567 if alias == "" {568 return "", errors.New("alias is required")569 }570 if kind == "" {571 kind = "mac"572 }573 clear, hash := secret.NewToken()574 _, err := db.Exec(ctx, `insert into agent_enrollment_tokens(id, token_hash, node_alias, node_kind, expires_at)575 values ($1,$2,$3,$4::node_kind, now() + $5::interval)`, ids.New(), hash, alias, kind, ttl.String())576 if err != nil {577 return "", err578 }579 return clear, nil580}581582// ListenCommandsForTest runs only the command notification listener (tests583// drive the other background jobs explicitly).584func (s *Server) ListenCommandsForTest(ctx context.Context) { s.listenCommands(ctx) }585