SPB Git

Repository Public

maintenance.go 20.1 KB 585 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/ingest/maintenance.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package ingest4344import (45	"context"46	"errors"47	"fmt"48	"net"49	"net/http"50	"regexp"51	"strings"52	"sync"53	"time"5455	"github.com/jackc/pgx/v5"5657	"clustr.spboucher.ai/mcc/internal/events"58	"clustr.spboucher.ai/mcc/internal/ids"59	"clustr.spboucher.ai/mcc/internal/protocol"60	"clustr.spboucher.ai/mcc/internal/secret"61)6263// Advisory lock keys of this service (namespace 0x6d6363 = "mcc").64const (65	lockRollup     = 0x6d6363_1066	lockPartitions = 0x6d6363_1167)6869// Rollup tuning.70const (71	rollupEvery     = 20 * time.Second72	minuteGrace     = 20 * time.Second // a minute is rolled once it is this old73	trailingRaw     = 3 * time.Minute  // always recomputed behind the 1m watermark74	trailingHours   = 2 * time.Hour    // always recomputed behind the 1h watermark75	partitionsAhead = 376)7778// RollupStatus is the outcome of the last rollup pass.79type RollupStatus struct {80	At          time.Time `json:"at"`81	Watermark1m time.Time `json:"watermark_1m"`82	Watermark1h time.Time `json:"watermark_1h"`83	Rows1m      int64     `json:"rows_1m"`84	Rows1h      int64     `json:"rows_1h"`85	Error       string    `json:"error,omitempty"`86}8788// SeedCatalog makes metric_catalog match protocol.Catalog.89func (s *Server) SeedCatalog(ctx context.Context) error {90	batch := &pgx.Batch{}91	for _, m := range protocol.Catalog {92		batch.Queue(`insert into metric_catalog(id, name, unit, kind, interval_s, labeled, source, description)93			values ($1,$2,$3,$4,$5,$6,$7,$8)94			on conflict (id) do update set name = excluded.name, unit = excluded.unit, kind = excluded.kind,95				interval_s = excluded.interval_s, labeled = excluded.labeled, source = excluded.source, description = excluded.description`,96			m.ID, m.Name, m.Unit, m.Kind, m.IntervalS, m.Labeled, m.Source, m.Description)97	}98	return s.pool.SendBatch(ctx, batch).Close()99}100101// ─────────────────────────── partitions ───────────────────────────102103func dayStart(t time.Time) time.Time {104	t = t.UTC()105	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC)106}107108func monthStart(t time.Time) time.Time {109	t = t.UTC()110	return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC)111}112113func rawPartition(day time.Time) string { return "samples_raw_p" + day.UTC().Format("20060102") }114func monthPartition(table string, m time.Time) string {115	return table + "_p" + m.UTC().Format("200601")116}117118const tsLit = "2006-01-02 15:04:05+00"119120func createPartition(ctx context.Context, q interface {121	Exec(context.Context, string, ...any) (pgconnTag, error)122}, parent, name string, from, to time.Time) error {123	// Identifiers are built from fixed prefixes and formatted dates only.124	_, err := q.Exec(ctx, fmt.Sprintf(`create table if not exists %s partition of %s for values from ('%s') to ('%s')`,125		name, parent, from.UTC().Format(tsLit), to.UTC().Format(tsLit)))126	return err127}128129// EnsurePartitions creates the partitions needed to store data at the given130// instants plus the standing window: every day from now-RetainRaw to131// now+3 days for samples_raw, and the matching months for the rollups.132func (s *Server) EnsurePartitions(ctx context.Context, extra ...time.Time) error {133	now := time.Now().UTC()134	conn, err := s.pool.Acquire(ctx)135	if err != nil {136		return err137	}138	defer conn.Release()139	// Serialize DDL between instances and with concurrent on-demand creation.140	if _, err := conn.Exec(ctx, `select pg_advisory_lock($1)`, lockPartitions); err != nil {141		return err142	}143	defer conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock($1)`, lockPartitions)144145	days := map[time.Time]bool{}146	for d := dayStart(now.Add(-s.cfg.RetainRaw)); !d.After(dayStart(now).AddDate(0, 0, partitionsAhead)); d = d.AddDate(0, 0, 1) {147		days[d] = true148	}149	for _, t := range extra {150		if t.After(now.Add(-s.cfg.RetainRaw-24*time.Hour)) && t.Before(now.Add(24*time.Hour)) {151			days[dayStart(t)] = true152		}153	}154	months := map[time.Time]bool{monthStart(now).AddDate(0, 1, 0): true}155	for d := range days {156		if err := createPartition(ctx, conn, "samples_raw", rawPartition(d), d, d.AddDate(0, 0, 1)); err != nil {157			return err158		}159		months[monthStart(d)] = true160	}161	for m := range months {162		for _, table := range []string{"samples_1m", "samples_1h"} {163			if err := createPartition(ctx, conn, table, monthPartition(table, m), m, m.AddDate(0, 1, 0)); err != nil {164				return err165			}166		}167	}168	return nil169}170171var partitionName = regexp.MustCompile(`^(samples_raw|samples_1m|samples_1h)_p(\d{6}|\d{8})$`)172173// DropExpiredPartitions drops partitions lying entirely before the retention174// of their tier and returns their names.175func (s *Server) DropExpiredPartitions(ctx context.Context) ([]string, error) {176	now := time.Now().UTC()177	rows, err := s.pool.Query(ctx, `select c.relname from pg_inherits i178		join pg_class c on c.oid = i.inhrelid join pg_class p on p.oid = i.inhparent179		where p.relname in ('samples_raw','samples_1m','samples_1h') and c.relnamespace = p.relnamespace`)180	if err != nil {181		return nil, err182	}183	names, err := pgx.CollectRows(rows, pgx.RowTo[string])184	if err != nil {185		return nil, err186	}187	var dropped []string188	for _, n := range names {189		m := partitionName.FindStringSubmatch(n)190		if m == nil {191			continue // not one of ours: never touched192		}193		var end, cutoff time.Time194		switch m[1] {195		case "samples_raw":196			d, err := time.Parse("20060102", m[2])197			if err != nil {198				continue199			}200			end, cutoff = d.AddDate(0, 0, 1), now.Add(-s.cfg.RetainRaw)201		default:202			d, err := time.Parse("200601", m[2])203			if err != nil {204				continue205			}206			end = d.AddDate(0, 1, 0)207			cutoff = now.Add(-s.cfg.Retain1m)208			if m[1] == "samples_1h" {209				cutoff = now.Add(-s.cfg.Retain1h)210			}211		}212		if !end.After(cutoff) {213			if _, err := s.pool.Exec(ctx, `drop table if exists `+n); err != nil {214				return dropped, err215			}216			dropped = append(dropped, n)217		}218	}219	return dropped, nil220}221222// Retention removes what partition drops cannot (rollup partitions are223// monthly, their retention is not) and the bookkeeping tables.224func (s *Server) Retention(ctx context.Context) error {225	stmts := []struct {226		sql string227		arg any228	}{229		{`delete from samples_1m where ts < now() - $1::interval`, s.cfg.Retain1m.String()},230		{`delete from samples_1h where ts < now() - $1::interval`, s.cfg.Retain1h.String()},231		{`delete from agent_report_seqs where received_at < now() - $1::interval`, (s.cfg.RetainRaw + 24*time.Hour).String()},232		{`delete from rollup_dirty where bucket < now() - $1::interval`, (s.cfg.RetainRaw + 24*time.Hour).String()},233		{`delete from nas_observations where observed_at < now() - $1::interval`, (24 * time.Hour).String()},234	}235	for _, st := range stmts {236		if _, err := s.pool.Exec(ctx, st.sql, st.arg); err != nil {237			return err238		}239	}240	return nil241}242243// ─────────────────────────── rollups ───────────────────────────244245const roll1mSQL = `insert into samples_1m(node_id, metric_id, label, ts, avg, min, max, n)246	select node_id, metric_id, label, date_trunc('minute', ts), avg(value), min(value), max(value), count(*)247	from samples_raw where ts >= $1 and ts < $2248	group by 1,2,3,4249	on conflict (node_id, metric_id, label, ts) do update set avg = excluded.avg, min = excluded.min, max = excluded.max, n = excluded.n`250251const roll1hSQL = `insert into samples_1h(node_id, metric_id, label, ts, avg, min, max, n)252	select node_id, metric_id, label, date_trunc('hour', ts), sum(avg * n) / sum(n), min(min), max(max), sum(n)253	from samples_1m where ts >= $1 and ts < $2254	group by 1,2,3,4255	on conflict (node_id, metric_id, label, ts) do update set avg = excluded.avg, min = excluded.min, max = excluded.max, n = excluded.n`256257// Rollup runs one raw→1m→1h pass. It is idempotent (buckets are recomputed258// from their source and upserted), tolerant of late data (a trailing window259// behind each watermark and the buckets flagged in rollup_dirty are260// recomputed) and exclusive (transaction-level advisory lock; a concurrent261// pass returns immediately with ran=false).262func (s *Server) Rollup(ctx context.Context, now time.Time) (st RollupStatus, ran bool, err error) {263	now = now.UTC()264	st.At = now265	if err := s.EnsurePartitions(ctx); err != nil {266		return st, false, err267	}268	err = s.inTx(ctx, func(tx pgx.Tx) error {269		var got bool270		if err := tx.QueryRow(ctx, `select pg_try_advisory_xact_lock($1)`, lockRollup).Scan(&got); err != nil {271			return err272		}273		if !got {274			return nil275		}276		ran = true277278		// ── raw → 1m ──279		until1m := now.Add(-minuteGrace).Truncate(time.Minute) // exclusive end of complete minutes280		var wm1m time.Time281		err := tx.QueryRow(ctx, `select watermark from rollup_state where tier = '1m' for update`).Scan(&wm1m)282		if errors.Is(err, pgx.ErrNoRows) {283			var first *time.Time284			if err := tx.QueryRow(ctx, `select min(ts) from samples_raw`).Scan(&first); err != nil {285				return err286			}287			wm1m = until1m288			if first != nil && first.Before(wm1m) {289				wm1m = first.UTC().Truncate(time.Minute)290			}291		} else if err != nil {292			return err293		}294		// Late buckets first (each is one minute), then the regular window.295		late, err := takeDirty(ctx, tx, "1m", until1m)296		if err != nil {297			return err298		}299		hours := map[time.Time]bool{}300		for _, b := range late {301			tag, err := tx.Exec(ctx, roll1mSQL, b, b.Add(time.Minute))302			if err != nil {303				return err304			}305			st.Rows1m += tag.RowsAffected()306			hours[b.Truncate(time.Hour)] = true307		}308		from := wm1m.Add(-trailingRaw)309		if until1m.After(from) {310			tag, err := tx.Exec(ctx, roll1mSQL, from, until1m)311			if err != nil {312				return err313			}314			st.Rows1m += tag.RowsAffected()315		}316		if until1m.After(wm1m) {317			wm1m = until1m318		}319		if _, err := tx.Exec(ctx, `insert into rollup_state(tier, watermark) values ('1m', $1)320			on conflict (tier) do update set watermark = excluded.watermark`, wm1m); err != nil {321			return err322		}323		st.Watermark1m = wm1m324325		// ── 1m → 1h ──326		until1h := wm1m.Truncate(time.Hour) // complete hours only327		var wm1h time.Time328		err = tx.QueryRow(ctx, `select watermark from rollup_state where tier = '1h' for update`).Scan(&wm1h)329		if errors.Is(err, pgx.ErrNoRows) {330			var first *time.Time331			if err := tx.QueryRow(ctx, `select min(ts) from samples_1m`).Scan(&first); err != nil {332				return err333			}334			wm1h = until1h335			if first != nil && first.Before(wm1h) {336				wm1h = first.UTC().Truncate(time.Hour)337			}338		} else if err != nil {339			return err340		}341		// Hours touched by late minutes, when already rolled.342		for h := range hours {343			if h.Before(wm1h) {344				if _, err := tx.Exec(ctx, `insert into rollup_dirty(tier, bucket) values ('1h', $1) on conflict do nothing`, h); err != nil {345					return err346				}347			}348		}349		lateH, err := takeDirty(ctx, tx, "1h", until1h)350		if err != nil {351			return err352		}353		for _, b := range lateH {354			tag, err := tx.Exec(ctx, roll1hSQL, b, b.Add(time.Hour))355			if err != nil {356				return err357			}358			st.Rows1h += tag.RowsAffected()359		}360		// The regular window runs when an hour completes; late minutes reach361		// already rolled hours through rollup_dirty above.362		fromH := wm1h.Add(-trailingHours)363		if until1h.After(wm1h) {364			tag, err := tx.Exec(ctx, roll1hSQL, fromH, until1h)365			if err != nil {366				return err367			}368			st.Rows1h += tag.RowsAffected()369		}370		if until1h.After(wm1h) {371			wm1h = until1h372		}373		if _, err := tx.Exec(ctx, `insert into rollup_state(tier, watermark) values ('1h', $1)374			on conflict (tier) do update set watermark = excluded.watermark`, wm1h); err != nil {375			return err376		}377		st.Watermark1h = wm1h378		return nil379	})380	return st, ran, err381}382383// takeDirty removes and returns the flagged buckets that are complete.384func takeDirty(ctx context.Context, tx pgx.Tx, tier string, before time.Time) ([]time.Time, error) {385	rows, err := tx.Query(ctx, `delete from rollup_dirty where tier = $1 and bucket < $2 returning bucket`, tier, before)386	if err != nil {387		return nil, err388	}389	return pgx.CollectRows(rows, pgx.RowTo[time.Time])390}391392// ─────────────────────────── background ───────────────────────────393394// Background runs the periodic jobs until ctx ends. Every job tolerates a395// database outage: it logs, and runs again at its next tick.396func (s *Server) Background(ctx context.Context) {397	var wg sync.WaitGroup398	run := func(name string, every time.Duration, immediate bool, fn func(context.Context) error) {399		wg.Add(1)400		go func() {401			defer wg.Done()402			t := time.NewTicker(every)403			defer t.Stop()404			for first := true; ; first = false {405				if !first || immediate {406					jctx, cancel := context.WithTimeout(ctx, 2*time.Minute)407					err := func() (err error) {408						defer func() {409							if v := recover(); v != nil {410								err = fmt.Errorf("panic: %v", v)411							}412						}()413						return fn(jctx)414					}()415					cancel()416					if err != nil && ctx.Err() == nil {417						s.log.Warn("ingest: background job failed", "job", name, "err", err)418					}419				}420				select {421				case <-ctx.Done():422					return423				case <-t.C:424				}425			}426		}()427	}428	wg.Add(1)429	go func() { defer wg.Done(); s.listenCommands(ctx) }()430431	run("startup", time.Minute, true, s.startupOnce())432	run("heartbeat", 10*time.Second, true, s.Heartbeat)433	run("rollup", rollupEvery, true, func(ctx context.Context) error {434		st, ran, err := s.Rollup(ctx, time.Now())435		if err != nil {436			st.Error = err.Error()437			if prev := s.lastRollup.Load(); prev != nil {438				st.Watermark1m, st.Watermark1h = prev.Watermark1m, prev.Watermark1h439			}440			s.lastRollup.Store(&st)441			return err442		}443		if ran {444			s.lastRollup.Store(&st)445		}446		return nil447	})448	run("partitions", time.Hour, false, func(ctx context.Context) error {449		if err := s.EnsurePartitions(ctx); err != nil {450			return err451		}452		dropped, err := s.DropExpiredPartitions(ctx)453		if len(dropped) > 0 {454			s.log.Info("ingest: dropped expired partitions", "partitions", dropped)455		}456		if err != nil {457			return err458		}459		return s.Retention(ctx)460	})461	run("events-prune", 5*time.Minute, false, func(ctx context.Context) error { return events.Prune(ctx, s.pool) })462	wg.Wait()463}464465// startupOnce retries the start-of-day work until it has succeeded once466// (the database may be down when the service starts).467func (s *Server) startupOnce() func(context.Context) error {468	done := false469	return func(ctx context.Context) error {470		if done {471			return nil472		}473		if err := s.SeedCatalog(ctx); err != nil {474			return err475		}476		if err := s.EnsurePartitions(ctx); err != nil {477			return err478		}479		if err := s.ca.Publish(ctx, s.pool, s.cfg.IngestServerName, s.cfg.IngestAdvertise); err != nil {480			return err481		}482		if _, err := s.DropExpiredPartitions(ctx); err != nil {483			return err484		}485		done = true486		s.log.Info("ingest: startup tasks done", "ca_fingerprint", s.ca.Fingerprint())487		return nil488	}489}490491// Heartbeat records this instance in service_heartbeats.492func (s *Server) Heartbeat(ctx context.Context) error {493	detail := map[string]any{"connected_agents": s.connectedAgents(), "listen": s.cfg.IngestListen}494	if r := s.lastRollup.Load(); r != nil {495		detail["last_rollup"] = r496	}497	_, err := s.pool.Exec(ctx, `insert into service_heartbeats(service, instance, version, started_at, beat_at, detail)498		values ('telemetry-ingest', $1, $2, $3, now(), $4::jsonb)499		on conflict (service) do update set instance = excluded.instance, version = excluded.version,500			started_at = excluded.started_at, beat_at = excluded.beat_at, detail = excluded.detail`,501		s.inst, s.version, s.started, mustJSON(detail))502	s.dbOK.Store(err == nil)503	return err504}505506// ─────────────────────────── health ───────────────────────────507508// HealthHandler serves GET /healthz for the loopback-only plain HTTP port.509func (s *Server) HealthHandler() http.Handler {510	mux := http.NewServeMux()511	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {512		ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)513		defer cancel()514		dbErr := s.pool.Ping(ctx)515		s.dbOK.Store(dbErr == nil)516		body := map[string]any{517			"service": "telemetry-ingest", "version": s.version, "instance": s.inst,518			"started_at": s.started, "uptime_s": int64(time.Since(s.started).Seconds()),519			"db": map[string]any{"ok": dbErr == nil}, "connected_agents": s.connectedAgents(), "long_polls": s.ActiveLongPolls(),520			"last_rollup": s.lastRollup.Load(), "ca_fingerprint": s.ca.Fingerprint(),521		}522		status := http.StatusOK523		body["status"] = "ok"524		if dbErr != nil {525			status, body["status"] = http.StatusServiceUnavailable, "degraded"526		}527		writeJSON(w, status, body)528	})529	return mux530}531532// ServeHealth serves the health endpoint on addr, which must be loopback.533func (s *Server) ServeHealth(ctx context.Context, addr string) error {534	host, _, err := net.SplitHostPort(addr)535	if err != nil {536		return fmt.Errorf("health listen address: %w", err)537	}538	if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) {539		return fmt.Errorf("health endpoint must listen on a loopback address, got %q", addr)540	}541	ln, err := net.Listen("tcp", addr)542	if err != nil {543		return err544	}545	srv := &http.Server{Handler: s.HealthHandler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second}546	go func() {547		<-ctx.Done()548		sctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)549		defer cancel()550		srv.Shutdown(sctx)551	}()552	if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {553		return err554	}555	return nil556}557558// ─────────────────────────── dev helper ───────────────────────────559560// CreateEnrollmentToken inserts a one-time enrollment token and returns its561// clear value. Production tokens are created by the control API; this exists562// for the dev-token subcommand and for tests.563func CreateEnrollmentToken(ctx context.Context, db interface {564	Exec(context.Context, string, ...any) (pgconnTag, error)565}, alias, kind string, ttl time.Duration) (string, error) {566	alias = strings.TrimSpace(alias)567	if alias == "" {568		return "", errors.New("alias is required")569	}570	if kind == "" {571		kind = "mac"572	}573	clear, hash := secret.NewToken()574	_, err := db.Exec(ctx, `insert into agent_enrollment_tokens(id, token_hash, node_alias, node_kind, expires_at)575		values ($1,$2,$3,$4::node_kind, now() + $5::interval)`, ids.New(), hash, alias, kind, ttl.String())576	if err != nil {577		return "", err578	}579	return clear, nil580}581582// ListenCommandsForTest runs only the command notification listener (tests583// drive the other background jobs explicitly).584func (s *Server) ListenCommandsForTest(ctx context.Context) { s.listenCommands(ctx) }585
Files main
Menu

Theme