Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/ingest/rollup_test.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package ingest_test4344import (45 "context"46 "math"47 "net/http/httptest"48 "strings"49 "testing"50 "time"5152 "clustr.spboucher.ai/mcc/internal/protocol"53)5455func TestRollupsAndLateData(t *testing.T) {56 h := newHarness(t, "mtls", hopt{})57 e := h.enroll("n1")58 ctx := context.Background()59 // Reports three hours ago, in two different minutes of the same hour.60 base := time.Now().UTC().Add(-3 * time.Hour).Truncate(time.Hour).Add(10 * time.Minute)61 seq := int64(0)62 push := func(ts time.Time, cpu float64, buffered bool) {63 seq++64 r := report(seq, ts, sample("cpu.usage", cpu), protocol.Sample{M: "net.rx", L: "en0", V: cpu * 10})65 r.Buffered = buffered66 if _, err := h.send(e, r); err != nil {67 t.Fatal(err)68 }69 }70 for i, v := range []float64{10, 20, 60} { // minute :1071 push(base.Add(time.Duration(i)*10*time.Second), v, true)72 }73 for i, v := range []float64{100, 200} { // minute :1174 push(base.Add(time.Minute+time.Duration(i)*10*time.Second), v, true)75 }76 st, ran, err := h.srv.Rollup(ctx, time.Now())77 if err != nil || !ran {78 t.Fatal(err, ran)79 }80 type agg struct {81 avg, min, max float6482 n int83 }84 get := func(table, metric, label string, ts time.Time) agg {85 var a agg86 err := h.pool.QueryRow(ctx, `select s.avg, s.min, s.max, s.n from `+table+` s join metric_catalog c on c.id=s.metric_id87 where s.node_id=$1 and c.name=$2 and s.label=$3 and s.ts=$4`, e.id.NodeID, metric, label, ts).Scan(&a.avg, &a.min, &a.max, &a.n)88 if err != nil {89 t.Fatalf("%s %s %s: %v", table, metric, ts, err)90 }91 return a92 }93 near := func(a, b float64) bool { return math.Abs(a-b) < 1e-9 }94 if a := get("samples_1m", "cpu.usage", "", base); !near(a.avg, 30) || a.min != 10 || a.max != 60 || a.n != 3 {95 t.Errorf("1m :10 = %+v", a)96 }97 if a := get("samples_1m", "net.rx", "en0", base.Add(time.Minute)); !near(a.avg, 1500) || a.min != 1000 || a.max != 2000 || a.n != 2 {98 t.Errorf("1m :11 labeled = %+v", a)99 }100 hour := base.Truncate(time.Hour)101 // The hour average is weighted by sample count: (10+20+60+100+200)/5 = 78, not the mean of minute means (90).102 if a := get("samples_1h", "cpu.usage", "", hour); !near(a.avg, 78) || a.min != 10 || a.max != 200 || a.n != 5 {103 t.Errorf("1h = %+v", a)104 }105 if st.Watermark1m.Before(time.Now().Add(-3*time.Minute)) || st.Watermark1h.Before(time.Now().Add(-2*time.Hour)) {106 t.Errorf("watermarks %+v", st)107 }108 // Idempotent.109 before := scan[int](h, `select (select count(*) from samples_1m) + (select count(*) from samples_1h)`)110 if _, _, err := h.srv.Rollup(ctx, time.Now()); err != nil {111 t.Fatal(err)112 }113 if after := scan[int](h, `select (select count(*) from samples_1m) + (select count(*) from samples_1h)`); after != before {114 t.Errorf("rollup not idempotent: %d -> %d rows", before, after)115 }116 if a := get("samples_1h", "cpu.usage", "", hour); !near(a.avg, 78) || a.n != 5 {117 t.Errorf("1h changed on re-run: %+v", a)118 }119 // Late data: a buffered report lands in a minute and an hour already rolled, far behind the trailing window.120 push(base.Add(30*time.Second), 110, true)121 if scan[int](h, `select count(*) from rollup_dirty where tier='1m'`) != 1 {122 t.Fatal("late report did not flag its minute")123 }124 if _, _, err := h.srv.Rollup(ctx, time.Now()); err != nil {125 t.Fatal(err)126 }127 if a := get("samples_1m", "cpu.usage", "", base); !near(a.avg, 50) || a.max != 110 || a.n != 4 {128 t.Errorf("late minute not re-rolled: %+v", a)129 }130 if a := get("samples_1h", "cpu.usage", "", hour); !near(a.avg, 500.0/6) || a.max != 200 || a.n != 6 {131 t.Errorf("late hour not re-rolled: %+v", a)132 }133 if scan[int](h, `select count(*) from rollup_dirty`) != 0 {134 t.Error("dirty flags not consumed")135 }136 // The current, incomplete minute is not rolled yet.137 push(time.Now(), 5, false)138 h.srv.Rollup(ctx, time.Now())139 if n := scan[int](h, `select count(*) from samples_1m where ts >= date_trunc('minute', now())`); n != 0 {140 t.Errorf("incomplete minute rolled (%d rows)", n)141 }142 // Two concurrent passes: the advisory lock lets only one run.143 done := make(chan bool, 4)144 for i := 0; i < 4; i++ {145 go func() { _, ran, _ := h.srv.Rollup(ctx, time.Now()); done <- ran }()146 }147 for i := 0; i < 4; i++ {148 <-done149 }150 if a := get("samples_1h", "cpu.usage", "", hour); a.n != 6 {151 t.Errorf("concurrent rollups corrupted the hour: %+v", a)152 }153}154155func TestPartitionsRetentionCatalogHeartbeat(t *testing.T) {156 h := newHarness(t, "mtls", hopt{retain: 3 * 24 * time.Hour})157 ctx := context.Background()158 has := func(name string) bool {159 return scan[bool](h, `select exists(select 1 from pg_class where relname=$1)`, name)160 }161 now := time.Now().UTC()162 for d := -3; d <= 3; d++ {163 if name := "samples_raw_p" + now.AddDate(0, 0, d).Format("20060102"); !has(name) {164 t.Errorf("partition %s missing", name)165 }166 }167 if !has("samples_1m_p"+now.Format("200601")) || !has("samples_1h_p"+now.AddDate(0, 1, 0).Format("200601")) {168 t.Error("monthly rollup partitions missing")169 }170 if has("samples_raw_p" + now.AddDate(0, 0, 4).Format("20060102")) {171 t.Error("too many partitions created ahead")172 }173 // Idempotent.174 if err := h.srv.EnsurePartitions(ctx); err != nil {175 t.Fatal(err)176 }177 // Expired partitions are dropped; anything not named by us is never touched.178 h.exec(`create table samples_raw_p20200101 partition of samples_raw for values from ('2020-01-01') to ('2020-01-02')`)179 h.exec(`create table samples_1m_p202001 partition of samples_1m for values from ('2020-01-01') to ('2020-02-01')`)180 h.exec(`create table samples_raw_legacy partition of samples_raw for values from ('2019-01-01') to ('2019-01-02')`)181 h.exec(`insert into samples_raw_p20200101(node_id, metric_id, ts, value) values (gen_random_uuid(), 1, '2020-01-01 12:00+00', 1)`)182 dropped, err := h.srv.DropExpiredPartitions(ctx)183 if err != nil {184 t.Fatal(err)185 }186 if strings.Join(dropped, ",") != "samples_1m_p202001,samples_raw_p20200101" && strings.Join(dropped, ",") != "samples_raw_p20200101,samples_1m_p202001" {187 t.Errorf("dropped %v", dropped)188 }189 if has("samples_raw_p20200101") || !has("samples_raw_legacy") || !has("samples_raw_p"+now.AddDate(0, 0, -3).Format("20060102")) {190 t.Error("wrong partitions dropped")191 }192 if err := h.srv.Retention(ctx); err != nil {193 t.Fatal(err)194 }195 // A buffered report for a day whose partition is missing creates it on demand.196 e := h.enroll("n1")197 day := now.AddDate(0, 0, -2)198 h.exec(`drop table samples_raw_p` + day.Format("20060102"))199 r := report(1, day, sample("cpu.usage", 1))200 r.Buffered = true201 if _, err := h.send(e, r); err != nil {202 t.Fatalf("report into a missing partition: %v", err)203 }204 if scan[int](h, `select count(*) from samples_raw`) != 1 {205 t.Error("sample not stored")206 }207 // Catalog seeded from protocol.Catalog, exactly.208 if n := scan[int](h, `select count(*) from metric_catalog`); n != len(protocol.Catalog) {209 t.Errorf("catalog rows %d, want %d", n, len(protocol.Catalog))210 }211 for _, m := range protocol.Catalog {212 if got := scan[string](h, `select name || '|' || unit || '|' || kind || '|' || interval_s || '|' || labeled from metric_catalog where id=$1`, m.ID); !strings.HasPrefix(got, m.Name+"|"+m.Unit+"|"+m.Kind+"|") {213 t.Errorf("catalog %d: %s", m.ID, got)214 }215 }216 // Heartbeat, health, published CA.217 if err := h.srv.Heartbeat(ctx); err != nil {218 t.Fatal(err)219 }220 if scan[string](h, `select version from service_heartbeats where service='telemetry-ingest'`) != "test" {221 t.Error("heartbeat")222 }223 rec := httptest.NewRecorder()224 h.health.ServeHTTP(rec, httptest.NewRequest("GET", "/healthz", nil))225 if rec.Code != 200 || !strings.Contains(rec.Body.String(), `"connected_agents":1`) {226 t.Errorf("healthz: %d %s", rec.Code, rec.Body.String())227 }228 if err := h.ca.Publish(ctx, h.pool, "ingest.mcc.internal", []string{"https://example.test"}); err != nil {229 t.Fatal(err)230 }231 if fp := scan[string](h, `select value->>'fingerprint' from settings where key='pki.ca'`); fp != h.ca.Fingerprint() || !strings.HasPrefix(fp, "sha256:") {232 t.Errorf("published fingerprint %q", fp)233 }234 if err := h.srv.ServeHealth(ctx, "0.0.0.0:0"); err == nil {235 t.Error("health endpoint accepted a non-loopback address")236 }237}238