SPB Git

Repository Public

rollup_test.go 9.5 KB 238 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/ingest/rollup_test.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package ingest_test4344import (45	"context"46	"math"47	"net/http/httptest"48	"strings"49	"testing"50	"time"5152	"clustr.spboucher.ai/mcc/internal/protocol"53)5455func TestRollupsAndLateData(t *testing.T) {56	h := newHarness(t, "mtls", hopt{})57	e := h.enroll("n1")58	ctx := context.Background()59	// Reports three hours ago, in two different minutes of the same hour.60	base := time.Now().UTC().Add(-3 * time.Hour).Truncate(time.Hour).Add(10 * time.Minute)61	seq := int64(0)62	push := func(ts time.Time, cpu float64, buffered bool) {63		seq++64		r := report(seq, ts, sample("cpu.usage", cpu), protocol.Sample{M: "net.rx", L: "en0", V: cpu * 10})65		r.Buffered = buffered66		if _, err := h.send(e, r); err != nil {67			t.Fatal(err)68		}69	}70	for i, v := range []float64{10, 20, 60} { // minute :1071		push(base.Add(time.Duration(i)*10*time.Second), v, true)72	}73	for i, v := range []float64{100, 200} { // minute :1174		push(base.Add(time.Minute+time.Duration(i)*10*time.Second), v, true)75	}76	st, ran, err := h.srv.Rollup(ctx, time.Now())77	if err != nil || !ran {78		t.Fatal(err, ran)79	}80	type agg struct {81		avg, min, max float6482		n             int83	}84	get := func(table, metric, label string, ts time.Time) agg {85		var a agg86		err := h.pool.QueryRow(ctx, `select s.avg, s.min, s.max, s.n from `+table+` s join metric_catalog c on c.id=s.metric_id87			where s.node_id=$1 and c.name=$2 and s.label=$3 and s.ts=$4`, e.id.NodeID, metric, label, ts).Scan(&a.avg, &a.min, &a.max, &a.n)88		if err != nil {89			t.Fatalf("%s %s %s: %v", table, metric, ts, err)90		}91		return a92	}93	near := func(a, b float64) bool { return math.Abs(a-b) < 1e-9 }94	if a := get("samples_1m", "cpu.usage", "", base); !near(a.avg, 30) || a.min != 10 || a.max != 60 || a.n != 3 {95		t.Errorf("1m :10 = %+v", a)96	}97	if a := get("samples_1m", "net.rx", "en0", base.Add(time.Minute)); !near(a.avg, 1500) || a.min != 1000 || a.max != 2000 || a.n != 2 {98		t.Errorf("1m :11 labeled = %+v", a)99	}100	hour := base.Truncate(time.Hour)101	// The hour average is weighted by sample count: (10+20+60+100+200)/5 = 78, not the mean of minute means (90).102	if a := get("samples_1h", "cpu.usage", "", hour); !near(a.avg, 78) || a.min != 10 || a.max != 200 || a.n != 5 {103		t.Errorf("1h = %+v", a)104	}105	if st.Watermark1m.Before(time.Now().Add(-3*time.Minute)) || st.Watermark1h.Before(time.Now().Add(-2*time.Hour)) {106		t.Errorf("watermarks %+v", st)107	}108	// Idempotent.109	before := scan[int](h, `select (select count(*) from samples_1m) + (select count(*) from samples_1h)`)110	if _, _, err := h.srv.Rollup(ctx, time.Now()); err != nil {111		t.Fatal(err)112	}113	if after := scan[int](h, `select (select count(*) from samples_1m) + (select count(*) from samples_1h)`); after != before {114		t.Errorf("rollup not idempotent: %d -> %d rows", before, after)115	}116	if a := get("samples_1h", "cpu.usage", "", hour); !near(a.avg, 78) || a.n != 5 {117		t.Errorf("1h changed on re-run: %+v", a)118	}119	// Late data: a buffered report lands in a minute and an hour already rolled, far behind the trailing window.120	push(base.Add(30*time.Second), 110, true)121	if scan[int](h, `select count(*) from rollup_dirty where tier='1m'`) != 1 {122		t.Fatal("late report did not flag its minute")123	}124	if _, _, err := h.srv.Rollup(ctx, time.Now()); err != nil {125		t.Fatal(err)126	}127	if a := get("samples_1m", "cpu.usage", "", base); !near(a.avg, 50) || a.max != 110 || a.n != 4 {128		t.Errorf("late minute not re-rolled: %+v", a)129	}130	if a := get("samples_1h", "cpu.usage", "", hour); !near(a.avg, 500.0/6) || a.max != 200 || a.n != 6 {131		t.Errorf("late hour not re-rolled: %+v", a)132	}133	if scan[int](h, `select count(*) from rollup_dirty`) != 0 {134		t.Error("dirty flags not consumed")135	}136	// The current, incomplete minute is not rolled yet.137	push(time.Now(), 5, false)138	h.srv.Rollup(ctx, time.Now())139	if n := scan[int](h, `select count(*) from samples_1m where ts >= date_trunc('minute', now())`); n != 0 {140		t.Errorf("incomplete minute rolled (%d rows)", n)141	}142	// Two concurrent passes: the advisory lock lets only one run.143	done := make(chan bool, 4)144	for i := 0; i < 4; i++ {145		go func() { _, ran, _ := h.srv.Rollup(ctx, time.Now()); done <- ran }()146	}147	for i := 0; i < 4; i++ {148		<-done149	}150	if a := get("samples_1h", "cpu.usage", "", hour); a.n != 6 {151		t.Errorf("concurrent rollups corrupted the hour: %+v", a)152	}153}154155func TestPartitionsRetentionCatalogHeartbeat(t *testing.T) {156	h := newHarness(t, "mtls", hopt{retain: 3 * 24 * time.Hour})157	ctx := context.Background()158	has := func(name string) bool {159		return scan[bool](h, `select exists(select 1 from pg_class where relname=$1)`, name)160	}161	now := time.Now().UTC()162	for d := -3; d <= 3; d++ {163		if name := "samples_raw_p" + now.AddDate(0, 0, d).Format("20060102"); !has(name) {164			t.Errorf("partition %s missing", name)165		}166	}167	if !has("samples_1m_p"+now.Format("200601")) || !has("samples_1h_p"+now.AddDate(0, 1, 0).Format("200601")) {168		t.Error("monthly rollup partitions missing")169	}170	if has("samples_raw_p" + now.AddDate(0, 0, 4).Format("20060102")) {171		t.Error("too many partitions created ahead")172	}173	// Idempotent.174	if err := h.srv.EnsurePartitions(ctx); err != nil {175		t.Fatal(err)176	}177	// Expired partitions are dropped; anything not named by us is never touched.178	h.exec(`create table samples_raw_p20200101 partition of samples_raw for values from ('2020-01-01') to ('2020-01-02')`)179	h.exec(`create table samples_1m_p202001 partition of samples_1m for values from ('2020-01-01') to ('2020-02-01')`)180	h.exec(`create table samples_raw_legacy partition of samples_raw for values from ('2019-01-01') to ('2019-01-02')`)181	h.exec(`insert into samples_raw_p20200101(node_id, metric_id, ts, value) values (gen_random_uuid(), 1, '2020-01-01 12:00+00', 1)`)182	dropped, err := h.srv.DropExpiredPartitions(ctx)183	if err != nil {184		t.Fatal(err)185	}186	if strings.Join(dropped, ",") != "samples_1m_p202001,samples_raw_p20200101" && strings.Join(dropped, ",") != "samples_raw_p20200101,samples_1m_p202001" {187		t.Errorf("dropped %v", dropped)188	}189	if has("samples_raw_p20200101") || !has("samples_raw_legacy") || !has("samples_raw_p"+now.AddDate(0, 0, -3).Format("20060102")) {190		t.Error("wrong partitions dropped")191	}192	if err := h.srv.Retention(ctx); err != nil {193		t.Fatal(err)194	}195	// A buffered report for a day whose partition is missing creates it on demand.196	e := h.enroll("n1")197	day := now.AddDate(0, 0, -2)198	h.exec(`drop table samples_raw_p` + day.Format("20060102"))199	r := report(1, day, sample("cpu.usage", 1))200	r.Buffered = true201	if _, err := h.send(e, r); err != nil {202		t.Fatalf("report into a missing partition: %v", err)203	}204	if scan[int](h, `select count(*) from samples_raw`) != 1 {205		t.Error("sample not stored")206	}207	// Catalog seeded from protocol.Catalog, exactly.208	if n := scan[int](h, `select count(*) from metric_catalog`); n != len(protocol.Catalog) {209		t.Errorf("catalog rows %d, want %d", n, len(protocol.Catalog))210	}211	for _, m := range protocol.Catalog {212		if got := scan[string](h, `select name || '|' || unit || '|' || kind || '|' || interval_s || '|' || labeled from metric_catalog where id=$1`, m.ID); !strings.HasPrefix(got, m.Name+"|"+m.Unit+"|"+m.Kind+"|") {213			t.Errorf("catalog %d: %s", m.ID, got)214		}215	}216	// Heartbeat, health, published CA.217	if err := h.srv.Heartbeat(ctx); err != nil {218		t.Fatal(err)219	}220	if scan[string](h, `select version from service_heartbeats where service='telemetry-ingest'`) != "test" {221		t.Error("heartbeat")222	}223	rec := httptest.NewRecorder()224	h.health.ServeHTTP(rec, httptest.NewRequest("GET", "/healthz", nil))225	if rec.Code != 200 || !strings.Contains(rec.Body.String(), `"connected_agents":1`) {226		t.Errorf("healthz: %d %s", rec.Code, rec.Body.String())227	}228	if err := h.ca.Publish(ctx, h.pool, "ingest.mcc.internal", []string{"https://example.test"}); err != nil {229		t.Fatal(err)230	}231	if fp := scan[string](h, `select value->>'fingerprint' from settings where key='pki.ca'`); fp != h.ca.Fingerprint() || !strings.HasPrefix(fp, "sha256:") {232		t.Errorf("published fingerprint %q", fp)233	}234	if err := h.srv.ServeHealth(ctx, "0.0.0.0:0"); err == nil {235		t.Error("health endpoint accepted a non-loopback address")236	}237}238
Files main
Menu

Theme