SPB Git

Repository Public

cluster_metrics_test.go 25.6 KB 513 lines go

1package api_test23import (4	"context"5	"fmt"6	"math"7	"net/url"8	"strings"9	"testing"10	"time"1112	"clustr.spboucher.ai/mcc/internal/ids"13	"clustr.spboucher.ai/mcc/internal/protocol"14)1516func rfc(t time.Time) string { return url.QueryEscape(t.UTC().Format(time.RFC3339)) }1718// points decodes the [ts_ms, avg, min, max] rows of one series.19func points(t *testing.T, series any) [][]float64 {20	t.Helper()21	raw, ok := series.(map[string]any)["points"].([]any)22	if !ok {23		t.Fatalf("series without points: %v", series)24	}25	out := make([][]float64, len(raw))26	for i, p := range raw {27		row := p.([]any)28		if len(row) != 4 {29			t.Fatalf("point %v is not [ts, avg, min, max]", row)30		}31		out[i] = []float64{row[0].(float64), row[1].(float64), row[2].(float64), row[3].(float64)}32	}33	return out34}3536func TestNodeMetricsResolutionBucketsGapsAndAvailability(t *testing.T) {37	h := newHarness(t)38	root := h.owner()39	h.seedPartitions()40	a := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second,41		Values: map[string]any{"cpu.usage": 12.5, "load.1": 1.0, "net.rx{en0}": 1.0, "net.rx{en1}": 2.0},42		Avail:  map[string]string{"thermal.temp": "permission_denied", "gpu.usage": "unsupported", "battery.charge": "something-new"}})43	stale := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Minute, Values: map[string]any{"cpu.usage": 1.0}})44	off := h.seedNode(nodeSeed{Alias: "charlie", Seen: true, Age: time.Hour, Values: map[string]any{"cpu.usage": 1.0}})45	cpu, load, mem, rx := seedMetricID(t, "cpu.usage"), seedMetricID(t, "load.1"), seedMetricID(t, "mem.used"), seedMetricID(t, "net.rx")4647	// cpu.usage: 30 minutes at 10 s with a 10-minute hole. Value cycles 0,1,2 with the 10-second slot number.48	const base = `date_bin('10 seconds', now(), 'epoch')`49	const val = `((extract(epoch from t)::bigint / 10) % 3)::float8`50	h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, '', t, `+val+`51		from generate_series(`+base+` - interval '30 minutes', `+base+`, interval '10 seconds') t52		where not (t >= `+base+` - interval '20 minutes' and t < `+base+` - interval '10 minutes')`, a, cpu)53	h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, '', t, 7 from generate_series(now() - interval '5 minutes', now(), interval '10 seconds') t`, off, cpu)54	// load.1: 12 hours at 10 s, same cycle.55	h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, '', t, `+val+`56		from generate_series(`+base+` - interval '12 hours', `+base+`, interval '10 seconds') t`, a, load)57	// net.rx: two interfaces.58	h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, l, t, case l when 'en0' then 100 else 200 end59		from generate_series(now() - interval '10 minutes', now(), interval '10 seconds') t, unnest(array['en0','en1']) l`, a, rx)60	// mem.used at 1 min, nine days ago: one minute in ten holds a single sample of 10, the others six samples of 0.61	h.dbExec(`insert into samples_1m(node_id, metric_id, label, ts, avg, min, max, n)62		select $1, $2, '', t, v, v - 1, v + 1, case when v = 10 then 1 else 6 end63		from (select t, case when (extract(epoch from t)::bigint / 60) % 10 = 0 then 10.0 else 0.0 end as v64			from generate_series(date_bin('10 minutes', now(), 'epoch') - interval '9 days', date_bin('10 minutes', now(), 'epoch') - interval '9 days' + interval '179 minutes', interval '1 minute') t) x`, a, mem)65	h.dbExec(`insert into samples_1h(node_id, metric_id, label, ts, avg, min, max, n) select $1, $2, '', t, 5, 1, 9, 36066		from generate_series(date_trunc('hour', now()) - interval '70 hours', date_trunc('hour', now()), interval '1 hour') t`, a, mem)6768	// Default range (1 h), auto resolution: raw points as stored; the hole is a hole.69	m := root.must("GET", "/nodes/"+a+"/metrics?metrics=cpu.usage", nil, 200).jsonObj(t)70	if m["resolution"] != "raw" || len(m["series"].([]any)) != 1 {71		t.Fatalf("default query: %v", m)72	}73	s0 := m["series"].([]any)[0].(map[string]any)74	if s0["metric"] != "cpu.usage" || s0["unit"] != "percent" || s0["label"] != "" || s0["node_id"] != nil {75		t.Fatalf("series header: %v", s0)76	}77	pts := points(t, s0)78	if len(pts) < 119 || len(pts) > 122 {79		t.Fatalf("want about 121 raw points (181 slots minus a 60-slot hole), got %d", len(pts))80	}81	holes := 082	for i, p := range pts {83		if p[1] != p[2] || p[1] != p[3] {84			t.Fatalf("raw point must have avg = min = max: %v", p)85		}86		if want := float64(int64(p[0]/10000) % 3); p[1] != want || math.Mod(p[0], 10000) != 0 {87			t.Fatalf("point %v does not match the stored sample (want value %v)", p, want)88		}89		if i > 0 {90			switch d := p[0] - pts[i-1][0]; d {91			case 10000:92			case 610000:93				holes++94			default:95				t.Fatalf("unexpected spacing %v ms between points %d and %d", d, i-1, i)96			}97		}98	}99	if holes != 1 {100		t.Fatalf("the 10-minute hole must appear exactly once as a gap, found %d", holes)101	}102103	// 12 h of raw data is 4320 points: reduced to 30-second buckets with true avg / min / max.104	now := time.Now()105	m = root.must("GET", "/nodes/"+a+"/metrics?metrics=load.1&resolution=raw&from="+rfc(now.Add(-12*time.Hour)), nil, 200).jsonObj(t)106	pts = points(t, m["series"].([]any)[0])107	if m["resolution"] != "raw" || len(pts) > 1500 || len(pts) < 1400 {108		t.Fatalf("12 h raw: resolution %v, %d points", m["resolution"], len(pts))109	}110	for i := 1; i < len(pts)-1; i++ { // the first and last buckets may be partial111		if p := pts[i]; p[1] != 1 || p[2] != 0 || p[3] != 2 || math.Mod(p[0], 30000) != 0 || p[0]-pts[i-1][0] != 30000 {112			t.Fatalf("bucket %d = %v, want [multiple of 30 s, 1, 0, 2] 30 s after the previous one", i, p)113		}114	}115116	// Rollup tier: buckets weight each row by its sample count.117	m = root.must("GET", "/nodes/"+a+"/metrics?metrics=mem.used&resolution=1m&from="+rfc(now.Add(-10*24*time.Hour)), nil, 200).jsonObj(t)118	pts = points(t, m["series"].([]any)[0])119	if m["resolution"] != "1m" || len(pts) != 18 {120		t.Fatalf("10 d at 1m: resolution %v, %d buckets (want 18 ten-minute buckets)", m["resolution"], len(pts))121	}122	for _, p := range pts {123		if math.Abs(p[1]-10.0/55.0) > 1e-9 || p[2] != -1 || p[3] != 11 || math.Mod(p[0], 600000) != 0 {124			t.Fatalf("weighted bucket = %v, want [_, 10/55, -1, 11]", p)125		}126	}127128	// auto: raw up to 2 h, 1 min up to 48 h, 1 h beyond.129	for from, want := range map[time.Duration]string{90 * time.Minute: "raw", 2 * time.Hour: "raw", 3 * time.Hour: "1m", 47 * time.Hour: "1m", 72 * time.Hour: "1h", 300 * 24 * time.Hour: "1h"} {130		at := now.Add(-from)131		r := root.must("GET", "/nodes/"+a+"/metrics?metrics=mem.used&from="+rfc(at)+"&to="+rfc(at.Add(from)), nil, 200).jsonObj(t)132		if r["resolution"] != want {133			t.Fatalf("range %v: resolution %v, want %s", from, r["resolution"], want)134		}135	}136	if r := root.must("GET", "/nodes/"+a+"/metrics?metrics=mem.used&from="+rfc(now.Add(-72*time.Hour)), nil, 200).jsonObj(t); len(points(t, r["series"].([]any)[0])) < 70 {137		t.Fatalf("1 h tier: %v", r)138	}139140	// Labeled metric: one series per label; a label filter keeps one and leaves unlabeled metrics alone.141	m = root.must("GET", "/nodes/"+a+"/metrics?metrics=net.rx,cpu.usage", nil, 200).jsonObj(t)142	var labels []string143	for _, s := range m["series"].([]any) {144		labels = append(labels, fmt.Sprint(s.(map[string]any)["metric"], "/", s.(map[string]any)["label"]))145	}146	if strings.Join(labels, " ") != "net.rx/en0 net.rx/en1 cpu.usage/" {147		t.Fatalf("series (in request order, labels sorted): %v", labels)148	}149	m = root.must("GET", "/nodes/"+a+"/metrics?metrics=net.rx,cpu.usage&label=en1", nil, 200).jsonObj(t)150	if ss := m["series"].([]any); len(ss) != 2 || ss[0].(map[string]any)["label"] != "en1" || points(t, ss[0])[0][1] != 200 {151		t.Fatalf("label filter: %v", m["series"])152	}153154	// Availability: what the agent declared, what it reports, and "unavailable" when it says nothing.155	m = root.must("GET", "/nodes/"+a+"/metrics?metrics=cpu.usage,thermal.temp,gpu.usage,power.draw,net.rx,battery.charge", nil, 200).jsonObj(t)156	want := map[string]string{"cpu.usage": "available", "thermal.temp": "permission_denied", "gpu.usage": "unsupported", "power.draw": "unavailable",157		"net.rx": "available", "battery.charge": "unavailable"}158	for name, w := range want {159		if dig(m, "availability", name) != w {160			t.Fatalf("availability[%s] = %v, want %s", name, dig(m, "availability", name), w)161		}162	}163	if len(m["availability"].(map[string]any)) != len(want) {164		t.Fatalf("availability must cover exactly the requested metrics: %v", m["availability"])165	}166	for _, s := range m["series"].([]any) { // no series is made up for a metric without data167		if name := s.(map[string]any)["metric"]; name != "cpu.usage" && name != "net.rx" {168			t.Fatalf("series for a metric without samples: %v", s)169		}170	}171	if r := root.must("GET", "/nodes/"+a+"/metrics?metrics=net.rx&label=en9", nil, 200).jsonObj(t); dig(r, "availability", "net.rx") != "unavailable" || len(r["series"].([]any)) != 0 {172		t.Fatalf("unknown label: %v", r)173	}174	if r := root.must("GET", "/nodes/"+stale+"/metrics?metrics=cpu.usage,load.1", nil, 200).jsonObj(t); dig(r, "availability", "cpu.usage") != "stale" || dig(r, "availability", "load.1") != "unavailable" {175		t.Fatalf("stale node: %v", r)176	}177	r := root.must("GET", "/nodes/"+off+"/metrics?metrics=cpu.usage,load.1", nil, 200).jsonObj(t)178	if dig(r, "availability", "cpu.usage") != "offline" || dig(r, "availability", "load.1") != "offline" || len(r["series"].([]any)) != 1 {179		t.Fatalf("offline node: history stays readable, availability says offline: %v", r)180	}181182	// Latest values on the node detail follow the same rules.183	d := root.must("GET", "/nodes/"+a, nil, 200).jsonObj(t)184	if dig(d, "latest", "cpu.usage", "value") != 12.5 || dig(d, "latest", "cpu.usage", "availability") != "available" ||185		dig(d, "latest", "thermal.temp", "availability") != "permission_denied" || dig(d, "latest", "thermal.temp", "value") != nil ||186		dig(d, "latest", "thermal.temp", "reason") == nil || dig(d, "latest", "power.draw", "availability") != "unavailable" || dig(d, "latest", "net.rx") != nil {187		t.Fatalf("latest: %v", d["latest"])188	}189	if d := root.must("GET", "/nodes/"+off, nil, 200).jsonObj(t); dig(d, "latest", "cpu.usage", "availability") != "offline" || dig(d, "latest", "cpu.usage", "value") != nil {190		t.Fatalf("latest of an offline node must carry no value: %v", dig(d, "latest", "cpu.usage"))191	}192193	// Invalid queries.194	many := make([]string, 0, 13)195	for _, c := range protocol.Catalog[:13] {196		many = append(many, c.Name)197	}198	for name, q := range map[string]string{199		"unknown metric": "metrics=cpu.usage,nope.metric", "no metric": "metrics=", "13 metrics": "metrics=" + strings.Join(many, ","),200		"from after to": "metrics=cpu.usage&from=" + rfc(now) + "&to=" + rfc(now.Add(-time.Hour)), "from in the future": "metrics=cpu.usage&from=" + rfc(now.Add(time.Hour)),201		"raw too wide": "metrics=cpu.usage&resolution=raw&from=" + rfc(now.Add(-72*time.Hour)), "1m too wide": "metrics=cpu.usage&resolution=1m&from=" + rfc(now.Add(-60*24*time.Hour)),202		"bad resolution": "metrics=cpu.usage&resolution=2m", "too long": "metrics=cpu.usage&from=" + rfc(now.Add(-500*24*time.Hour)),203		"label": "metrics=net.rx&label=" + strings.Repeat("x", 65), "bad date": "metrics=cpu.usage&from=yesterday",204	} {205		r := root.do("GET", "/nodes/"+a+"/metrics?"+q, nil)206		if r.Status != 400 || r.code() != "validation_failed" {207			t.Fatalf("%s: %d %s", name, r.Status, r.Raw)208		}209		if name == "unknown metric" && !strings.Contains(string(r.Raw), `"unknown":["nope.metric"]`) {210			t.Fatalf("the unknown metric must be named: %s", r.Raw)211		}212	}213	if r := root.do("GET", "/nodes/"+ids.New().String()+"/metrics?metrics=cpu.usage", nil); r.Status != 404 {214		t.Fatalf("unknown node: %d", r.Status)215	}216217	// Comparison across nodes.218	c := root.must("GET", "/metrics/compare?metric=cpu.usage&node_ids="+off+","+a, nil, 200).jsonObj(t)219	cs := c["series"].([]any)220	if c["metric"] != "cpu.usage" || c["unit"] != "percent" || c["resolution"] != "raw" || len(cs) != 2 ||221		cs[0].(map[string]any)["node_id"] != off || cs[0].(map[string]any)["node_alias"] != "charlie" || cs[1].(map[string]any)["node_id"] != a ||222		dig(c, "availability", a) != "available" || dig(c, "availability", off) != "offline" {223		t.Fatalf("compare: %v", c)224	}225	if r := root.do("GET", "/metrics/compare?metric=cpu.usage&node_ids="+a+","+ids.New().String(), nil); r.Status != 404 {226		t.Fatalf("compare with an unknown node: %d %s", r.Status, r.Raw)227	}228	if r := root.do("GET", "/metrics/compare?metric=nope&node_ids="+a, nil); r.Status != 400 {229		t.Fatalf("compare with an unknown metric: %d", r.Status)230	}231232	// Catalog: served from the table, seeded by the API itself.233	cat := root.must("GET", "/metrics/catalog", nil, 200).jsonItems(t)234	if len(cat) != len(protocol.Catalog) || h.count(`select count(*) from metric_catalog`) != len(protocol.Catalog) {235		t.Fatalf("catalog: %d entries, want %d", len(cat), len(protocol.Catalog))236	}237	if e := rowBy(cat, "name", "net.rx"); e["unit"] != "bytes_per_s" || e["labeled"] != true || e["kind"] != "counter_rate" || e["interval_s"] != 10.0 {238		t.Fatalf("catalog entry: %v", e)239	}240}241242func TestClusterTrends(t *testing.T) {243	h := newHarness(t)244	root := h.owner()245	h.seedPartitions()246	n1 := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second, Tags: []string{"gpu", "studio"}})247	n2 := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Second, Site: "lab", Role: "gateway", Tags: []string{"gpu"}})248	n3 := h.seedNode(nodeSeed{Alias: "charlie", Seen: true, Age: time.Second})249	h.dbExec(`update nodes set retired_at = now() where id = $1`, n3)250	h.seedNode(nodeSeed{Alias: "delta", Manual: true})251	cpu, mem, rx := seedMetricID(t, "cpu.usage"), seedMetricID(t, "mem.used"), seedMetricID(t, "net.rx")252	raw := func(node string, metric int16, label string, v float64, minutes int) {253		h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, $3, t, $4254			from generate_series(date_bin('10 seconds', now(), 'epoch') - make_interval(mins => $5), date_bin('10 seconds', now(), 'epoch'), interval '10 seconds') t`,255			node, metric, label, v, minutes)256	}257	raw(n1, cpu, "", 10, 30)258	raw(n2, cpu, "", 30, 10) // reports for the last 10 minutes only259	raw(n3, cpu, "", 99, 30) // retired: never part of the cluster figure260	raw(n1, mem, "", 100, 5)261	raw(n2, mem, "", 200, 5)262	raw(n1, rx, "en0", 5, 5)263	raw(n1, rx, "en1", 7, 5)264	raw(n2, rx, "en0", 1, 5)265266	tr := root.must("GET", "/cluster/trends?metric=cpu.usage&range=1h", nil, 200).jsonObj(t)267	if tr["aggregation"] != "mean" || tr["resolution"] != "raw" || tr["unit"] != "percent" || tr["group_by"] != "cluster" || tr["range"] != "1h" {268		t.Fatalf("trend header: %v", tr)269	}270	ss := tr["series"].([]any)271	if len(ss) != 1 || ss[0].(map[string]any)["group"] != "cluster" || ss[0].(map[string]any)["coverage"] != 2.0 {272		t.Fatalf("cluster series: %v", ss)273	}274	pts := points(t, ss[0])275	if len(pts) < 175 || len(pts) > 185 {276		t.Fatalf("want ~181 ten-second points, got %d", len(pts))277	}278	both := 0279	for i, p := range pts {280		switch {281		case p[1] == 10 && p[2] == 10 && p[3] == 10: // only alpha reported in this bucket: no value is carried forward for bravo282		case p[1] == 20 && p[2] == 10 && p[3] == 30:283			both++284		default:285			t.Fatalf("point %d = %v", i, p)286		}287		if i > 0 && p[0]-pts[i-1][0] != 10000 {288			t.Fatalf("spacing at %d", i)289		}290	}291	if both < 55 || both > 65 {292		t.Fatalf("both nodes contribute for ~10 minutes, got %d buckets", both)293	}294295	bySite := root.must("GET", "/cluster/trends?metric=cpu.usage&range=1h&group_by=site", nil, 200).jsonObj(t)["series"].([]any)296	if len(bySite) != 2 || bySite[0].(map[string]any)["group"] != "default" || points(t, bySite[0])[0][1] != 10 ||297		bySite[1].(map[string]any)["group"] != "lab" || points(t, bySite[1])[0][1] != 30 {298		t.Fatalf("by site: %v", bySite)299	}300	byTag := root.must("GET", "/cluster/trends?metric=cpu.usage&range=1h&group_by=tag", nil, 200).jsonObj(t)["series"].([]any)301	if len(byTag) != 2 || byTag[0].(map[string]any)["group"] != "gpu" || byTag[0].(map[string]any)["coverage"] != 2.0 || byTag[1].(map[string]any)["group"] != "studio" {302		t.Fatalf("by tag: %v", byTag)303	}304	if byRole := root.must("GET", "/cluster/trends?metric=cpu.usage&range=1h&group_by=role", nil, 200).jsonObj(t)["series"].([]any); len(byRole) != 2 {305		t.Fatalf("by role: %v", byRole)306	}307308	// Additive quantities are summed; labels of one node are combined first.309	sum := root.must("GET", "/cluster/trends?metric=mem.used&range=1h", nil, 200).jsonObj(t)310	if p := points(t, sum["series"].([]any)[0]); sum["aggregation"] != "sum" || p[len(p)/2][1] != 300 || p[len(p)/2][2] != 300 {311		t.Fatalf("mem.used sum: %v %v", sum["aggregation"], p[len(p)/2])312	}313	net := root.must("GET", "/cluster/trends?metric=net.rx&range=1h", nil, 200).jsonObj(t)314	if p := points(t, net["series"].([]any)[0]); p[len(p)/2][1] != 13 {315		t.Fatalf("net.rx: 5 + 7 on alpha, 1 on bravo = 13, got %v", p[len(p)/2])316	}317318	// Longer ranges read the rollup tiers.319	h.dbExec(`insert into samples_1m(node_id, metric_id, label, ts, avg, min, max, n) values ($1, $2, '', date_trunc('minute', now()) - interval '3 hours', 40, 35, 45, 6)`, n1, cpu)320	h.dbExec(`insert into samples_1h(node_id, metric_id, label, ts, avg, min, max, n) values ($1, $2, '', date_trunc('hour', now()) - interval '3 days', 60, 50, 70, 360)`, n1, cpu)321	for rng, want := range map[string][]any{"24h": {"1m", 40.0, 35.0, 45.0}, "7d": {"1h", 60.0, 50.0, 70.0}, "30d": {"1h", 60.0, 50.0, 70.0}} {322		tr := root.must("GET", "/cluster/trends?metric=cpu.usage&range="+rng, nil, 200).jsonObj(t)323		p := points(t, tr["series"].([]any)[0])324		if tr["resolution"] != want[0] || len(p) != 1 || p[0][1] != want[1] || p[0][2] != want[2] || p[0][3] != want[3] {325			t.Fatalf("range %s: %v %v", rng, tr["resolution"], p)326		}327	}328	// Nothing reported: an empty series with coverage 0, not zeros.329	empty := root.must("GET", "/cluster/trends?metric=gpu.usage", nil, 200).jsonObj(t)["series"].([]any)330	if len(empty) != 1 || empty[0].(map[string]any)["coverage"] != 0.0 || len(points(t, empty[0])) != 0 {331		t.Fatalf("empty trend: %v", empty)332	}333	for _, bad := range []string{"metric=nope", "metric=cpu.usage&range=2h", "metric=cpu.usage&group_by=color", "range=1h"} {334		if r := root.do("GET", "/cluster/trends?"+bad, nil); r.Status != 400 {335			t.Fatalf("%s: %d %s", bad, r.Status, r.Raw)336		}337	}338}339340func TestNodeHealthHistory(t *testing.T) {341	h := newHarness(t)342	root := h.owner()343	h.seedPartitions()344	n := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second})345	// Reports from 30 minutes before the current 5-minute bucket until now, silent from -20 to -12 minutes.346	const b = `date_bin('5 minutes', now(), 'epoch')`347	h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, '', t, 1348		from generate_series(`+b+` - interval '30 minutes', now(), interval '10 seconds') t349		where not (t >= `+b+` - interval '20 minutes' and t < `+b+` - interval '12 minutes')`, n, seedMetricID(t, "cpu.usage"))350	h.seedAlert(n, "firing", "warning", "hot") // fired 9 minutes ago351352	states := func(hh map[string]any) string {353		var s []string354		for _, it := range hh["items"].([]any) {355			s = append(s, it.(map[string]any)["state"].(string))356		}357		return strings.Join(s, " ")358	}359	hh := root.must("GET", "/nodes/"+n+"/health-history?range=1h", nil, 200).jsonObj(t)360	if hh["bucket_seconds"] != 300.0 || hh["range"] != "1h" || hh["node_id"] != n {361		t.Fatalf("header: %v", hh)362	}363	// 12 buckets, oldest first: before the first report, then up, up, down (silent), degraded (partly silent), up, up, current.364	if got := states(hh); got != "unknown unknown unknown unknown unknown up up down degraded up up up" {365		t.Fatalf("states: %s", got)366	}367	if r, _ := hh["uptime_ratio"].(float64); math.Abs(r-6.0/7.0) > 1e-6 {368		t.Fatalf("uptime ratio %v, want 6/7", hh["uptime_ratio"])369	}370	items := hh["items"].([]any)371	if items[11].(map[string]any)["alerts_firing"] != 1.0 || items[10].(map[string]any)["alerts_firing"] != 1.0 || items[5].(map[string]any)["alerts_firing"] != 0.0 {372		t.Fatalf("alerts per bucket: %v", items)373	}374	for i := 1; i < len(items); i++ {375		if items[i].(map[string]any)["start"] != items[i-1].(map[string]any)["end"] {376			t.Fatalf("buckets must be contiguous: %v then %v", items[i-1], items[i])377		}378	}379380	// An applied maintenance window marks its buckets as maintenance.381	h.dbExec(`insert into maintenance_windows(id, title, starts_at, ends_at, scope, status) values ($1, 'swap disk', `+b+` - interval '30 minutes',382		`+b+` - interval '25 minutes', jsonb_build_object('node_ids', jsonb_build_array($2::text)), 'completed')`, ids.New().String(), n)383	h.dbExec(`insert into maintenance_windows(id, title, starts_at, ends_at, scope, status) values ($1, 'never applied', `+b+` - interval '10 minutes',384		`+b+`, '{"all": true}', 'cancelled')`, ids.New().String())385	hh = root.must("GET", "/nodes/"+n+"/health-history?range=1h", nil, 200).jsonObj(t)386	if got := states(hh); got != "unknown unknown unknown unknown unknown maintenance up down degraded up up up" {387		t.Fatalf("states with maintenance: %s", got)388	}389	if r, _ := hh["uptime_ratio"].(float64); math.Abs(r-5.0/7.0) > 1e-6 {390		t.Fatalf("uptime ratio %v, want 5/7", hh["uptime_ratio"])391	}392393	for rng, want := range map[string][]float64{"24h": {48, 1800}, "7d": {56, 10800}, "30d": {60, 43200}} {394		hh := root.must("GET", "/nodes/"+n+"/health-history?range="+rng, nil, 200).jsonObj(t)395		items := hh["items"].([]any)396		if float64(len(items)) != want[0] || hh["bucket_seconds"] != want[1] || items[0].(map[string]any)["state"] != "unknown" ||397			items[len(items)-1].(map[string]any)["state"] == "unknown" {398			t.Fatalf("range %s: %d buckets of %v s, first %v, last %v", rng, len(items), hh["bucket_seconds"], items[0], items[len(items)-1])399		}400	}401	if r := root.do("GET", "/nodes/"+n+"/health-history?range=1y", nil); r.Status != 400 {402		t.Fatalf("bad range: %d", r.Status)403	}404}405406// The time-series queries must only touch the partitions that overlap the407// requested range, and reach their rows through the primary key. This is408// checked on the statistics PostgreSQL keeps for each partition, after409// running the real endpoints against daily partitions holding 30 nodes of410// 10-second samples.411func TestSeriesQueriesArePrunedAndIndexDriven(t *testing.T) {412	if testing.Short() {413		t.Skip("loads several hundred thousand samples")414	}415	h := newHarness(t)416	root := h.owner()417	ctx := context.Background()418	day := time.Now().UTC().Truncate(24 * time.Hour)419	for d := -4; d <= 1; d++ {420		from := day.AddDate(0, 0, d)421		h.dbExec(fmt.Sprintf(`create table samples_raw_d%s partition of samples_raw for values from ('%s') to ('%s')`,422			from.Format("20060102"), from.Format(time.RFC3339), from.AddDate(0, 0, 1).Format(time.RFC3339)))423	}424	h.dbExec(`create table samples_1m_all partition of samples_1m for values from (minvalue) to (maxvalue)`)425	h.dbExec(`create table samples_1h_all partition of samples_1h for values from (minvalue) to (maxvalue)`)426	var nodes []string427	for i := 0; i < 30; i++ {428		nodes = append(nodes, h.seedNode(nodeSeed{Alias: fmt.Sprintf("node%02d", i), Seen: true, Age: time.Second, Values: map[string]any{"cpu.usage": 1.0}}))429	}430	// 30 nodes x 3 metrics x 10 s over the last 50 hours: about 1.6 M rows in three daily partitions.431	h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value)432		select n.id, m, '', t, random() * 100 from nodes n, unnest($1::smallint[]) m,433			generate_series(now() - interval '50 hours', now(), interval '10 seconds') t`,434		[]int16{seedMetricID(t, "cpu.usage"), seedMetricID(t, "load.1"), seedMetricID(t, "mem.used")})435	h.dbExec(`analyze samples_raw`)436437	type stat struct{ seq, idx int64 }438	read := func() map[string]stat {439		out := map[string]stat{}440		rows, err := h.Pool.Query(ctx, `select relname, coalesce(seq_scan, 0), coalesce(idx_scan, 0) from pg_stat_user_tables where relname like 'samples_raw_d%'`)441		if err != nil {442			t.Fatal(err)443		}444		defer rows.Close()445		for rows.Next() {446			var name string447			var s stat448			if err := rows.Scan(&name, &s.seq, &s.idx); err != nil {449				t.Fatal(err)450			}451			out[name] = s452		}453		return out454	}455	// settle waits until the per-table counters (flushed asynchronously by the backends) stop moving.456	settle := func() map[string]stat {457		prev := read()458		for i := 0; i < 40; i++ {459			time.Sleep(300 * time.Millisecond)460			cur := read()461			same := true462			for k, v := range cur {463				same = same && prev[k] == v464			}465			if same && i >= 5 {466				return cur467			}468			prev = cur469		}470		return prev471	}472	before := settle()473	now := time.Now().UTC()474	root.must("GET", "/nodes/"+nodes[3]+"/metrics?metrics=cpu.usage,load.1", nil, 200)                                           // last hour, raw points475	root.must("GET", "/nodes/"+nodes[3]+"/metrics?metrics=cpu.usage&resolution=raw&from="+rfc(now.Add(-20*time.Hour)), nil, 200) // bucketed476	root.must("GET", "/metrics/compare?metric=cpu.usage&node_ids="+strings.Join(nodes[:6], ","), nil, 200)477	root.must("GET", "/cluster/trends?metric=cpu.usage&range=1h", nil, 200)478	after := settle()479480	oldest := now.Add(-20*time.Hour - time.Minute) // earliest instant any of the queries above may read481	used := 0482	for name, a := range after {483		b := before[name]484		pday, err := time.Parse("20060102", strings.TrimPrefix(name, "samples_raw_d"))485		if err != nil {486			t.Fatal(err)487		}488		inRange := pday.Add(24*time.Hour).After(oldest) && !pday.After(now)489		if a.seq != b.seq {490			t.Errorf("%s: %d sequential scans by the series queries", name, a.seq-b.seq)491		}492		switch {493		case inRange && a.idx > b.idx:494			used++495		case !inRange && a.idx != b.idx:496			t.Errorf("%s is outside the requested ranges but was scanned %d times: partitions are not pruned", name, a.idx-b.idx)497		}498	}499	t.Logf("per-partition {seq_scan idx_scan} before %v, after %v", before, after)500	if used == 0 {501		t.Fatalf("no partition of the requested range shows an index scan: before %v after %v", before, after)502	}503504	// Health history reads one node through the primary key as well (its "first sample" probe touches every partition, by index).505	root.must("GET", "/nodes/"+nodes[3]+"/health-history?range=24h", nil, 200)506	final := settle()507	for name, f := range final {508		if f.seq != after[name].seq {509			t.Errorf("%s: sequential scan by the health history", name)510		}511	}512}513
Files main
Menu

Theme