Repository Public
1package api_test23import (4 "context"5 "fmt"6 "math"7 "net/url"8 "strings"9 "testing"10 "time"1112 "clustr.spboucher.ai/mcc/internal/ids"13 "clustr.spboucher.ai/mcc/internal/protocol"14)1516func rfc(t time.Time) string { return url.QueryEscape(t.UTC().Format(time.RFC3339)) }1718// points decodes the [ts_ms, avg, min, max] rows of one series.19func points(t *testing.T, series any) [][]float64 {20 t.Helper()21 raw, ok := series.(map[string]any)["points"].([]any)22 if !ok {23 t.Fatalf("series without points: %v", series)24 }25 out := make([][]float64, len(raw))26 for i, p := range raw {27 row := p.([]any)28 if len(row) != 4 {29 t.Fatalf("point %v is not [ts, avg, min, max]", row)30 }31 out[i] = []float64{row[0].(float64), row[1].(float64), row[2].(float64), row[3].(float64)}32 }33 return out34}3536func TestNodeMetricsResolutionBucketsGapsAndAvailability(t *testing.T) {37 h := newHarness(t)38 root := h.owner()39 h.seedPartitions()40 a := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second,41 Values: map[string]any{"cpu.usage": 12.5, "load.1": 1.0, "net.rx{en0}": 1.0, "net.rx{en1}": 2.0},42 Avail: map[string]string{"thermal.temp": "permission_denied", "gpu.usage": "unsupported", "battery.charge": "something-new"}})43 stale := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Minute, Values: map[string]any{"cpu.usage": 1.0}})44 off := h.seedNode(nodeSeed{Alias: "charlie", Seen: true, Age: time.Hour, Values: map[string]any{"cpu.usage": 1.0}})45 cpu, load, mem, rx := seedMetricID(t, "cpu.usage"), seedMetricID(t, "load.1"), seedMetricID(t, "mem.used"), seedMetricID(t, "net.rx")4647 // cpu.usage: 30 minutes at 10 s with a 10-minute hole. Value cycles 0,1,2 with the 10-second slot number.48 const base = `date_bin('10 seconds', now(), 'epoch')`49 const val = `((extract(epoch from t)::bigint / 10) % 3)::float8`50 h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, '', t, `+val+`51 from generate_series(`+base+` - interval '30 minutes', `+base+`, interval '10 seconds') t52 where not (t >= `+base+` - interval '20 minutes' and t < `+base+` - interval '10 minutes')`, a, cpu)53 h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, '', t, 7 from generate_series(now() - interval '5 minutes', now(), interval '10 seconds') t`, off, cpu)54 // load.1: 12 hours at 10 s, same cycle.55 h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, '', t, `+val+`56 from generate_series(`+base+` - interval '12 hours', `+base+`, interval '10 seconds') t`, a, load)57 // net.rx: two interfaces.58 h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, l, t, case l when 'en0' then 100 else 200 end59 from generate_series(now() - interval '10 minutes', now(), interval '10 seconds') t, unnest(array['en0','en1']) l`, a, rx)60 // mem.used at 1 min, nine days ago: one minute in ten holds a single sample of 10, the others six samples of 0.61 h.dbExec(`insert into samples_1m(node_id, metric_id, label, ts, avg, min, max, n)62 select $1, $2, '', t, v, v - 1, v + 1, case when v = 10 then 1 else 6 end63 from (select t, case when (extract(epoch from t)::bigint / 60) % 10 = 0 then 10.0 else 0.0 end as v64 from generate_series(date_bin('10 minutes', now(), 'epoch') - interval '9 days', date_bin('10 minutes', now(), 'epoch') - interval '9 days' + interval '179 minutes', interval '1 minute') t) x`, a, mem)65 h.dbExec(`insert into samples_1h(node_id, metric_id, label, ts, avg, min, max, n) select $1, $2, '', t, 5, 1, 9, 36066 from generate_series(date_trunc('hour', now()) - interval '70 hours', date_trunc('hour', now()), interval '1 hour') t`, a, mem)6768 // Default range (1 h), auto resolution: raw points as stored; the hole is a hole.69 m := root.must("GET", "/nodes/"+a+"/metrics?metrics=cpu.usage", nil, 200).jsonObj(t)70 if m["resolution"] != "raw" || len(m["series"].([]any)) != 1 {71 t.Fatalf("default query: %v", m)72 }73 s0 := m["series"].([]any)[0].(map[string]any)74 if s0["metric"] != "cpu.usage" || s0["unit"] != "percent" || s0["label"] != "" || s0["node_id"] != nil {75 t.Fatalf("series header: %v", s0)76 }77 pts := points(t, s0)78 if len(pts) < 119 || len(pts) > 122 {79 t.Fatalf("want about 121 raw points (181 slots minus a 60-slot hole), got %d", len(pts))80 }81 holes := 082 for i, p := range pts {83 if p[1] != p[2] || p[1] != p[3] {84 t.Fatalf("raw point must have avg = min = max: %v", p)85 }86 if want := float64(int64(p[0]/10000) % 3); p[1] != want || math.Mod(p[0], 10000) != 0 {87 t.Fatalf("point %v does not match the stored sample (want value %v)", p, want)88 }89 if i > 0 {90 switch d := p[0] - pts[i-1][0]; d {91 case 10000:92 case 610000:93 holes++94 default:95 t.Fatalf("unexpected spacing %v ms between points %d and %d", d, i-1, i)96 }97 }98 }99 if holes != 1 {100 t.Fatalf("the 10-minute hole must appear exactly once as a gap, found %d", holes)101 }102103 // 12 h of raw data is 4320 points: reduced to 30-second buckets with true avg / min / max.104 now := time.Now()105 m = root.must("GET", "/nodes/"+a+"/metrics?metrics=load.1&resolution=raw&from="+rfc(now.Add(-12*time.Hour)), nil, 200).jsonObj(t)106 pts = points(t, m["series"].([]any)[0])107 if m["resolution"] != "raw" || len(pts) > 1500 || len(pts) < 1400 {108 t.Fatalf("12 h raw: resolution %v, %d points", m["resolution"], len(pts))109 }110 for i := 1; i < len(pts)-1; i++ { // the first and last buckets may be partial111 if p := pts[i]; p[1] != 1 || p[2] != 0 || p[3] != 2 || math.Mod(p[0], 30000) != 0 || p[0]-pts[i-1][0] != 30000 {112 t.Fatalf("bucket %d = %v, want [multiple of 30 s, 1, 0, 2] 30 s after the previous one", i, p)113 }114 }115116 // Rollup tier: buckets weight each row by its sample count.117 m = root.must("GET", "/nodes/"+a+"/metrics?metrics=mem.used&resolution=1m&from="+rfc(now.Add(-10*24*time.Hour)), nil, 200).jsonObj(t)118 pts = points(t, m["series"].([]any)[0])119 if m["resolution"] != "1m" || len(pts) != 18 {120 t.Fatalf("10 d at 1m: resolution %v, %d buckets (want 18 ten-minute buckets)", m["resolution"], len(pts))121 }122 for _, p := range pts {123 if math.Abs(p[1]-10.0/55.0) > 1e-9 || p[2] != -1 || p[3] != 11 || math.Mod(p[0], 600000) != 0 {124 t.Fatalf("weighted bucket = %v, want [_, 10/55, -1, 11]", p)125 }126 }127128 // auto: raw up to 2 h, 1 min up to 48 h, 1 h beyond.129 for from, want := range map[time.Duration]string{90 * time.Minute: "raw", 2 * time.Hour: "raw", 3 * time.Hour: "1m", 47 * time.Hour: "1m", 72 * time.Hour: "1h", 300 * 24 * time.Hour: "1h"} {130 at := now.Add(-from)131 r := root.must("GET", "/nodes/"+a+"/metrics?metrics=mem.used&from="+rfc(at)+"&to="+rfc(at.Add(from)), nil, 200).jsonObj(t)132 if r["resolution"] != want {133 t.Fatalf("range %v: resolution %v, want %s", from, r["resolution"], want)134 }135 }136 if r := root.must("GET", "/nodes/"+a+"/metrics?metrics=mem.used&from="+rfc(now.Add(-72*time.Hour)), nil, 200).jsonObj(t); len(points(t, r["series"].([]any)[0])) < 70 {137 t.Fatalf("1 h tier: %v", r)138 }139140 // Labeled metric: one series per label; a label filter keeps one and leaves unlabeled metrics alone.141 m = root.must("GET", "/nodes/"+a+"/metrics?metrics=net.rx,cpu.usage", nil, 200).jsonObj(t)142 var labels []string143 for _, s := range m["series"].([]any) {144 labels = append(labels, fmt.Sprint(s.(map[string]any)["metric"], "/", s.(map[string]any)["label"]))145 }146 if strings.Join(labels, " ") != "net.rx/en0 net.rx/en1 cpu.usage/" {147 t.Fatalf("series (in request order, labels sorted): %v", labels)148 }149 m = root.must("GET", "/nodes/"+a+"/metrics?metrics=net.rx,cpu.usage&label=en1", nil, 200).jsonObj(t)150 if ss := m["series"].([]any); len(ss) != 2 || ss[0].(map[string]any)["label"] != "en1" || points(t, ss[0])[0][1] != 200 {151 t.Fatalf("label filter: %v", m["series"])152 }153154 // Availability: what the agent declared, what it reports, and "unavailable" when it says nothing.155 m = root.must("GET", "/nodes/"+a+"/metrics?metrics=cpu.usage,thermal.temp,gpu.usage,power.draw,net.rx,battery.charge", nil, 200).jsonObj(t)156 want := map[string]string{"cpu.usage": "available", "thermal.temp": "permission_denied", "gpu.usage": "unsupported", "power.draw": "unavailable",157 "net.rx": "available", "battery.charge": "unavailable"}158 for name, w := range want {159 if dig(m, "availability", name) != w {160 t.Fatalf("availability[%s] = %v, want %s", name, dig(m, "availability", name), w)161 }162 }163 if len(m["availability"].(map[string]any)) != len(want) {164 t.Fatalf("availability must cover exactly the requested metrics: %v", m["availability"])165 }166 for _, s := range m["series"].([]any) { // no series is made up for a metric without data167 if name := s.(map[string]any)["metric"]; name != "cpu.usage" && name != "net.rx" {168 t.Fatalf("series for a metric without samples: %v", s)169 }170 }171 if r := root.must("GET", "/nodes/"+a+"/metrics?metrics=net.rx&label=en9", nil, 200).jsonObj(t); dig(r, "availability", "net.rx") != "unavailable" || len(r["series"].([]any)) != 0 {172 t.Fatalf("unknown label: %v", r)173 }174 if r := root.must("GET", "/nodes/"+stale+"/metrics?metrics=cpu.usage,load.1", nil, 200).jsonObj(t); dig(r, "availability", "cpu.usage") != "stale" || dig(r, "availability", "load.1") != "unavailable" {175 t.Fatalf("stale node: %v", r)176 }177 r := root.must("GET", "/nodes/"+off+"/metrics?metrics=cpu.usage,load.1", nil, 200).jsonObj(t)178 if dig(r, "availability", "cpu.usage") != "offline" || dig(r, "availability", "load.1") != "offline" || len(r["series"].([]any)) != 1 {179 t.Fatalf("offline node: history stays readable, availability says offline: %v", r)180 }181182 // Latest values on the node detail follow the same rules.183 d := root.must("GET", "/nodes/"+a, nil, 200).jsonObj(t)184 if dig(d, "latest", "cpu.usage", "value") != 12.5 || dig(d, "latest", "cpu.usage", "availability") != "available" ||185 dig(d, "latest", "thermal.temp", "availability") != "permission_denied" || dig(d, "latest", "thermal.temp", "value") != nil ||186 dig(d, "latest", "thermal.temp", "reason") == nil || dig(d, "latest", "power.draw", "availability") != "unavailable" || dig(d, "latest", "net.rx") != nil {187 t.Fatalf("latest: %v", d["latest"])188 }189 if d := root.must("GET", "/nodes/"+off, nil, 200).jsonObj(t); dig(d, "latest", "cpu.usage", "availability") != "offline" || dig(d, "latest", "cpu.usage", "value") != nil {190 t.Fatalf("latest of an offline node must carry no value: %v", dig(d, "latest", "cpu.usage"))191 }192193 // Invalid queries.194 many := make([]string, 0, 13)195 for _, c := range protocol.Catalog[:13] {196 many = append(many, c.Name)197 }198 for name, q := range map[string]string{199 "unknown metric": "metrics=cpu.usage,nope.metric", "no metric": "metrics=", "13 metrics": "metrics=" + strings.Join(many, ","),200 "from after to": "metrics=cpu.usage&from=" + rfc(now) + "&to=" + rfc(now.Add(-time.Hour)), "from in the future": "metrics=cpu.usage&from=" + rfc(now.Add(time.Hour)),201 "raw too wide": "metrics=cpu.usage&resolution=raw&from=" + rfc(now.Add(-72*time.Hour)), "1m too wide": "metrics=cpu.usage&resolution=1m&from=" + rfc(now.Add(-60*24*time.Hour)),202 "bad resolution": "metrics=cpu.usage&resolution=2m", "too long": "metrics=cpu.usage&from=" + rfc(now.Add(-500*24*time.Hour)),203 "label": "metrics=net.rx&label=" + strings.Repeat("x", 65), "bad date": "metrics=cpu.usage&from=yesterday",204 } {205 r := root.do("GET", "/nodes/"+a+"/metrics?"+q, nil)206 if r.Status != 400 || r.code() != "validation_failed" {207 t.Fatalf("%s: %d %s", name, r.Status, r.Raw)208 }209 if name == "unknown metric" && !strings.Contains(string(r.Raw), `"unknown":["nope.metric"]`) {210 t.Fatalf("the unknown metric must be named: %s", r.Raw)211 }212 }213 if r := root.do("GET", "/nodes/"+ids.New().String()+"/metrics?metrics=cpu.usage", nil); r.Status != 404 {214 t.Fatalf("unknown node: %d", r.Status)215 }216217 // Comparison across nodes.218 c := root.must("GET", "/metrics/compare?metric=cpu.usage&node_ids="+off+","+a, nil, 200).jsonObj(t)219 cs := c["series"].([]any)220 if c["metric"] != "cpu.usage" || c["unit"] != "percent" || c["resolution"] != "raw" || len(cs) != 2 ||221 cs[0].(map[string]any)["node_id"] != off || cs[0].(map[string]any)["node_alias"] != "charlie" || cs[1].(map[string]any)["node_id"] != a ||222 dig(c, "availability", a) != "available" || dig(c, "availability", off) != "offline" {223 t.Fatalf("compare: %v", c)224 }225 if r := root.do("GET", "/metrics/compare?metric=cpu.usage&node_ids="+a+","+ids.New().String(), nil); r.Status != 404 {226 t.Fatalf("compare with an unknown node: %d %s", r.Status, r.Raw)227 }228 if r := root.do("GET", "/metrics/compare?metric=nope&node_ids="+a, nil); r.Status != 400 {229 t.Fatalf("compare with an unknown metric: %d", r.Status)230 }231232 // Catalog: served from the table, seeded by the API itself.233 cat := root.must("GET", "/metrics/catalog", nil, 200).jsonItems(t)234 if len(cat) != len(protocol.Catalog) || h.count(`select count(*) from metric_catalog`) != len(protocol.Catalog) {235 t.Fatalf("catalog: %d entries, want %d", len(cat), len(protocol.Catalog))236 }237 if e := rowBy(cat, "name", "net.rx"); e["unit"] != "bytes_per_s" || e["labeled"] != true || e["kind"] != "counter_rate" || e["interval_s"] != 10.0 {238 t.Fatalf("catalog entry: %v", e)239 }240}241242func TestClusterTrends(t *testing.T) {243 h := newHarness(t)244 root := h.owner()245 h.seedPartitions()246 n1 := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second, Tags: []string{"gpu", "studio"}})247 n2 := h.seedNode(nodeSeed{Alias: "bravo", Seen: true, Age: time.Second, Site: "lab", Role: "gateway", Tags: []string{"gpu"}})248 n3 := h.seedNode(nodeSeed{Alias: "charlie", Seen: true, Age: time.Second})249 h.dbExec(`update nodes set retired_at = now() where id = $1`, n3)250 h.seedNode(nodeSeed{Alias: "delta", Manual: true})251 cpu, mem, rx := seedMetricID(t, "cpu.usage"), seedMetricID(t, "mem.used"), seedMetricID(t, "net.rx")252 raw := func(node string, metric int16, label string, v float64, minutes int) {253 h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, $3, t, $4254 from generate_series(date_bin('10 seconds', now(), 'epoch') - make_interval(mins => $5), date_bin('10 seconds', now(), 'epoch'), interval '10 seconds') t`,255 node, metric, label, v, minutes)256 }257 raw(n1, cpu, "", 10, 30)258 raw(n2, cpu, "", 30, 10) // reports for the last 10 minutes only259 raw(n3, cpu, "", 99, 30) // retired: never part of the cluster figure260 raw(n1, mem, "", 100, 5)261 raw(n2, mem, "", 200, 5)262 raw(n1, rx, "en0", 5, 5)263 raw(n1, rx, "en1", 7, 5)264 raw(n2, rx, "en0", 1, 5)265266 tr := root.must("GET", "/cluster/trends?metric=cpu.usage&range=1h", nil, 200).jsonObj(t)267 if tr["aggregation"] != "mean" || tr["resolution"] != "raw" || tr["unit"] != "percent" || tr["group_by"] != "cluster" || tr["range"] != "1h" {268 t.Fatalf("trend header: %v", tr)269 }270 ss := tr["series"].([]any)271 if len(ss) != 1 || ss[0].(map[string]any)["group"] != "cluster" || ss[0].(map[string]any)["coverage"] != 2.0 {272 t.Fatalf("cluster series: %v", ss)273 }274 pts := points(t, ss[0])275 if len(pts) < 175 || len(pts) > 185 {276 t.Fatalf("want ~181 ten-second points, got %d", len(pts))277 }278 both := 0279 for i, p := range pts {280 switch {281 case p[1] == 10 && p[2] == 10 && p[3] == 10: // only alpha reported in this bucket: no value is carried forward for bravo282 case p[1] == 20 && p[2] == 10 && p[3] == 30:283 both++284 default:285 t.Fatalf("point %d = %v", i, p)286 }287 if i > 0 && p[0]-pts[i-1][0] != 10000 {288 t.Fatalf("spacing at %d", i)289 }290 }291 if both < 55 || both > 65 {292 t.Fatalf("both nodes contribute for ~10 minutes, got %d buckets", both)293 }294295 bySite := root.must("GET", "/cluster/trends?metric=cpu.usage&range=1h&group_by=site", nil, 200).jsonObj(t)["series"].([]any)296 if len(bySite) != 2 || bySite[0].(map[string]any)["group"] != "default" || points(t, bySite[0])[0][1] != 10 ||297 bySite[1].(map[string]any)["group"] != "lab" || points(t, bySite[1])[0][1] != 30 {298 t.Fatalf("by site: %v", bySite)299 }300 byTag := root.must("GET", "/cluster/trends?metric=cpu.usage&range=1h&group_by=tag", nil, 200).jsonObj(t)["series"].([]any)301 if len(byTag) != 2 || byTag[0].(map[string]any)["group"] != "gpu" || byTag[0].(map[string]any)["coverage"] != 2.0 || byTag[1].(map[string]any)["group"] != "studio" {302 t.Fatalf("by tag: %v", byTag)303 }304 if byRole := root.must("GET", "/cluster/trends?metric=cpu.usage&range=1h&group_by=role", nil, 200).jsonObj(t)["series"].([]any); len(byRole) != 2 {305 t.Fatalf("by role: %v", byRole)306 }307308 // Additive quantities are summed; labels of one node are combined first.309 sum := root.must("GET", "/cluster/trends?metric=mem.used&range=1h", nil, 200).jsonObj(t)310 if p := points(t, sum["series"].([]any)[0]); sum["aggregation"] != "sum" || p[len(p)/2][1] != 300 || p[len(p)/2][2] != 300 {311 t.Fatalf("mem.used sum: %v %v", sum["aggregation"], p[len(p)/2])312 }313 net := root.must("GET", "/cluster/trends?metric=net.rx&range=1h", nil, 200).jsonObj(t)314 if p := points(t, net["series"].([]any)[0]); p[len(p)/2][1] != 13 {315 t.Fatalf("net.rx: 5 + 7 on alpha, 1 on bravo = 13, got %v", p[len(p)/2])316 }317318 // Longer ranges read the rollup tiers.319 h.dbExec(`insert into samples_1m(node_id, metric_id, label, ts, avg, min, max, n) values ($1, $2, '', date_trunc('minute', now()) - interval '3 hours', 40, 35, 45, 6)`, n1, cpu)320 h.dbExec(`insert into samples_1h(node_id, metric_id, label, ts, avg, min, max, n) values ($1, $2, '', date_trunc('hour', now()) - interval '3 days', 60, 50, 70, 360)`, n1, cpu)321 for rng, want := range map[string][]any{"24h": {"1m", 40.0, 35.0, 45.0}, "7d": {"1h", 60.0, 50.0, 70.0}, "30d": {"1h", 60.0, 50.0, 70.0}} {322 tr := root.must("GET", "/cluster/trends?metric=cpu.usage&range="+rng, nil, 200).jsonObj(t)323 p := points(t, tr["series"].([]any)[0])324 if tr["resolution"] != want[0] || len(p) != 1 || p[0][1] != want[1] || p[0][2] != want[2] || p[0][3] != want[3] {325 t.Fatalf("range %s: %v %v", rng, tr["resolution"], p)326 }327 }328 // Nothing reported: an empty series with coverage 0, not zeros.329 empty := root.must("GET", "/cluster/trends?metric=gpu.usage", nil, 200).jsonObj(t)["series"].([]any)330 if len(empty) != 1 || empty[0].(map[string]any)["coverage"] != 0.0 || len(points(t, empty[0])) != 0 {331 t.Fatalf("empty trend: %v", empty)332 }333 for _, bad := range []string{"metric=nope", "metric=cpu.usage&range=2h", "metric=cpu.usage&group_by=color", "range=1h"} {334 if r := root.do("GET", "/cluster/trends?"+bad, nil); r.Status != 400 {335 t.Fatalf("%s: %d %s", bad, r.Status, r.Raw)336 }337 }338}339340func TestNodeHealthHistory(t *testing.T) {341 h := newHarness(t)342 root := h.owner()343 h.seedPartitions()344 n := h.seedNode(nodeSeed{Alias: "alpha", Seen: true, Age: time.Second})345 // Reports from 30 minutes before the current 5-minute bucket until now, silent from -20 to -12 minutes.346 const b = `date_bin('5 minutes', now(), 'epoch')`347 h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value) select $1, $2, '', t, 1348 from generate_series(`+b+` - interval '30 minutes', now(), interval '10 seconds') t349 where not (t >= `+b+` - interval '20 minutes' and t < `+b+` - interval '12 minutes')`, n, seedMetricID(t, "cpu.usage"))350 h.seedAlert(n, "firing", "warning", "hot") // fired 9 minutes ago351352 states := func(hh map[string]any) string {353 var s []string354 for _, it := range hh["items"].([]any) {355 s = append(s, it.(map[string]any)["state"].(string))356 }357 return strings.Join(s, " ")358 }359 hh := root.must("GET", "/nodes/"+n+"/health-history?range=1h", nil, 200).jsonObj(t)360 if hh["bucket_seconds"] != 300.0 || hh["range"] != "1h" || hh["node_id"] != n {361 t.Fatalf("header: %v", hh)362 }363 // 12 buckets, oldest first: before the first report, then up, up, down (silent), degraded (partly silent), up, up, current.364 if got := states(hh); got != "unknown unknown unknown unknown unknown up up down degraded up up up" {365 t.Fatalf("states: %s", got)366 }367 if r, _ := hh["uptime_ratio"].(float64); math.Abs(r-6.0/7.0) > 1e-6 {368 t.Fatalf("uptime ratio %v, want 6/7", hh["uptime_ratio"])369 }370 items := hh["items"].([]any)371 if items[11].(map[string]any)["alerts_firing"] != 1.0 || items[10].(map[string]any)["alerts_firing"] != 1.0 || items[5].(map[string]any)["alerts_firing"] != 0.0 {372 t.Fatalf("alerts per bucket: %v", items)373 }374 for i := 1; i < len(items); i++ {375 if items[i].(map[string]any)["start"] != items[i-1].(map[string]any)["end"] {376 t.Fatalf("buckets must be contiguous: %v then %v", items[i-1], items[i])377 }378 }379380 // An applied maintenance window marks its buckets as maintenance.381 h.dbExec(`insert into maintenance_windows(id, title, starts_at, ends_at, scope, status) values ($1, 'swap disk', `+b+` - interval '30 minutes',382 `+b+` - interval '25 minutes', jsonb_build_object('node_ids', jsonb_build_array($2::text)), 'completed')`, ids.New().String(), n)383 h.dbExec(`insert into maintenance_windows(id, title, starts_at, ends_at, scope, status) values ($1, 'never applied', `+b+` - interval '10 minutes',384 `+b+`, '{"all": true}', 'cancelled')`, ids.New().String())385 hh = root.must("GET", "/nodes/"+n+"/health-history?range=1h", nil, 200).jsonObj(t)386 if got := states(hh); got != "unknown unknown unknown unknown unknown maintenance up down degraded up up up" {387 t.Fatalf("states with maintenance: %s", got)388 }389 if r, _ := hh["uptime_ratio"].(float64); math.Abs(r-5.0/7.0) > 1e-6 {390 t.Fatalf("uptime ratio %v, want 5/7", hh["uptime_ratio"])391 }392393 for rng, want := range map[string][]float64{"24h": {48, 1800}, "7d": {56, 10800}, "30d": {60, 43200}} {394 hh := root.must("GET", "/nodes/"+n+"/health-history?range="+rng, nil, 200).jsonObj(t)395 items := hh["items"].([]any)396 if float64(len(items)) != want[0] || hh["bucket_seconds"] != want[1] || items[0].(map[string]any)["state"] != "unknown" ||397 items[len(items)-1].(map[string]any)["state"] == "unknown" {398 t.Fatalf("range %s: %d buckets of %v s, first %v, last %v", rng, len(items), hh["bucket_seconds"], items[0], items[len(items)-1])399 }400 }401 if r := root.do("GET", "/nodes/"+n+"/health-history?range=1y", nil); r.Status != 400 {402 t.Fatalf("bad range: %d", r.Status)403 }404}405406// The time-series queries must only touch the partitions that overlap the407// requested range, and reach their rows through the primary key. This is408// checked on the statistics PostgreSQL keeps for each partition, after409// running the real endpoints against daily partitions holding 30 nodes of410// 10-second samples.411func TestSeriesQueriesArePrunedAndIndexDriven(t *testing.T) {412 if testing.Short() {413 t.Skip("loads several hundred thousand samples")414 }415 h := newHarness(t)416 root := h.owner()417 ctx := context.Background()418 day := time.Now().UTC().Truncate(24 * time.Hour)419 for d := -4; d <= 1; d++ {420 from := day.AddDate(0, 0, d)421 h.dbExec(fmt.Sprintf(`create table samples_raw_d%s partition of samples_raw for values from ('%s') to ('%s')`,422 from.Format("20060102"), from.Format(time.RFC3339), from.AddDate(0, 0, 1).Format(time.RFC3339)))423 }424 h.dbExec(`create table samples_1m_all partition of samples_1m for values from (minvalue) to (maxvalue)`)425 h.dbExec(`create table samples_1h_all partition of samples_1h for values from (minvalue) to (maxvalue)`)426 var nodes []string427 for i := 0; i < 30; i++ {428 nodes = append(nodes, h.seedNode(nodeSeed{Alias: fmt.Sprintf("node%02d", i), Seen: true, Age: time.Second, Values: map[string]any{"cpu.usage": 1.0}}))429 }430 // 30 nodes x 3 metrics x 10 s over the last 50 hours: about 1.6 M rows in three daily partitions.431 h.dbExec(`insert into samples_raw(node_id, metric_id, label, ts, value)432 select n.id, m, '', t, random() * 100 from nodes n, unnest($1::smallint[]) m,433 generate_series(now() - interval '50 hours', now(), interval '10 seconds') t`,434 []int16{seedMetricID(t, "cpu.usage"), seedMetricID(t, "load.1"), seedMetricID(t, "mem.used")})435 h.dbExec(`analyze samples_raw`)436437 type stat struct{ seq, idx int64 }438 read := func() map[string]stat {439 out := map[string]stat{}440 rows, err := h.Pool.Query(ctx, `select relname, coalesce(seq_scan, 0), coalesce(idx_scan, 0) from pg_stat_user_tables where relname like 'samples_raw_d%'`)441 if err != nil {442 t.Fatal(err)443 }444 defer rows.Close()445 for rows.Next() {446 var name string447 var s stat448 if err := rows.Scan(&name, &s.seq, &s.idx); err != nil {449 t.Fatal(err)450 }451 out[name] = s452 }453 return out454 }455 // settle waits until the per-table counters (flushed asynchronously by the backends) stop moving.456 settle := func() map[string]stat {457 prev := read()458 for i := 0; i < 40; i++ {459 time.Sleep(300 * time.Millisecond)460 cur := read()461 same := true462 for k, v := range cur {463 same = same && prev[k] == v464 }465 if same && i >= 5 {466 return cur467 }468 prev = cur469 }470 return prev471 }472 before := settle()473 now := time.Now().UTC()474 root.must("GET", "/nodes/"+nodes[3]+"/metrics?metrics=cpu.usage,load.1", nil, 200) // last hour, raw points475 root.must("GET", "/nodes/"+nodes[3]+"/metrics?metrics=cpu.usage&resolution=raw&from="+rfc(now.Add(-20*time.Hour)), nil, 200) // bucketed476 root.must("GET", "/metrics/compare?metric=cpu.usage&node_ids="+strings.Join(nodes[:6], ","), nil, 200)477 root.must("GET", "/cluster/trends?metric=cpu.usage&range=1h", nil, 200)478 after := settle()479480 oldest := now.Add(-20*time.Hour - time.Minute) // earliest instant any of the queries above may read481 used := 0482 for name, a := range after {483 b := before[name]484 pday, err := time.Parse("20060102", strings.TrimPrefix(name, "samples_raw_d"))485 if err != nil {486 t.Fatal(err)487 }488 inRange := pday.Add(24*time.Hour).After(oldest) && !pday.After(now)489 if a.seq != b.seq {490 t.Errorf("%s: %d sequential scans by the series queries", name, a.seq-b.seq)491 }492 switch {493 case inRange && a.idx > b.idx:494 used++495 case !inRange && a.idx != b.idx:496 t.Errorf("%s is outside the requested ranges but was scanned %d times: partitions are not pruned", name, a.idx-b.idx)497 }498 }499 t.Logf("per-partition {seq_scan idx_scan} before %v, after %v", before, after)500 if used == 0 {501 t.Fatalf("no partition of the requested range shows an index scan: before %v after %v", before, after)502 }503504 // Health history reads one node through the primary key as well (its "first sample" probe touches every partition, by index).505 root.must("GET", "/nodes/"+nodes[3]+"/health-history?range=24h", nil, 200)506 final := settle()507 for name, f := range final {508 if f.seq != after[name].seq {509 t.Errorf("%s: sequential scan by the health history", name)510 }511 }512}513