Repository Public
1package api23import (4 "context"5 "math"6 "slices"7 "sort"8 "strings"9 "time"1011 "github.com/google/uuid"12 "github.com/jackc/pgx/v5"1314 "clustr.spboucher.ai/mcc/internal/authz"15 "clustr.spboucher.ai/mcc/internal/httpx"16)1718// ── Time-series engine ─────────────────────────────────────────────────────19//20// Samples live in three partitioned tables keyed by (node_id, metric_id,21// label, ts). Every query below fixes node_id and metric_id and bounds ts on22// both sides, so it is served by the primary key inside the partitions that23// overlap the range (the others are pruned).24//25// A series never carries more than clMaxPoints points: wider ranges are26// reduced on the server into buckets that keep the true average (weighted by27// sample count for the rollup tiers), minimum and maximum. A bucket without28// data produces no point: gaps stay gaps.2930type clTier struct {31 res Resolution32 table string33 step int64 // native spacing in seconds (bucket widths are multiples of it)34 maxRange time.Duration // widest range served from this tier35 rollup bool // columns avg/min/max/n instead of value36}3738var (39 clTierRaw = clTier{res: ResolutionRaw, table: "samples_raw", step: 10, maxRange: 24 * time.Hour}40 clTier1m = clTier{res: ResolutionN1m, table: "samples_1m", step: 60, maxRange: 31 * 24 * time.Hour, rollup: true}41 clTier1h = clTier{res: ResolutionN1h, table: "samples_1h", step: 3600, maxRange: 400 * 24 * time.Hour, rollup: true}42)4344const (45 clMaxPoints = 1500 // per series46 clMinPoints = 120 // per series, however many series are requested47 clPointBudget = 30000 // per response, across series48 clMaxSeriesRows = 25000049 clAutoRawUpTo = 2 * time.Hour50 clAuto1mUpTo = 48 * time.Hour51 clDefaultRange = time.Hour52 clMaxMetricRange = 400 * 24 * time.Hour53)5455// clRange validates the requested time range. `to` defaults to now and is56// never in the future; `from` defaults to one hour before `to`.57func clRange(from, to *time.Time, now time.Time) (time.Time, time.Time, error) {58 t := now59 if to != nil && to.Before(now) {60 t = to.UTC()61 }62 f := t.Add(-clDefaultRange)63 if from != nil {64 f = from.UTC()65 }66 switch {67 case !f.Before(t):68 return f, t, clInvalid("from", "must be before to (and before now)")69 case t.Sub(f) > clMaxMetricRange:70 return f, t, clInvalid("from", "the range must not exceed 400 days")71 }72 return f, t, nil73}7475// clPickTier chooses the table. auto: raw up to 2 h, 1 min up to 48 h, 1 h76// beyond (the contract's rule). An explicit resolution is honoured within the77// range that tier can serve at a bounded cost.78func clPickTier(res *Resolution, span time.Duration) (clTier, error) {79 r := ResolutionAuto80 if res != nil {81 r = *res82 }83 switch r {84 case ResolutionAuto:85 switch {86 case span <= clAutoRawUpTo:87 return clTierRaw, nil88 case span <= clAuto1mUpTo:89 return clTier1m, nil90 }91 return clTier1h, nil92 case ResolutionRaw:93 if span > clTierRaw.maxRange {94 return clTierRaw, clInvalid("resolution", "raw is limited to a 24 hour range; use 1m, 1h or auto")95 }96 return clTierRaw, nil97 case ResolutionN1m:98 if span > clTier1m.maxRange {99 return clTier1m, clInvalid("resolution", "1m is limited to a 31 day range; use 1h or auto")100 }101 return clTier1m, nil102 case ResolutionN1h:103 return clTier1h, nil104 }105 return clTierRaw, clInvalid("resolution", "must be auto, raw, 1m or 1h")106}107108// clBucketWidth returns the bucket width in seconds: the tier's native step109// when the range fits in maxPoints, else the smallest multiple of the step110// that does.111func clBucketWidth(span time.Duration, step int64, maxPoints int) int64 {112 secs := int64(math.Ceil(span.Seconds()))113 if secs <= step*int64(maxPoints) {114 return step115 }116 w := (secs + int64(maxPoints) - 1) / int64(maxPoints)117 return (w + step - 1) / step * step118}119120type clSeriesKey struct {121 node uuid.UUID122 metric int16123 label string124}125126type clSeriesData struct {127 key clSeriesKey128 points MetricPoints129}130131// clSeries reads the series of the given nodes and metrics over [from, to).132// When width equals the tier step the stored points are returned as they are;133// otherwise they are reduced into buckets of `width` seconds aligned on the134// epoch, stamped with the bucket start.135func (h *clusterHandlers) clSeries(ctx context.Context, tier clTier, nodes []uuid.UUID, metrics []int16, from, to time.Time, width int64, label string) ([]clSeriesData, error) {136 var sel string137 switch {138 case width == tier.step && !tier.rollup:139 sel = `select node_id, metric_id, label, extract(epoch from ts)::float8, value, value, value from ` + tier.table +140 ` where node_id = any($1) and metric_id = any($2) and ts >= $3 and ts < $4 and ($5 = '' or label = $5 or label = '')141 order by node_id, metric_id, label, ts limit $6`142 case width == tier.step:143 sel = `select node_id, metric_id, label, extract(epoch from ts)::float8, avg, min, max from ` + tier.table +144 ` where node_id = any($1) and metric_id = any($2) and ts >= $3 and ts < $4 and ($5 = '' or label = $5 or label = '')145 order by node_id, metric_id, label, ts limit $6`146 case !tier.rollup:147 sel = `select node_id, metric_id, label, (floor(extract(epoch from ts) / $7::float8) * $7::float8)::float8 as b, avg(value), min(value), max(value)148 from ` + tier.table + ` where node_id = any($1) and metric_id = any($2) and ts >= $3 and ts < $4 and ($5 = '' or label = $5 or label = '')149 group by node_id, metric_id, label, b order by node_id, metric_id, label, b limit $6`150 default:151 sel = `select node_id, metric_id, label, (floor(extract(epoch from ts) / $7::float8) * $7::float8)::float8 as b,152 sum(avg * n) / nullif(sum(n), 0), min(min), max(max)153 from ` + tier.table + ` where node_id = any($1) and metric_id = any($2) and ts >= $3 and ts < $4 and ($5 = '' or label = $5 or label = '')154 group by node_id, metric_id, label, b order by node_id, metric_id, label, b limit $6`155 }156 args := []any{nodes, metrics, from, to, label, clMaxSeriesRows + 1}157 if width != tier.step {158 args = append(args, float64(width))159 }160 rows, err := h.Pool.Query(ctx, sel, args...)161 if err != nil {162 return nil, err163 }164 defer rows.Close()165 var out []clSeriesData166 n := 0167 for rows.Next() {168 var k clSeriesKey169 var ts float64170 var avg, min, max *float64171 if err := rows.Scan(&k.node, &k.metric, &k.label, &ts, &avg, &min, &max); err != nil {172 return nil, err173 }174 if n++; n > clMaxSeriesRows {175 return nil, clInvalid("from", "the query returns too many points; narrow the range, the metrics or the label")176 }177 if avg == nil || min == nil || max == nil {178 continue // a rollup row without samples carries no information179 }180 if len(out) == 0 || out[len(out)-1].key != k {181 out = append(out, clSeriesData{key: k, points: MetricPoints{}})182 }183 s := &out[len(out)-1]184 s.points = append(s.points, []float64{math.Round(ts * 1000), *avg, *min, *max})185 }186 return out, rows.Err()187}188189// clMetricNow is the CURRENT availability of a metric on a node (for one190// label, or for any of its labels when label is empty).191func clMetricNow(m clMetric, label string, fresh Freshness, reported bool, values map[string]any, avail map[string]string) Availability {192 if !reported {193 return AvailabilityOffline194 }195 has := false196 switch {197 case !m.Labeled:198 _, has = clNumber(values, m.Name)199 label = ""200 case label != "":201 _, has = clNumber(values, clValueKey(m.Name, label))202 default:203 has = len(clLabels(values, m.Name)) > 0204 }205 if has {206 return clAvail(fresh)207 }208 if a, ok := clDeclared(avail, m.Name, label); ok {209 return a210 }211 if fresh == FreshnessOffline {212 return AvailabilityOffline213 }214 return AvailabilityUnavailable215}216217func clLabelParam(l *string) (string, error) {218 if l == nil {219 return "", nil220 }221 if len(*l) > 64 || !clPrintable(*l) {222 return "", clInvalid("label", "must be at most 64 printable characters")223 }224 return *l, nil225}226227// clMetricNames validates the requested metric names against the catalog.228func clMetricNames(cat *clCatalog, field string, names []string, max int) ([]clMetric, error) {229 var out []clMetric230 var unknown []string231 for _, raw := range names {232 name := strings.TrimSpace(raw)233 if name == "" {234 continue235 }236 m, ok := cat.byName[name]237 switch {238 case !ok:239 if len(name) > 64 {240 name = name[:64]241 }242 unknown = append(unknown, name)243 case !slices.ContainsFunc(out, func(x clMetric) bool { return x.ID == m.ID }):244 out = append(out, m)245 }246 }247 if len(unknown) > 0 {248 return nil, clInvalid(field, "unknown metric (see GET /metrics/catalog)").With("unknown", unknown)249 }250 if len(out) == 0 {251 return nil, clInvalid(field, "at least one metric is required")252 }253 if len(out) > max {254 return nil, clInvalid(field, "at most "+itoaInt(max)+" metrics")255 }256 return out, nil257}258259func (h *clusterHandlers) GetNodeMetrics(ctx context.Context, req GetNodeMetricsRequestObject) (GetNodeMetricsResponseObject, error) {260 p := req.Params261 cat, err := h.clCatalogFor(ctx)262 if err != nil {263 return nil, err264 }265 metrics, err := clMetricNames(cat, "metrics", p.Metrics, 12)266 if err != nil {267 return nil, err268 }269 label, err := clLabelParam(p.Label)270 if err != nil {271 return nil, err272 }273 now := time.Now().UTC()274 from, to, err := clRange(p.From, p.To, now)275 if err != nil {276 return nil, err277 }278 tier, err := clPickTier(p.Resolution, to.Sub(from))279 if err != nil {280 return nil, err281 }282 acc, err := h.clAccessFor(ctx)283 if err != nil {284 return nil, err285 }286 var valuesRaw, availRaw []byte287 n, err := h.readableNode(ctx, acc, now, req.Id, `, s.values, s.availability`, &valuesRaw, &availRaw)288 if err != nil {289 return nil, err290 }291 values, avail := clValues(valuesRaw), clStringMap(availRaw)292 fresh := h.clFresh(now, n.Seen)293294 // Point budget: the more series, the fewer points each.295 estimate := 0296 mids := make([]int16, 0, len(metrics))297 byID := map[int16]clMetric{}298 for _, m := range metrics {299 mids = append(mids, m.ID)300 byID[m.ID] = m301 if k := len(clLabels(values, m.Name)); m.Labeled && label == "" && k > 1 {302 estimate += k303 } else {304 estimate++305 }306 }307 width := clBucketWidth(to.Sub(from), tier.step, max(clMinPoints, min(clMaxPoints, clPointBudget/estimate)))308 data, err := h.clSeries(ctx, tier, []uuid.UUID{n.ID}, mids, from, to, width, label)309 if err != nil {310 return nil, err311 }312 out := MetricSeriesResponse{Resolution: tier.res, From: from, To: to, Series: make([]MetricSeries, 0, len(data)), Availability: map[string]Availability{}}313 for _, m := range metrics { // series in the order the metrics were requested314 for _, s := range data {315 if s.key.metric == m.ID {316 out.Series = append(out.Series, MetricSeries{Metric: m.Name, Unit: m.Unit, Label: s.key.label, Points: s.points})317 }318 }319 out.Availability[m.Name] = clMetricNow(m, label, fresh, n.ObservedAt != nil, values, avail)320 }321 return GetNodeMetrics200JSONResponse{Body: out}, nil322}323324func (h *clusterHandlers) CompareMetrics(ctx context.Context, req CompareMetricsRequestObject) (CompareMetricsResponseObject, error) {325 p := req.Params326 cat, err := h.clCatalogFor(ctx)327 if err != nil {328 return nil, err329 }330 ms, err := clMetricNames(cat, "metric", []string{p.Metric}, 1)331 if err != nil {332 return nil, err333 }334 m := ms[0]335 var nodeIDs []uuid.UUID336 for _, id := range p.NodeIds {337 if !slices.Contains(nodeIDs, id) {338 nodeIDs = append(nodeIDs, id)339 }340 }341 if len(nodeIDs) < 1 || len(nodeIDs) > 12 {342 return nil, clInvalid("node_ids", "give 1 to 12 node ids")343 }344 label, err := clLabelParam(p.Label)345 if err != nil {346 return nil, err347 }348 now := time.Now().UTC()349 from, to, err := clRange(p.From, p.To, now)350 if err != nil {351 return nil, err352 }353 tier, err := clPickTier(p.Resolution, to.Sub(from))354 if err != nil {355 return nil, err356 }357 acc, err := h.clAccessFor(ctx)358 if err != nil {359 return nil, err360 }361 nodes, err := h.netNodes(ctx, now, ` and n.id = any($4)`, []any{nodeIDs}, "")362 if err != nil {363 return nil, err364 }365 byID := map[uuid.UUID]*clNetNode{}366 estimate := 0367 for i := range nodes {368 x := &nodes[i]369 if !acc.can(authz.NodesRead, x.n.facets()...) {370 continue371 }372 byID[x.n.ID] = x373 if k := len(clLabels(x.values, m.Name)); m.Labeled && label == "" && k > 1 {374 estimate += k375 } else {376 estimate++377 }378 }379 for _, id := range nodeIDs {380 if byID[id] == nil {381 return nil, httpx.ErrNotFound.With("node_id", id.String()) // unknown, retired or hidden382 }383 }384 width := clBucketWidth(to.Sub(from), tier.step, max(clMinPoints, min(clMaxPoints, clPointBudget/estimate)))385 data, err := h.clSeries(ctx, tier, nodeIDs, []int16{m.ID}, from, to, width, label)386 if err != nil {387 return nil, err388 }389 out := MetricCompareResponse{Metric: m.Name, Unit: m.Unit, Resolution: tier.res, From: from, To: to,390 Series: make([]MetricSeries, 0, len(data)), Availability: map[string]Availability{}}391 for _, id := range nodeIDs { // series in the order the nodes were requested392 x := byID[id]393 for _, s := range data {394 if s.key.node == id {395 out.Series = append(out.Series, MetricSeries{Metric: m.Name, Unit: m.Unit, Label: s.key.label, Points: s.points,396 NodeId: &x.n.ID, NodeAlias: &x.n.Alias})397 }398 }399 out.Availability[id.String()] = clMetricNow(m, label, h.clFresh(now, x.n.Seen), x.n.ObservedAt != nil, x.values, x.avail)400 }401 return CompareMetrics200JSONResponse{Body: out}, nil402}403404func (h *clusterHandlers) GetMetricCatalog(ctx context.Context, _ GetMetricCatalogRequestObject) (GetMetricCatalogResponseObject, error) {405 cat, err := h.clCatalogFor(ctx)406 if err != nil {407 return nil, err408 }409 out := MetricCatalog{Items: make([]MetricCatalogEntry, 0, len(cat.list))}410 for _, m := range cat.list {411 kind := MetricCatalogEntryKindGauge412 if m.Kind == string(MetricCatalogEntryKindCounterRate) {413 kind = MetricCatalogEntryKindCounterRate414 }415 out.Items = append(out.Items, MetricCatalogEntry{Name: m.Name, Unit: m.Unit, Kind: kind, IntervalS: m.IntervalS, Labeled: m.Labeled,416 Source: m.Source, Description: m.Description})417 }418 return GetMetricCatalog200JSONResponse{Body: out}, nil419}420421// ── Cluster trends ─────────────────────────────────────────────────────────422423type clTrendPlan struct {424 span time.Duration425 tier clTier426 width int64427}428429// The contract fixes the resolution of each range.430var clTrendPlans = map[TimeRange]clTrendPlan{431 TimeRangeN1h: {time.Hour, clTierRaw, 10},432 TimeRangeN24h: {24 * time.Hour, clTier1m, 60},433 TimeRangeN7d: {7 * 24 * time.Hour, clTier1h, 3600},434 TimeRangeN30d: {30 * 24 * time.Hour, clTier1h, 3600},435}436437// clAdditive says how nodes (and the labels of one node) combine: quantities438// that add up are summed, everything else is averaged.439func clAdditive(unit string) bool {440 switch unit {441 case "bytes", "bytes_per_s", "watts":442 return true443 }444 return false445}446447const clMaxTrendNodes = 1000448449// GetClusterTrends aggregates one metric across nodes. For each bucket a node450// contributes only if it has data in that bucket (no carry-forward). With451// `mean`, a point is [mean of node averages, lowest node minimum, highest452// node maximum]; with `sum`, [sum of averages, sum of minima, sum of maxima]453// (bounds of the total). `coverage` is the number of nodes that contributed454// to the series at least once in the range.455func (h *clusterHandlers) GetClusterTrends(ctx context.Context, req GetClusterTrendsRequestObject) (GetClusterTrendsResponseObject, error) {456 p := req.Params457 cat, err := h.clCatalogFor(ctx)458 if err != nil {459 return nil, err460 }461 ms, err := clMetricNames(cat, "metric", []string{p.Metric}, 1)462 if err != nil {463 return nil, err464 }465 m := ms[0]466 rng := TimeRangeN24h467 if p.Range != nil {468 rng = *p.Range469 }470 plan, ok := clTrendPlans[rng]471 if !ok {472 return nil, clInvalid("range", "must be 1h, 24h, 7d or 30d")473 }474 groupBy := TrendGroupByCluster475 if p.GroupBy != nil {476 groupBy = *p.GroupBy477 }478 if !slices.Contains([]TrendGroupBy{TrendGroupByCluster, TrendGroupBySite, TrendGroupByRole, TrendGroupByTag}, groupBy) {479 return nil, clInvalid("group_by", "must be cluster, site, role or tag")480 }481 acc, err := h.clAccessFor(ctx)482 if err != nil {483 return nil, err484 }485 now := time.Now().UTC()486 from := time.Unix((now.Add(-plan.span).Unix()/plan.width)*plan.width, 0).UTC()487488 rows, err := h.Pool.Query(ctx, `select id, site, role, tags from nodes where retired_at is null order by id limit $1`, clMaxTrendNodes)489 if err != nil {490 return nil, err491 }492 type member struct {493 id uuid.UUID494 site, role string495 tags []string496 }497 members, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (member, error) {498 var x member499 err := r.Scan(&x.id, &x.site, &x.role, &x.tags)500 return x, err501 })502 if err != nil {503 return nil, err504 }505 groupsOf := map[uuid.UUID][]string{}506 var nodeIDs []uuid.UUID507 filter := acc.scoped(authz.NodesRead)508 for _, x := range members {509 if filter && !acc.can(authz.NodesRead, clNodeFacets(x.id, x.site, x.tags)...) {510 continue511 }512 nodeIDs = append(nodeIDs, x.id)513 switch groupBy {514 case TrendGroupBySite:515 groupsOf[x.id] = []string{x.site}516 case TrendGroupByRole:517 groupsOf[x.id] = []string{x.role}518 case TrendGroupByTag:519 groupsOf[x.id] = x.tags520 default:521 groupsOf[x.id] = []string{"cluster"}522 }523 }524 agg := TrendsResponseAggregationMean525 if clAdditive(m.Unit) {526 agg = TrendsResponseAggregationSum527 }528 out := TrendsResponse{Metric: m.Name, Unit: m.Unit, Range: rng, GroupBy: groupBy, Resolution: plan.tier.res, Aggregation: agg,529 Series: []TrendSeries{}, GeneratedAt: now}530531 // One row per node and bucket: the labels of a node (interfaces, mount points) are combined first.532 inner := `select node_id, label, (floor(extract(epoch from ts) / $5::float8) * $5::float8)::float8 as b, avg(value) as a, min(value) as mn, max(value) as mx533 from samples_raw where node_id = any($1) and metric_id = $2 and ts >= $3 and ts < $4 group by node_id, label, b`534 if plan.tier.rollup {535 inner = `select node_id, label, (floor(extract(epoch from ts) / $5::float8) * $5::float8)::float8 as b,536 sum(avg * n) / nullif(sum(n), 0) as a, min(min) as mn, max(max) as mx537 from ` + plan.tier.table + ` where node_id = any($1) and metric_id = $2 and ts >= $3 and ts < $4 group by node_id, label, b`538 }539 outer := `select x.node_id, x.b, avg(x.a), min(x.mn), max(x.mx) from (` + inner + `) x where x.a is not null group by x.node_id, x.b`540 if agg == TrendsResponseAggregationSum {541 outer = `select x.node_id, x.b, sum(x.a), sum(x.mn), sum(x.mx) from (` + inner + `) x where x.a is not null group by x.node_id, x.b`542 }543 type cell struct {544 n int545 avg, lo, hi float64546 sumLo, sumHi float64547 }548 type group struct {549 buckets map[float64]*cell550 nodes map[uuid.UUID]struct{}551 }552 groups := map[string]*group{}553 if len(nodeIDs) > 0 {554 rs, err := h.Pool.Query(ctx, outer, nodeIDs, m.ID, from, now, float64(plan.width))555 if err != nil {556 return nil, err557 }558 defer rs.Close()559 for rs.Next() {560 var id uuid.UUID561 var b, a, lo, hi float64562 if err := rs.Scan(&id, &b, &a, &lo, &hi); err != nil {563 return nil, err564 }565 for _, name := range groupsOf[id] {566 g := groups[name]567 if g == nil {568 g = &group{buckets: map[float64]*cell{}, nodes: map[uuid.UUID]struct{}{}}569 groups[name] = g570 }571 g.nodes[id] = struct{}{}572 c := g.buckets[b]573 if c == nil {574 c = &cell{lo: lo, hi: hi}575 g.buckets[b] = c576 }577 c.n++578 c.avg += a579 c.sumLo += lo580 c.sumHi += hi581 c.lo, c.hi = math.Min(c.lo, lo), math.Max(c.hi, hi)582 }583 }584 if err := rs.Err(); err != nil {585 return nil, err586 }587 }588 if groupBy == TrendGroupByCluster && groups["cluster"] == nil {589 groups["cluster"] = &group{} // an empty series states that no node contributed590 }591 names := make([]string, 0, len(groups))592 for name := range groups {593 names = append(names, name)594 }595 sort.Strings(names)596 for _, name := range names {597 g := groups[name]598 ts := make([]float64, 0, len(g.buckets))599 for b := range g.buckets {600 ts = append(ts, b)601 }602 sort.Float64s(ts)603 s := TrendSeries{Group: name, Coverage: len(g.nodes), Points: make(MetricPoints, 0, len(ts))}604 for _, b := range ts {605 c := g.buckets[b]606 if agg == TrendsResponseAggregationSum {607 s.Points = append(s.Points, []float64{b * 1000, c.avg, c.sumLo, c.sumHi})608 } else {609 s.Points = append(s.Points, []float64{b * 1000, c.avg / float64(c.n), c.lo, c.hi})610 }611 }612 out.Series = append(out.Series, s)613 }614 return GetClusterTrends200JSONResponse{Body: out}, nil615}616617// ── Health history ─────────────────────────────────────────────────────────618619type clHealthPlan struct {620 span time.Duration621 bucket int64 // seconds622 rollup bool // read samples_1m (plus raw for the most recent minutes)623}624625var clHealthPlans = map[TimeRange]clHealthPlan{626 TimeRangeN1h: {time.Hour, 300, false},627 TimeRangeN24h: {24 * time.Hour, 1800, false},628 TimeRangeN7d: {7 * 24 * time.Hour, 3 * 3600, true},629 TimeRangeN30d: {30 * 24 * time.Hour, 12 * 3600, true},630}631632const (633 clHealthSlot = 60 // seconds: a node is "heard" in a slot when a report landed in it634 clHealthTail = 15 * time.Minute // most recent part always read from raw (rollups lag)635)636637// Metrics every agent reports at each tick; a slot is covered when any of them has a sample.638var clHealthProbes = []string{"cpu.usage", "load.1", "mem.used"}639640// GetNodeHealthHistory derives reachability from the reports actually641// received: time is cut into one-minute slots, a slot is covered when at642// least one sample of a per-tick metric falls in it, and a bucket is `up`643// when all its (elapsed) slots are covered, `degraded` when some are, `down`644// when none is. Buckets before the node's first sample are `unknown`; buckets645// overlapping an applied maintenance window are `maintenance`.646// uptime_ratio = (up + degraded) / buckets that are not unknown.647func (h *clusterHandlers) GetNodeHealthHistory(ctx context.Context, req GetNodeHealthHistoryRequestObject) (GetNodeHealthHistoryResponseObject, error) {648 rng := TimeRangeN24h649 if req.Params.Range != nil {650 rng = *req.Params.Range651 }652 plan, ok := clHealthPlans[rng]653 if !ok {654 return nil, clInvalid("range", "must be 1h, 24h, 7d or 30d")655 }656 cat, err := h.clCatalogFor(ctx)657 if err != nil {658 return nil, err659 }660 acc, err := h.clAccessFor(ctx)661 if err != nil {662 return nil, err663 }664 now := time.Now().UTC()665 n, err := h.readableNode(ctx, acc, now, req.Id, "")666 if err != nil {667 return nil, err668 }669 var probes []int16670 for _, name := range clHealthProbes {671 if m, ok := cat.byName[name]; ok {672 probes = append(probes, m.ID)673 }674 }675 count := int64(plan.span.Seconds()) / plan.bucket676 lastStart := now.Unix() / plan.bucket * plan.bucket677 start := time.Unix(lastStart-(count-1)*plan.bucket, 0).UTC()678 out := HealthHistory{NodeId: n.ID, Range: rng, BucketSeconds: int(plan.bucket), Items: make([]HealthBucket, 0, count)}679680 // First sample ever kept for this node, across tiers (each lookup is one index probe per metric).681 var first *time.Time682 err = h.Pool.QueryRow(ctx, `select min(t.ts) from unnest($2::smallint[]) as mid, lateral (683 (select ts from samples_1h where node_id = $1 and metric_id = mid and label = '' order by ts limit 1)684 union all (select ts from samples_1m where node_id = $1 and metric_id = mid and label = '' order by ts limit 1)685 union all (select ts from samples_raw where node_id = $1 and metric_id = mid and label = '' order by ts limit 1)) t`,686 n.ID, probes).Scan(&first)687 if err != nil {688 return nil, err689 }690691 covered := map[int64]int64{} // bucket start -> covered slots692 if first != nil {693 const slot = `floor(extract(epoch from ts) / 60)::bigint`694 sql := `select (u.slot * 60 / $5::bigint) * $5::bigint, count(*) from (695 select distinct ` + slot + ` as slot from samples_raw696 where node_id = $1 and metric_id = any($2) and label = '' and ts >= $3 and ts < $4) u group by 1`697 args := []any{n.ID, probes, start, now, plan.bucket}698 if cut := now.Add(-clHealthTail).Truncate(time.Minute); plan.rollup && cut.After(start) {699 sql = `select (u.slot * 60 / $6::bigint) * $6::bigint, count(*) from (700 select distinct ` + slot + ` as slot from samples_1m701 where node_id = $1 and metric_id = any($2) and label = '' and ts >= $3 and ts < $4 and n > 0702 union703 select distinct ` + slot + ` from samples_raw704 where node_id = $1 and metric_id = any($2) and label = '' and ts >= $4 and ts < $5) u group by 1`705 args = []any{n.ID, probes, start, cut, now, plan.bucket}706 }707 rows, err := h.Pool.Query(ctx, sql, args...)708 if err != nil {709 return nil, err710 }711 var b, c int64712 if _, err := pgx.ForEachRow(rows, []any{&b, &c}, func() error { covered[b] = c; return nil }); err != nil {713 return nil, err714 }715 }716717 // Maintenance windows that were applied to this node, and its alerts, over the range.718 type span struct{ from, to time.Time }719 var maint, firing []span720 rows, err := h.Pool.Query(ctx, `select starts_at, ends_at from maintenance_windows721 where status in ('in_progress','completed') and set_maintenance_mode and recurrence = 'none' and starts_at < $3 and ends_at > $2722 and (scope->'node_ids' ? $1 or scope->'all' = 'true'::jsonb)`, n.ID.String(), start, now)723 if err != nil {724 return nil, err725 }726 if maint, err = pgx.CollectRows(rows, func(r pgx.CollectableRow) (span, error) {727 var s span728 err := r.Scan(&s.from, &s.to)729 return s, err730 }); err != nil {731 return nil, err732 }733 if acc.can(authz.AlertsRead) {734 rows, err := h.Pool.Query(ctx, `select fired_at, resolved_at from alerts735 where node_id = $1 and fired_at is not null and fired_at < $3 and (resolved_at is null or resolved_at > $2) limit 5000`, n.ID, start, now)736 if err != nil {737 return nil, err738 }739 if firing, err = pgx.CollectRows(rows, func(r pgx.CollectableRow) (span, error) {740 var s span741 var to *time.Time742 err := r.Scan(&s.from, &to)743 s.to = now.Add(time.Hour) // still firing744 if to != nil {745 s.to = *to746 }747 return s, err748 }); err != nil {749 return nil, err750 }751 }752753 known, good := 0, 0754 for i := int64(0); i < count; i++ {755 bs := start.Add(time.Duration(i*plan.bucket) * time.Second)756 be := bs.Add(time.Duration(plan.bucket) * time.Second)757 item := HealthBucket{Start: bs, End: be, State: HealthBucketStateUnknown}758 for _, f := range firing {759 if f.from.Before(be) && f.to.After(bs) {760 item.AlertsFiring++761 }762 }763 inMaint := n.Maintenance && !now.Before(bs) && now.Before(be)764 for _, w := range maint {765 inMaint = inMaint || (w.from.Before(be) && w.to.After(bs))766 }767 switch {768 case first == nil || !be.After(*first):769 // before the first report: nothing is known770 case inMaint:771 item.State = HealthBucketStateMaintenance772 default:773 // Slots that should have been heard: from the first report (if it falls in this bucket) to the last complete minute.774 lo, hi := bs.Unix()/clHealthSlot, be.Unix()/clHealthSlot775 if f := first.Unix() / clHealthSlot; f > lo {776 lo = f777 }778 if nowSlot := now.Unix() / clHealthSlot; nowSlot < hi {779 hi = nowSlot780 }781 expected, got := hi-lo, covered[bs.Unix()]782 switch {783 case expected <= 0 && got == 0:784 // the bucket has only just begun785 case got >= expected:786 item.State = HealthBucketStateUp787 case got > 0:788 item.State = HealthBucketStateDegraded789 default:790 item.State = HealthBucketStateDown791 }792 }793 if item.State != HealthBucketStateUnknown {794 known++795 if item.State == HealthBucketStateUp || item.State == HealthBucketStateDegraded {796 good++797 }798 }799 out.Items = append(out.Items, item)800 }801 if known > 0 {802 out.UptimeRatio = ptr(float32(good) / float32(known))803 }804 return GetNodeHealthHistory200JSONResponse{Body: out}, nil805}806