SPB Git

Repository Public

cluster_metrics.go 27.2 KB 806 lines go

1package api23import (4	"context"5	"math"6	"slices"7	"sort"8	"strings"9	"time"1011	"github.com/google/uuid"12	"github.com/jackc/pgx/v5"1314	"clustr.spboucher.ai/mcc/internal/authz"15	"clustr.spboucher.ai/mcc/internal/httpx"16)1718// ── Time-series engine ─────────────────────────────────────────────────────19//20// Samples live in three partitioned tables keyed by (node_id, metric_id,21// label, ts). Every query below fixes node_id and metric_id and bounds ts on22// both sides, so it is served by the primary key inside the partitions that23// overlap the range (the others are pruned).24//25// A series never carries more than clMaxPoints points: wider ranges are26// reduced on the server into buckets that keep the true average (weighted by27// sample count for the rollup tiers), minimum and maximum. A bucket without28// data produces no point: gaps stay gaps.2930type clTier struct {31	res      Resolution32	table    string33	step     int64         // native spacing in seconds (bucket widths are multiples of it)34	maxRange time.Duration // widest range served from this tier35	rollup   bool          // columns avg/min/max/n instead of value36}3738var (39	clTierRaw = clTier{res: ResolutionRaw, table: "samples_raw", step: 10, maxRange: 24 * time.Hour}40	clTier1m  = clTier{res: ResolutionN1m, table: "samples_1m", step: 60, maxRange: 31 * 24 * time.Hour, rollup: true}41	clTier1h  = clTier{res: ResolutionN1h, table: "samples_1h", step: 3600, maxRange: 400 * 24 * time.Hour, rollup: true}42)4344const (45	clMaxPoints      = 1500  // per series46	clMinPoints      = 120   // per series, however many series are requested47	clPointBudget    = 30000 // per response, across series48	clMaxSeriesRows  = 25000049	clAutoRawUpTo    = 2 * time.Hour50	clAuto1mUpTo     = 48 * time.Hour51	clDefaultRange   = time.Hour52	clMaxMetricRange = 400 * 24 * time.Hour53)5455// clRange validates the requested time range. `to` defaults to now and is56// never in the future; `from` defaults to one hour before `to`.57func clRange(from, to *time.Time, now time.Time) (time.Time, time.Time, error) {58	t := now59	if to != nil && to.Before(now) {60		t = to.UTC()61	}62	f := t.Add(-clDefaultRange)63	if from != nil {64		f = from.UTC()65	}66	switch {67	case !f.Before(t):68		return f, t, clInvalid("from", "must be before to (and before now)")69	case t.Sub(f) > clMaxMetricRange:70		return f, t, clInvalid("from", "the range must not exceed 400 days")71	}72	return f, t, nil73}7475// clPickTier chooses the table. auto: raw up to 2 h, 1 min up to 48 h, 1 h76// beyond (the contract's rule). An explicit resolution is honoured within the77// range that tier can serve at a bounded cost.78func clPickTier(res *Resolution, span time.Duration) (clTier, error) {79	r := ResolutionAuto80	if res != nil {81		r = *res82	}83	switch r {84	case ResolutionAuto:85		switch {86		case span <= clAutoRawUpTo:87			return clTierRaw, nil88		case span <= clAuto1mUpTo:89			return clTier1m, nil90		}91		return clTier1h, nil92	case ResolutionRaw:93		if span > clTierRaw.maxRange {94			return clTierRaw, clInvalid("resolution", "raw is limited to a 24 hour range; use 1m, 1h or auto")95		}96		return clTierRaw, nil97	case ResolutionN1m:98		if span > clTier1m.maxRange {99			return clTier1m, clInvalid("resolution", "1m is limited to a 31 day range; use 1h or auto")100		}101		return clTier1m, nil102	case ResolutionN1h:103		return clTier1h, nil104	}105	return clTierRaw, clInvalid("resolution", "must be auto, raw, 1m or 1h")106}107108// clBucketWidth returns the bucket width in seconds: the tier's native step109// when the range fits in maxPoints, else the smallest multiple of the step110// that does.111func clBucketWidth(span time.Duration, step int64, maxPoints int) int64 {112	secs := int64(math.Ceil(span.Seconds()))113	if secs <= step*int64(maxPoints) {114		return step115	}116	w := (secs + int64(maxPoints) - 1) / int64(maxPoints)117	return (w + step - 1) / step * step118}119120type clSeriesKey struct {121	node   uuid.UUID122	metric int16123	label  string124}125126type clSeriesData struct {127	key    clSeriesKey128	points MetricPoints129}130131// clSeries reads the series of the given nodes and metrics over [from, to).132// When width equals the tier step the stored points are returned as they are;133// otherwise they are reduced into buckets of `width` seconds aligned on the134// epoch, stamped with the bucket start.135func (h *clusterHandlers) clSeries(ctx context.Context, tier clTier, nodes []uuid.UUID, metrics []int16, from, to time.Time, width int64, label string) ([]clSeriesData, error) {136	var sel string137	switch {138	case width == tier.step && !tier.rollup:139		sel = `select node_id, metric_id, label, extract(epoch from ts)::float8, value, value, value from ` + tier.table +140			` where node_id = any($1) and metric_id = any($2) and ts >= $3 and ts < $4 and ($5 = '' or label = $5 or label = '')141			order by node_id, metric_id, label, ts limit $6`142	case width == tier.step:143		sel = `select node_id, metric_id, label, extract(epoch from ts)::float8, avg, min, max from ` + tier.table +144			` where node_id = any($1) and metric_id = any($2) and ts >= $3 and ts < $4 and ($5 = '' or label = $5 or label = '')145			order by node_id, metric_id, label, ts limit $6`146	case !tier.rollup:147		sel = `select node_id, metric_id, label, (floor(extract(epoch from ts) / $7::float8) * $7::float8)::float8 as b, avg(value), min(value), max(value)148			from ` + tier.table + ` where node_id = any($1) and metric_id = any($2) and ts >= $3 and ts < $4 and ($5 = '' or label = $5 or label = '')149			group by node_id, metric_id, label, b order by node_id, metric_id, label, b limit $6`150	default:151		sel = `select node_id, metric_id, label, (floor(extract(epoch from ts) / $7::float8) * $7::float8)::float8 as b,152				sum(avg * n) / nullif(sum(n), 0), min(min), max(max)153			from ` + tier.table + ` where node_id = any($1) and metric_id = any($2) and ts >= $3 and ts < $4 and ($5 = '' or label = $5 or label = '')154			group by node_id, metric_id, label, b order by node_id, metric_id, label, b limit $6`155	}156	args := []any{nodes, metrics, from, to, label, clMaxSeriesRows + 1}157	if width != tier.step {158		args = append(args, float64(width))159	}160	rows, err := h.Pool.Query(ctx, sel, args...)161	if err != nil {162		return nil, err163	}164	defer rows.Close()165	var out []clSeriesData166	n := 0167	for rows.Next() {168		var k clSeriesKey169		var ts float64170		var avg, min, max *float64171		if err := rows.Scan(&k.node, &k.metric, &k.label, &ts, &avg, &min, &max); err != nil {172			return nil, err173		}174		if n++; n > clMaxSeriesRows {175			return nil, clInvalid("from", "the query returns too many points; narrow the range, the metrics or the label")176		}177		if avg == nil || min == nil || max == nil {178			continue // a rollup row without samples carries no information179		}180		if len(out) == 0 || out[len(out)-1].key != k {181			out = append(out, clSeriesData{key: k, points: MetricPoints{}})182		}183		s := &out[len(out)-1]184		s.points = append(s.points, []float64{math.Round(ts * 1000), *avg, *min, *max})185	}186	return out, rows.Err()187}188189// clMetricNow is the CURRENT availability of a metric on a node (for one190// label, or for any of its labels when label is empty).191func clMetricNow(m clMetric, label string, fresh Freshness, reported bool, values map[string]any, avail map[string]string) Availability {192	if !reported {193		return AvailabilityOffline194	}195	has := false196	switch {197	case !m.Labeled:198		_, has = clNumber(values, m.Name)199		label = ""200	case label != "":201		_, has = clNumber(values, clValueKey(m.Name, label))202	default:203		has = len(clLabels(values, m.Name)) > 0204	}205	if has {206		return clAvail(fresh)207	}208	if a, ok := clDeclared(avail, m.Name, label); ok {209		return a210	}211	if fresh == FreshnessOffline {212		return AvailabilityOffline213	}214	return AvailabilityUnavailable215}216217func clLabelParam(l *string) (string, error) {218	if l == nil {219		return "", nil220	}221	if len(*l) > 64 || !clPrintable(*l) {222		return "", clInvalid("label", "must be at most 64 printable characters")223	}224	return *l, nil225}226227// clMetricNames validates the requested metric names against the catalog.228func clMetricNames(cat *clCatalog, field string, names []string, max int) ([]clMetric, error) {229	var out []clMetric230	var unknown []string231	for _, raw := range names {232		name := strings.TrimSpace(raw)233		if name == "" {234			continue235		}236		m, ok := cat.byName[name]237		switch {238		case !ok:239			if len(name) > 64 {240				name = name[:64]241			}242			unknown = append(unknown, name)243		case !slices.ContainsFunc(out, func(x clMetric) bool { return x.ID == m.ID }):244			out = append(out, m)245		}246	}247	if len(unknown) > 0 {248		return nil, clInvalid(field, "unknown metric (see GET /metrics/catalog)").With("unknown", unknown)249	}250	if len(out) == 0 {251		return nil, clInvalid(field, "at least one metric is required")252	}253	if len(out) > max {254		return nil, clInvalid(field, "at most "+itoaInt(max)+" metrics")255	}256	return out, nil257}258259func (h *clusterHandlers) GetNodeMetrics(ctx context.Context, req GetNodeMetricsRequestObject) (GetNodeMetricsResponseObject, error) {260	p := req.Params261	cat, err := h.clCatalogFor(ctx)262	if err != nil {263		return nil, err264	}265	metrics, err := clMetricNames(cat, "metrics", p.Metrics, 12)266	if err != nil {267		return nil, err268	}269	label, err := clLabelParam(p.Label)270	if err != nil {271		return nil, err272	}273	now := time.Now().UTC()274	from, to, err := clRange(p.From, p.To, now)275	if err != nil {276		return nil, err277	}278	tier, err := clPickTier(p.Resolution, to.Sub(from))279	if err != nil {280		return nil, err281	}282	acc, err := h.clAccessFor(ctx)283	if err != nil {284		return nil, err285	}286	var valuesRaw, availRaw []byte287	n, err := h.readableNode(ctx, acc, now, req.Id, `, s.values, s.availability`, &valuesRaw, &availRaw)288	if err != nil {289		return nil, err290	}291	values, avail := clValues(valuesRaw), clStringMap(availRaw)292	fresh := h.clFresh(now, n.Seen)293294	// Point budget: the more series, the fewer points each.295	estimate := 0296	mids := make([]int16, 0, len(metrics))297	byID := map[int16]clMetric{}298	for _, m := range metrics {299		mids = append(mids, m.ID)300		byID[m.ID] = m301		if k := len(clLabels(values, m.Name)); m.Labeled && label == "" && k > 1 {302			estimate += k303		} else {304			estimate++305		}306	}307	width := clBucketWidth(to.Sub(from), tier.step, max(clMinPoints, min(clMaxPoints, clPointBudget/estimate)))308	data, err := h.clSeries(ctx, tier, []uuid.UUID{n.ID}, mids, from, to, width, label)309	if err != nil {310		return nil, err311	}312	out := MetricSeriesResponse{Resolution: tier.res, From: from, To: to, Series: make([]MetricSeries, 0, len(data)), Availability: map[string]Availability{}}313	for _, m := range metrics { // series in the order the metrics were requested314		for _, s := range data {315			if s.key.metric == m.ID {316				out.Series = append(out.Series, MetricSeries{Metric: m.Name, Unit: m.Unit, Label: s.key.label, Points: s.points})317			}318		}319		out.Availability[m.Name] = clMetricNow(m, label, fresh, n.ObservedAt != nil, values, avail)320	}321	return GetNodeMetrics200JSONResponse{Body: out}, nil322}323324func (h *clusterHandlers) CompareMetrics(ctx context.Context, req CompareMetricsRequestObject) (CompareMetricsResponseObject, error) {325	p := req.Params326	cat, err := h.clCatalogFor(ctx)327	if err != nil {328		return nil, err329	}330	ms, err := clMetricNames(cat, "metric", []string{p.Metric}, 1)331	if err != nil {332		return nil, err333	}334	m := ms[0]335	var nodeIDs []uuid.UUID336	for _, id := range p.NodeIds {337		if !slices.Contains(nodeIDs, id) {338			nodeIDs = append(nodeIDs, id)339		}340	}341	if len(nodeIDs) < 1 || len(nodeIDs) > 12 {342		return nil, clInvalid("node_ids", "give 1 to 12 node ids")343	}344	label, err := clLabelParam(p.Label)345	if err != nil {346		return nil, err347	}348	now := time.Now().UTC()349	from, to, err := clRange(p.From, p.To, now)350	if err != nil {351		return nil, err352	}353	tier, err := clPickTier(p.Resolution, to.Sub(from))354	if err != nil {355		return nil, err356	}357	acc, err := h.clAccessFor(ctx)358	if err != nil {359		return nil, err360	}361	nodes, err := h.netNodes(ctx, now, ` and n.id = any($4)`, []any{nodeIDs}, "")362	if err != nil {363		return nil, err364	}365	byID := map[uuid.UUID]*clNetNode{}366	estimate := 0367	for i := range nodes {368		x := &nodes[i]369		if !acc.can(authz.NodesRead, x.n.facets()...) {370			continue371		}372		byID[x.n.ID] = x373		if k := len(clLabels(x.values, m.Name)); m.Labeled && label == "" && k > 1 {374			estimate += k375		} else {376			estimate++377		}378	}379	for _, id := range nodeIDs {380		if byID[id] == nil {381			return nil, httpx.ErrNotFound.With("node_id", id.String()) // unknown, retired or hidden382		}383	}384	width := clBucketWidth(to.Sub(from), tier.step, max(clMinPoints, min(clMaxPoints, clPointBudget/estimate)))385	data, err := h.clSeries(ctx, tier, nodeIDs, []int16{m.ID}, from, to, width, label)386	if err != nil {387		return nil, err388	}389	out := MetricCompareResponse{Metric: m.Name, Unit: m.Unit, Resolution: tier.res, From: from, To: to,390		Series: make([]MetricSeries, 0, len(data)), Availability: map[string]Availability{}}391	for _, id := range nodeIDs { // series in the order the nodes were requested392		x := byID[id]393		for _, s := range data {394			if s.key.node == id {395				out.Series = append(out.Series, MetricSeries{Metric: m.Name, Unit: m.Unit, Label: s.key.label, Points: s.points,396					NodeId: &x.n.ID, NodeAlias: &x.n.Alias})397			}398		}399		out.Availability[id.String()] = clMetricNow(m, label, h.clFresh(now, x.n.Seen), x.n.ObservedAt != nil, x.values, x.avail)400	}401	return CompareMetrics200JSONResponse{Body: out}, nil402}403404func (h *clusterHandlers) GetMetricCatalog(ctx context.Context, _ GetMetricCatalogRequestObject) (GetMetricCatalogResponseObject, error) {405	cat, err := h.clCatalogFor(ctx)406	if err != nil {407		return nil, err408	}409	out := MetricCatalog{Items: make([]MetricCatalogEntry, 0, len(cat.list))}410	for _, m := range cat.list {411		kind := MetricCatalogEntryKindGauge412		if m.Kind == string(MetricCatalogEntryKindCounterRate) {413			kind = MetricCatalogEntryKindCounterRate414		}415		out.Items = append(out.Items, MetricCatalogEntry{Name: m.Name, Unit: m.Unit, Kind: kind, IntervalS: m.IntervalS, Labeled: m.Labeled,416			Source: m.Source, Description: m.Description})417	}418	return GetMetricCatalog200JSONResponse{Body: out}, nil419}420421// ── Cluster trends ─────────────────────────────────────────────────────────422423type clTrendPlan struct {424	span  time.Duration425	tier  clTier426	width int64427}428429// The contract fixes the resolution of each range.430var clTrendPlans = map[TimeRange]clTrendPlan{431	TimeRangeN1h:  {time.Hour, clTierRaw, 10},432	TimeRangeN24h: {24 * time.Hour, clTier1m, 60},433	TimeRangeN7d:  {7 * 24 * time.Hour, clTier1h, 3600},434	TimeRangeN30d: {30 * 24 * time.Hour, clTier1h, 3600},435}436437// clAdditive says how nodes (and the labels of one node) combine: quantities438// that add up are summed, everything else is averaged.439func clAdditive(unit string) bool {440	switch unit {441	case "bytes", "bytes_per_s", "watts":442		return true443	}444	return false445}446447const clMaxTrendNodes = 1000448449// GetClusterTrends aggregates one metric across nodes. For each bucket a node450// contributes only if it has data in that bucket (no carry-forward). With451// `mean`, a point is [mean of node averages, lowest node minimum, highest452// node maximum]; with `sum`, [sum of averages, sum of minima, sum of maxima]453// (bounds of the total). `coverage` is the number of nodes that contributed454// to the series at least once in the range.455func (h *clusterHandlers) GetClusterTrends(ctx context.Context, req GetClusterTrendsRequestObject) (GetClusterTrendsResponseObject, error) {456	p := req.Params457	cat, err := h.clCatalogFor(ctx)458	if err != nil {459		return nil, err460	}461	ms, err := clMetricNames(cat, "metric", []string{p.Metric}, 1)462	if err != nil {463		return nil, err464	}465	m := ms[0]466	rng := TimeRangeN24h467	if p.Range != nil {468		rng = *p.Range469	}470	plan, ok := clTrendPlans[rng]471	if !ok {472		return nil, clInvalid("range", "must be 1h, 24h, 7d or 30d")473	}474	groupBy := TrendGroupByCluster475	if p.GroupBy != nil {476		groupBy = *p.GroupBy477	}478	if !slices.Contains([]TrendGroupBy{TrendGroupByCluster, TrendGroupBySite, TrendGroupByRole, TrendGroupByTag}, groupBy) {479		return nil, clInvalid("group_by", "must be cluster, site, role or tag")480	}481	acc, err := h.clAccessFor(ctx)482	if err != nil {483		return nil, err484	}485	now := time.Now().UTC()486	from := time.Unix((now.Add(-plan.span).Unix()/plan.width)*plan.width, 0).UTC()487488	rows, err := h.Pool.Query(ctx, `select id, site, role, tags from nodes where retired_at is null order by id limit $1`, clMaxTrendNodes)489	if err != nil {490		return nil, err491	}492	type member struct {493		id         uuid.UUID494		site, role string495		tags       []string496	}497	members, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (member, error) {498		var x member499		err := r.Scan(&x.id, &x.site, &x.role, &x.tags)500		return x, err501	})502	if err != nil {503		return nil, err504	}505	groupsOf := map[uuid.UUID][]string{}506	var nodeIDs []uuid.UUID507	filter := acc.scoped(authz.NodesRead)508	for _, x := range members {509		if filter && !acc.can(authz.NodesRead, clNodeFacets(x.id, x.site, x.tags)...) {510			continue511		}512		nodeIDs = append(nodeIDs, x.id)513		switch groupBy {514		case TrendGroupBySite:515			groupsOf[x.id] = []string{x.site}516		case TrendGroupByRole:517			groupsOf[x.id] = []string{x.role}518		case TrendGroupByTag:519			groupsOf[x.id] = x.tags520		default:521			groupsOf[x.id] = []string{"cluster"}522		}523	}524	agg := TrendsResponseAggregationMean525	if clAdditive(m.Unit) {526		agg = TrendsResponseAggregationSum527	}528	out := TrendsResponse{Metric: m.Name, Unit: m.Unit, Range: rng, GroupBy: groupBy, Resolution: plan.tier.res, Aggregation: agg,529		Series: []TrendSeries{}, GeneratedAt: now}530531	// One row per node and bucket: the labels of a node (interfaces, mount points) are combined first.532	inner := `select node_id, label, (floor(extract(epoch from ts) / $5::float8) * $5::float8)::float8 as b, avg(value) as a, min(value) as mn, max(value) as mx533		from samples_raw where node_id = any($1) and metric_id = $2 and ts >= $3 and ts < $4 group by node_id, label, b`534	if plan.tier.rollup {535		inner = `select node_id, label, (floor(extract(epoch from ts) / $5::float8) * $5::float8)::float8 as b,536				sum(avg * n) / nullif(sum(n), 0) as a, min(min) as mn, max(max) as mx537			from ` + plan.tier.table + ` where node_id = any($1) and metric_id = $2 and ts >= $3 and ts < $4 group by node_id, label, b`538	}539	outer := `select x.node_id, x.b, avg(x.a), min(x.mn), max(x.mx) from (` + inner + `) x where x.a is not null group by x.node_id, x.b`540	if agg == TrendsResponseAggregationSum {541		outer = `select x.node_id, x.b, sum(x.a), sum(x.mn), sum(x.mx) from (` + inner + `) x where x.a is not null group by x.node_id, x.b`542	}543	type cell struct {544		n            int545		avg, lo, hi  float64546		sumLo, sumHi float64547	}548	type group struct {549		buckets map[float64]*cell550		nodes   map[uuid.UUID]struct{}551	}552	groups := map[string]*group{}553	if len(nodeIDs) > 0 {554		rs, err := h.Pool.Query(ctx, outer, nodeIDs, m.ID, from, now, float64(plan.width))555		if err != nil {556			return nil, err557		}558		defer rs.Close()559		for rs.Next() {560			var id uuid.UUID561			var b, a, lo, hi float64562			if err := rs.Scan(&id, &b, &a, &lo, &hi); err != nil {563				return nil, err564			}565			for _, name := range groupsOf[id] {566				g := groups[name]567				if g == nil {568					g = &group{buckets: map[float64]*cell{}, nodes: map[uuid.UUID]struct{}{}}569					groups[name] = g570				}571				g.nodes[id] = struct{}{}572				c := g.buckets[b]573				if c == nil {574					c = &cell{lo: lo, hi: hi}575					g.buckets[b] = c576				}577				c.n++578				c.avg += a579				c.sumLo += lo580				c.sumHi += hi581				c.lo, c.hi = math.Min(c.lo, lo), math.Max(c.hi, hi)582			}583		}584		if err := rs.Err(); err != nil {585			return nil, err586		}587	}588	if groupBy == TrendGroupByCluster && groups["cluster"] == nil {589		groups["cluster"] = &group{} // an empty series states that no node contributed590	}591	names := make([]string, 0, len(groups))592	for name := range groups {593		names = append(names, name)594	}595	sort.Strings(names)596	for _, name := range names {597		g := groups[name]598		ts := make([]float64, 0, len(g.buckets))599		for b := range g.buckets {600			ts = append(ts, b)601		}602		sort.Float64s(ts)603		s := TrendSeries{Group: name, Coverage: len(g.nodes), Points: make(MetricPoints, 0, len(ts))}604		for _, b := range ts {605			c := g.buckets[b]606			if agg == TrendsResponseAggregationSum {607				s.Points = append(s.Points, []float64{b * 1000, c.avg, c.sumLo, c.sumHi})608			} else {609				s.Points = append(s.Points, []float64{b * 1000, c.avg / float64(c.n), c.lo, c.hi})610			}611		}612		out.Series = append(out.Series, s)613	}614	return GetClusterTrends200JSONResponse{Body: out}, nil615}616617// ── Health history ─────────────────────────────────────────────────────────618619type clHealthPlan struct {620	span   time.Duration621	bucket int64 // seconds622	rollup bool  // read samples_1m (plus raw for the most recent minutes)623}624625var clHealthPlans = map[TimeRange]clHealthPlan{626	TimeRangeN1h:  {time.Hour, 300, false},627	TimeRangeN24h: {24 * time.Hour, 1800, false},628	TimeRangeN7d:  {7 * 24 * time.Hour, 3 * 3600, true},629	TimeRangeN30d: {30 * 24 * time.Hour, 12 * 3600, true},630}631632const (633	clHealthSlot = 60               // seconds: a node is "heard" in a slot when a report landed in it634	clHealthTail = 15 * time.Minute // most recent part always read from raw (rollups lag)635)636637// Metrics every agent reports at each tick; a slot is covered when any of them has a sample.638var clHealthProbes = []string{"cpu.usage", "load.1", "mem.used"}639640// GetNodeHealthHistory derives reachability from the reports actually641// received: time is cut into one-minute slots, a slot is covered when at642// least one sample of a per-tick metric falls in it, and a bucket is `up`643// when all its (elapsed) slots are covered, `degraded` when some are, `down`644// when none is. Buckets before the node's first sample are `unknown`; buckets645// overlapping an applied maintenance window are `maintenance`.646// uptime_ratio = (up + degraded) / buckets that are not unknown.647func (h *clusterHandlers) GetNodeHealthHistory(ctx context.Context, req GetNodeHealthHistoryRequestObject) (GetNodeHealthHistoryResponseObject, error) {648	rng := TimeRangeN24h649	if req.Params.Range != nil {650		rng = *req.Params.Range651	}652	plan, ok := clHealthPlans[rng]653	if !ok {654		return nil, clInvalid("range", "must be 1h, 24h, 7d or 30d")655	}656	cat, err := h.clCatalogFor(ctx)657	if err != nil {658		return nil, err659	}660	acc, err := h.clAccessFor(ctx)661	if err != nil {662		return nil, err663	}664	now := time.Now().UTC()665	n, err := h.readableNode(ctx, acc, now, req.Id, "")666	if err != nil {667		return nil, err668	}669	var probes []int16670	for _, name := range clHealthProbes {671		if m, ok := cat.byName[name]; ok {672			probes = append(probes, m.ID)673		}674	}675	count := int64(plan.span.Seconds()) / plan.bucket676	lastStart := now.Unix() / plan.bucket * plan.bucket677	start := time.Unix(lastStart-(count-1)*plan.bucket, 0).UTC()678	out := HealthHistory{NodeId: n.ID, Range: rng, BucketSeconds: int(plan.bucket), Items: make([]HealthBucket, 0, count)}679680	// First sample ever kept for this node, across tiers (each lookup is one index probe per metric).681	var first *time.Time682	err = h.Pool.QueryRow(ctx, `select min(t.ts) from unnest($2::smallint[]) as mid, lateral (683			(select ts from samples_1h where node_id = $1 and metric_id = mid and label = '' order by ts limit 1)684			union all (select ts from samples_1m where node_id = $1 and metric_id = mid and label = '' order by ts limit 1)685			union all (select ts from samples_raw where node_id = $1 and metric_id = mid and label = '' order by ts limit 1)) t`,686		n.ID, probes).Scan(&first)687	if err != nil {688		return nil, err689	}690691	covered := map[int64]int64{} // bucket start -> covered slots692	if first != nil {693		const slot = `floor(extract(epoch from ts) / 60)::bigint`694		sql := `select (u.slot * 60 / $5::bigint) * $5::bigint, count(*) from (695				select distinct ` + slot + ` as slot from samples_raw696				where node_id = $1 and metric_id = any($2) and label = '' and ts >= $3 and ts < $4) u group by 1`697		args := []any{n.ID, probes, start, now, plan.bucket}698		if cut := now.Add(-clHealthTail).Truncate(time.Minute); plan.rollup && cut.After(start) {699			sql = `select (u.slot * 60 / $6::bigint) * $6::bigint, count(*) from (700					select distinct ` + slot + ` as slot from samples_1m701					where node_id = $1 and metric_id = any($2) and label = '' and ts >= $3 and ts < $4 and n > 0702					union703					select distinct ` + slot + ` from samples_raw704					where node_id = $1 and metric_id = any($2) and label = '' and ts >= $4 and ts < $5) u group by 1`705			args = []any{n.ID, probes, start, cut, now, plan.bucket}706		}707		rows, err := h.Pool.Query(ctx, sql, args...)708		if err != nil {709			return nil, err710		}711		var b, c int64712		if _, err := pgx.ForEachRow(rows, []any{&b, &c}, func() error { covered[b] = c; return nil }); err != nil {713			return nil, err714		}715	}716717	// Maintenance windows that were applied to this node, and its alerts, over the range.718	type span struct{ from, to time.Time }719	var maint, firing []span720	rows, err := h.Pool.Query(ctx, `select starts_at, ends_at from maintenance_windows721		where status in ('in_progress','completed') and set_maintenance_mode and recurrence = 'none' and starts_at < $3 and ends_at > $2722			and (scope->'node_ids' ? $1 or scope->'all' = 'true'::jsonb)`, n.ID.String(), start, now)723	if err != nil {724		return nil, err725	}726	if maint, err = pgx.CollectRows(rows, func(r pgx.CollectableRow) (span, error) {727		var s span728		err := r.Scan(&s.from, &s.to)729		return s, err730	}); err != nil {731		return nil, err732	}733	if acc.can(authz.AlertsRead) {734		rows, err := h.Pool.Query(ctx, `select fired_at, resolved_at from alerts735			where node_id = $1 and fired_at is not null and fired_at < $3 and (resolved_at is null or resolved_at > $2) limit 5000`, n.ID, start, now)736		if err != nil {737			return nil, err738		}739		if firing, err = pgx.CollectRows(rows, func(r pgx.CollectableRow) (span, error) {740			var s span741			var to *time.Time742			err := r.Scan(&s.from, &to)743			s.to = now.Add(time.Hour) // still firing744			if to != nil {745				s.to = *to746			}747			return s, err748		}); err != nil {749			return nil, err750		}751	}752753	known, good := 0, 0754	for i := int64(0); i < count; i++ {755		bs := start.Add(time.Duration(i*plan.bucket) * time.Second)756		be := bs.Add(time.Duration(plan.bucket) * time.Second)757		item := HealthBucket{Start: bs, End: be, State: HealthBucketStateUnknown}758		for _, f := range firing {759			if f.from.Before(be) && f.to.After(bs) {760				item.AlertsFiring++761			}762		}763		inMaint := n.Maintenance && !now.Before(bs) && now.Before(be)764		for _, w := range maint {765			inMaint = inMaint || (w.from.Before(be) && w.to.After(bs))766		}767		switch {768		case first == nil || !be.After(*first):769			// before the first report: nothing is known770		case inMaint:771			item.State = HealthBucketStateMaintenance772		default:773			// Slots that should have been heard: from the first report (if it falls in this bucket) to the last complete minute.774			lo, hi := bs.Unix()/clHealthSlot, be.Unix()/clHealthSlot775			if f := first.Unix() / clHealthSlot; f > lo {776				lo = f777			}778			if nowSlot := now.Unix() / clHealthSlot; nowSlot < hi {779				hi = nowSlot780			}781			expected, got := hi-lo, covered[bs.Unix()]782			switch {783			case expected <= 0 && got == 0:784				// the bucket has only just begun785			case got >= expected:786				item.State = HealthBucketStateUp787			case got > 0:788				item.State = HealthBucketStateDegraded789			default:790				item.State = HealthBucketStateDown791			}792		}793		if item.State != HealthBucketStateUnknown {794			known++795			if item.State == HealthBucketStateUp || item.State == HealthBucketStateDegraded {796				good++797			}798		}799		out.Items = append(out.Items, item)800	}801	if known > 0 {802		out.UptimeRatio = ptr(float32(good) / float32(known))803	}804	return GetNodeHealthHistory200JSONResponse{Body: out}, nil805}806
Files main
Menu

Theme