SPB Git

Repository Public

replay.go 8.6 KB 303 lines go

1/**2 * =============================================================================3 *                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : MacLustr Command Center (MCC)8 * File             : services/internal/alerts/replay.go9 * Module           : services10 * Repository       : https://git.spboucher.ai/maclustr-command-center11 * Project Website  : https://clustr.spboucher.ai12 * Created          : 2026-11-1813 * Last Modified    : 2026-11-1814 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package alerts4344import (45	"context"46	"errors"47	"fmt"48	"sort"49	"time"5051	"github.com/google/uuid"52	"github.com/jackc/pgx/v5/pgxpool"5354	"clustr.spboucher.ai/mcc/internal/protocol"55)5657// Replay limits.58const (59	MaxReplaySpan      = 31 * 24 * time.Hour60	maxReplayRows      = 500_00061	maxReplayIntervals = 100062)6364// ErrNotReplayable: the rule kind or metric has no stored history to replay.65var ErrNotReplayable = errors.New("only threshold rules on stored node metrics (catalog metrics, disk.used_pct, swap.used_pct) can be replayed on history")6667type Interval struct {68	NodeID    uuid.UUID69	NodeAlias string70	Label     string71	StartedAt time.Time72	EndedAt   *time.Time // nil: still firing at the end of the window73	Peak      float6474}7576type ReplayResult struct {77	From, To         time.Time78	Resolution       string // raw | 1m | 1h79	NodesEvaluated   int80	NodesWithoutData int81	Intervals        []Interval82	Warnings         []string83}8485type point struct {86	ts time.Time87	v  float6488}8990type seriesKey struct {91	node  uuid.UUID92	label string93}9495// Replay evaluates a draft rule on stored samples and returns when it would96// have fired. Nothing is written.97func Replay(ctx context.Context, pool *pgxpool.Pool, r Rule, from, to time.Time) (*ReplayResult, error) {98	if r.Kind != "threshold" {99		return nil, ErrNotReplayable100	}101	var metrics []string102	switch r.Metric {103	case MetricDiskPct:104		metrics = []string{"disk.used", "disk.total"}105	case MetricSwapPct:106		metrics = []string{"swap.used", "swap.total"}107	default:108		if _, ok := protocol.MetricByName(r.Metric); !ok {109			return nil, ErrNotReplayable110		}111		metrics = []string{r.Metric}112	}113	span := to.Sub(from)114	if span <= 0 || span > MaxReplaySpan {115		return nil, fmt.Errorf("the replay window must be positive and at most %d days", int(MaxReplaySpan.Hours()/24))116	}117	res := &ReplayResult{From: from, To: to, Intervals: []Interval{}, Warnings: []string{}}118	table, col, maxGap := "samples_raw", "value", 2*time.Minute119	switch {120	case span <= 6*time.Hour:121		res.Resolution = "raw"122	case span <= 8*24*time.Hour:123		res.Resolution, table, col, maxGap = "1m", "samples_1m", "avg", 3*time.Minute124		res.Warnings = append(res.Warnings, "evaluated on 1-minute averages: spikes shorter than a minute are not visible")125	default:126		res.Resolution, table, col, maxGap = "1h", "samples_1h", "avg", 2*time.Hour+time.Minute127		res.Warnings = append(res.Warnings, "evaluated on 1-hour averages: durations shorter than an hour are approximate")128	}129130	// Nodes in scope.131	rows, err := pool.Query(ctx, `select id, alias::text, tags from nodes where retired_at is null`)132	if err != nil {133		return nil, err134	}135	alias := map[uuid.UUID]string{}136	for rows.Next() {137		var id uuid.UUID138		var a string139		var tags []string140		if err := rows.Scan(&id, &a, &tags); err != nil {141			rows.Close()142			return nil, err143		}144		if r.Scope.Matches(&id, tags) {145			alias[id] = a146		}147	}148	rows.Close()149	if err := rows.Err(); err != nil {150		return nil, err151	}152	res.NodesEvaluated = len(alias)153	nodeIDs := make([]uuid.UUID, 0, len(alias))154	for id := range alias {155		nodeIDs = append(nodeIDs, id)156	}157158	load := func(metric string) (map[seriesKey][]point, error) {159		m, _ := protocol.MetricByName(metric)160		q := `select node_id, label, ts, ` + col + ` from ` + table + ` where metric_id=$1 and ts >= $2 and ts < $3 and node_id = any($4)161			and ($5 = '' or $5 = '*' or label = $5) order by node_id, label, ts limit $6`162		rows, err := pool.Query(ctx, q, m.ID, from, to, nodeIDs, r.LabelMatch, maxReplayRows+1)163		if err != nil {164			return nil, err165		}166		defer rows.Close()167		out := map[seriesKey][]point{}168		n := 0169		for rows.Next() {170			var k seriesKey171			var p point172			if err := rows.Scan(&k.node, &k.label, &p.ts, &p.v); err != nil {173				return nil, err174			}175			if n++; n > maxReplayRows {176				res.Warnings = append(res.Warnings, "too many samples in the window: the evaluation is truncated; narrow the range or the scope")177				break178			}179			out[k] = append(out[k], p)180		}181		return out, rows.Err()182	}183	data, err := load(metrics[0])184	if err != nil {185		return nil, err186	}187	if len(metrics) == 2 { // derived percentage: used / total at the same instant188		totals, err := load(metrics[1])189		if err != nil {190			return nil, err191		}192		pct := map[seriesKey][]point{}193		for k, used := range data {194			at := map[int64]float64{}195			for _, p := range totals[k] {196				at[p.ts.UnixNano()] = p.v197			}198			var last float64 // totals are reported less often than used: carry the last known total199			if len(totals[k]) > 0 {200				last = totals[k][0].v201			}202			for _, p := range used {203				if t, ok := at[p.ts.UnixNano()]; ok {204					last = t205				}206				if last > 0 {207					pct[k] = append(pct[k], point{p.ts, 100 * p.v / last})208				}209			}210		}211		data = pct212	}213214	withData := map[uuid.UUID]bool{}215	keys := make([]seriesKey, 0, len(data))216	for k := range data {217		keys = append(keys, k)218		withData[k.node] = true219	}220	res.NodesWithoutData = res.NodesEvaluated - len(withData)221	sort.Slice(keys, func(i, j int) bool {222		if alias[keys[i].node] != alias[keys[j].node] {223			return alias[keys[i].node] < alias[keys[j].node]224		}225		return keys[i].label < keys[j].label226	})227	forD, clearD := time.Duration(r.ForSeconds)*time.Second, time.Duration(r.ClearSeconds)*time.Second228	for _, k := range keys {229		var (230			pendingSince, clearSince *time.Time231			cur                      *Interval232			prev                     time.Time233		)234		closeAt := func(t time.Time) {235			cur.EndedAt = &t236			res.Intervals = append(res.Intervals, *cur)237			cur, clearSince = nil, nil238		}239		pts := data[k]240		for i, p := range pts {241			if i > 0 && p.ts.Sub(prev) > maxGap {242				// Hole in the data: nothing can be asserted across it.243				pendingSince = nil244				if cur != nil {245					since := prev246					if clearSince != nil {247						since = *clearSince248					}249					if p.ts.Sub(since) >= clearD {250						closeAt(since.Add(clearD))251					} else {252						clearSince = &since253					}254				}255			}256			prev = p.ts257			if cur == nil {258				if !r.Firing(p.v) {259					pendingSince = nil260					continue261				}262				if pendingSince == nil {263					t := p.ts264					pendingSince = &t265				}266				if p.ts.Sub(*pendingSince) >= forD {267					cur = &Interval{NodeID: k.node, NodeAlias: alias[k.node], Label: k.label, StartedAt: p.ts, Peak: p.v}268					pendingSince = nil269				}270				continue271			}272			if (r.Op == ">" || r.Op == ">=") && p.v > cur.Peak || (r.Op == "<" || r.Op == "<=") && p.v < cur.Peak {273				cur.Peak = p.v274			}275			if r.Clear(p.v) {276				if clearSince == nil {277					t := p.ts278					clearSince = &t279				}280				if p.ts.Sub(*clearSince) >= clearD {281					closeAt(p.ts)282				}283			} else {284				clearSince = nil285			}286		}287		if cur != nil {288			// Data stopped before the end of the window: the alert would have closed for lack of data.289			if end := prev.Add(clearD); to.Sub(prev) > maxGap && end.Before(to) {290				closeAt(end)291			} else {292				res.Intervals = append(res.Intervals, *cur)293			}294		}295		if len(res.Intervals) >= maxReplayIntervals {296			res.Intervals = res.Intervals[:maxReplayIntervals]297			res.Warnings = append(res.Warnings, "more than 1000 firings: the list is truncated")298			break299		}300	}301	return res, nil302}303
Files main
Menu

Theme