Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | INFRASTRUCTURE & AUTOMATION4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : MacLustr Command Center (MCC)8 * File : services/internal/alerts/replay.go9 * Module : services10 * Repository : https://git.spboucher.ai/maclustr-command-center11 * Project Website : https://clustr.spboucher.ai12 * Created : 2026-11-1813 * Last Modified : 2026-11-1814 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: admin@spboucher.ai23 * Technical Contact: dev@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142package alerts4344import (45 "context"46 "errors"47 "fmt"48 "sort"49 "time"5051 "github.com/google/uuid"52 "github.com/jackc/pgx/v5/pgxpool"5354 "clustr.spboucher.ai/mcc/internal/protocol"55)5657// Replay limits.58const (59 MaxReplaySpan = 31 * 24 * time.Hour60 maxReplayRows = 500_00061 maxReplayIntervals = 100062)6364// ErrNotReplayable: the rule kind or metric has no stored history to replay.65var ErrNotReplayable = errors.New("only threshold rules on stored node metrics (catalog metrics, disk.used_pct, swap.used_pct) can be replayed on history")6667type Interval struct {68 NodeID uuid.UUID69 NodeAlias string70 Label string71 StartedAt time.Time72 EndedAt *time.Time // nil: still firing at the end of the window73 Peak float6474}7576type ReplayResult struct {77 From, To time.Time78 Resolution string // raw | 1m | 1h79 NodesEvaluated int80 NodesWithoutData int81 Intervals []Interval82 Warnings []string83}8485type point struct {86 ts time.Time87 v float6488}8990type seriesKey struct {91 node uuid.UUID92 label string93}9495// Replay evaluates a draft rule on stored samples and returns when it would96// have fired. Nothing is written.97func Replay(ctx context.Context, pool *pgxpool.Pool, r Rule, from, to time.Time) (*ReplayResult, error) {98 if r.Kind != "threshold" {99 return nil, ErrNotReplayable100 }101 var metrics []string102 switch r.Metric {103 case MetricDiskPct:104 metrics = []string{"disk.used", "disk.total"}105 case MetricSwapPct:106 metrics = []string{"swap.used", "swap.total"}107 default:108 if _, ok := protocol.MetricByName(r.Metric); !ok {109 return nil, ErrNotReplayable110 }111 metrics = []string{r.Metric}112 }113 span := to.Sub(from)114 if span <= 0 || span > MaxReplaySpan {115 return nil, fmt.Errorf("the replay window must be positive and at most %d days", int(MaxReplaySpan.Hours()/24))116 }117 res := &ReplayResult{From: from, To: to, Intervals: []Interval{}, Warnings: []string{}}118 table, col, maxGap := "samples_raw", "value", 2*time.Minute119 switch {120 case span <= 6*time.Hour:121 res.Resolution = "raw"122 case span <= 8*24*time.Hour:123 res.Resolution, table, col, maxGap = "1m", "samples_1m", "avg", 3*time.Minute124 res.Warnings = append(res.Warnings, "evaluated on 1-minute averages: spikes shorter than a minute are not visible")125 default:126 res.Resolution, table, col, maxGap = "1h", "samples_1h", "avg", 2*time.Hour+time.Minute127 res.Warnings = append(res.Warnings, "evaluated on 1-hour averages: durations shorter than an hour are approximate")128 }129130 // Nodes in scope.131 rows, err := pool.Query(ctx, `select id, alias::text, tags from nodes where retired_at is null`)132 if err != nil {133 return nil, err134 }135 alias := map[uuid.UUID]string{}136 for rows.Next() {137 var id uuid.UUID138 var a string139 var tags []string140 if err := rows.Scan(&id, &a, &tags); err != nil {141 rows.Close()142 return nil, err143 }144 if r.Scope.Matches(&id, tags) {145 alias[id] = a146 }147 }148 rows.Close()149 if err := rows.Err(); err != nil {150 return nil, err151 }152 res.NodesEvaluated = len(alias)153 nodeIDs := make([]uuid.UUID, 0, len(alias))154 for id := range alias {155 nodeIDs = append(nodeIDs, id)156 }157158 load := func(metric string) (map[seriesKey][]point, error) {159 m, _ := protocol.MetricByName(metric)160 q := `select node_id, label, ts, ` + col + ` from ` + table + ` where metric_id=$1 and ts >= $2 and ts < $3 and node_id = any($4)161 and ($5 = '' or $5 = '*' or label = $5) order by node_id, label, ts limit $6`162 rows, err := pool.Query(ctx, q, m.ID, from, to, nodeIDs, r.LabelMatch, maxReplayRows+1)163 if err != nil {164 return nil, err165 }166 defer rows.Close()167 out := map[seriesKey][]point{}168 n := 0169 for rows.Next() {170 var k seriesKey171 var p point172 if err := rows.Scan(&k.node, &k.label, &p.ts, &p.v); err != nil {173 return nil, err174 }175 if n++; n > maxReplayRows {176 res.Warnings = append(res.Warnings, "too many samples in the window: the evaluation is truncated; narrow the range or the scope")177 break178 }179 out[k] = append(out[k], p)180 }181 return out, rows.Err()182 }183 data, err := load(metrics[0])184 if err != nil {185 return nil, err186 }187 if len(metrics) == 2 { // derived percentage: used / total at the same instant188 totals, err := load(metrics[1])189 if err != nil {190 return nil, err191 }192 pct := map[seriesKey][]point{}193 for k, used := range data {194 at := map[int64]float64{}195 for _, p := range totals[k] {196 at[p.ts.UnixNano()] = p.v197 }198 var last float64 // totals are reported less often than used: carry the last known total199 if len(totals[k]) > 0 {200 last = totals[k][0].v201 }202 for _, p := range used {203 if t, ok := at[p.ts.UnixNano()]; ok {204 last = t205 }206 if last > 0 {207 pct[k] = append(pct[k], point{p.ts, 100 * p.v / last})208 }209 }210 }211 data = pct212 }213214 withData := map[uuid.UUID]bool{}215 keys := make([]seriesKey, 0, len(data))216 for k := range data {217 keys = append(keys, k)218 withData[k.node] = true219 }220 res.NodesWithoutData = res.NodesEvaluated - len(withData)221 sort.Slice(keys, func(i, j int) bool {222 if alias[keys[i].node] != alias[keys[j].node] {223 return alias[keys[i].node] < alias[keys[j].node]224 }225 return keys[i].label < keys[j].label226 })227 forD, clearD := time.Duration(r.ForSeconds)*time.Second, time.Duration(r.ClearSeconds)*time.Second228 for _, k := range keys {229 var (230 pendingSince, clearSince *time.Time231 cur *Interval232 prev time.Time233 )234 closeAt := func(t time.Time) {235 cur.EndedAt = &t236 res.Intervals = append(res.Intervals, *cur)237 cur, clearSince = nil, nil238 }239 pts := data[k]240 for i, p := range pts {241 if i > 0 && p.ts.Sub(prev) > maxGap {242 // Hole in the data: nothing can be asserted across it.243 pendingSince = nil244 if cur != nil {245 since := prev246 if clearSince != nil {247 since = *clearSince248 }249 if p.ts.Sub(since) >= clearD {250 closeAt(since.Add(clearD))251 } else {252 clearSince = &since253 }254 }255 }256 prev = p.ts257 if cur == nil {258 if !r.Firing(p.v) {259 pendingSince = nil260 continue261 }262 if pendingSince == nil {263 t := p.ts264 pendingSince = &t265 }266 if p.ts.Sub(*pendingSince) >= forD {267 cur = &Interval{NodeID: k.node, NodeAlias: alias[k.node], Label: k.label, StartedAt: p.ts, Peak: p.v}268 pendingSince = nil269 }270 continue271 }272 if (r.Op == ">" || r.Op == ">=") && p.v > cur.Peak || (r.Op == "<" || r.Op == "<=") && p.v < cur.Peak {273 cur.Peak = p.v274 }275 if r.Clear(p.v) {276 if clearSince == nil {277 t := p.ts278 clearSince = &t279 }280 if p.ts.Sub(*clearSince) >= clearD {281 closeAt(p.ts)282 }283 } else {284 clearSince = nil285 }286 }287 if cur != nil {288 // Data stopped before the end of the window: the alert would have closed for lack of data.289 if end := prev.Add(clearD); to.Sub(prev) > maxGap && end.Before(to) {290 closeAt(end)291 } else {292 res.Intervals = append(res.Intervals, *cur)293 }294 }295 if len(res.Intervals) >= maxReplayIntervals {296 res.Intervals = res.Intervals[:maxReplayIntervals]297 res.Warnings = append(res.Warnings, "more than 1000 firings: the list is truncated")298 break299 }300 }301 return res, nil302}303