Repository Public
1// Package alerts evaluates alert rules from real data and maintains the2// alert state machine:3//4// pending ──(condition held for_seconds)──▶ firing ──(clear for clear_seconds)──▶ resolved5// └─(condition gone, or no data)──▶ resolved (silently)6//7// Rules of honesty:8// - a threshold or service rule is only evaluated on FRESH data. The last9// value of a node that stopped reporting neither opens nor keeps an alert:10// without data an open alert is closed after clear_seconds with the reason11// "no recent data"; the absence rule is what reports the silent node;12// - an alert suppressed by a silence or by maintenance mode is still tracked13// (suppressed_by is set) but produces no notification;14// - one open alert per (rule, node, label): the unique partial index on15// alerts.fingerprint makes concurrent evaluators converge on one row.16package alerts1718import (19 "context"20 "errors"21 "fmt"22 "slices"23 "strings"2425 "github.com/google/uuid"26 "github.com/jackc/pgx/v5"27 "github.com/jackc/pgx/v5/pgconn"2829 "clustr.spboucher.ai/mcc/internal/protocol"30)3132// Scope selects the nodes a rule or a silence applies to: exactly one form.33type Scope struct {34 All bool `json:"all,omitempty"`35 NodeIDs []uuid.UUID `json:"node_ids,omitempty"`36 Tags []string `json:"tags,omitempty"`37}3839func (s Scope) Validate() error {40 forms := 041 if s.All {42 forms++43 }44 if len(s.NodeIDs) > 0 {45 forms++46 }47 if len(s.Tags) > 0 {48 forms++49 }50 if forms != 1 {51 return errors.New(`scope must be exactly one of {"all":true}, {"node_ids":[…]} or {"tags":[…]}`)52 }53 if len(s.NodeIDs) > 200 || len(s.Tags) > 32 {54 return errors.New("scope is too large")55 }56 return nil57}5859// Matches reports whether a node is in scope. A nil node (cluster-wide60// alerts) is only matched by {"all":true}.61func (s Scope) Matches(node *uuid.UUID, tags []string) bool {62 if s.All {63 return true64 }65 if node == nil {66 return false67 }68 if slices.Contains(s.NodeIDs, *node) {69 return true70 }71 for _, t := range s.Tags {72 if slices.Contains(tags, t) {73 return true74 }75 }76 return false77}7879type Rule struct {80 ID uuid.UUID81 Name string82 Description string83 Enabled bool84 Builtin bool85 Kind string // threshold | absence | service_state86 Metric string87 LabelMatch string88 Op string // > >= < <=89 Threshold *float6490 ClearThreshold *float6491 ForSeconds int92 ClearSeconds int93 Severity string94 Scope Scope95 Notify bool96}9798// Server-side metrics usable in threshold rules besides the agent catalog.99const (100 MetricDiskPct = "disk.used_pct" // derived: disk.used / disk.total per volume101 MetricSwapPct = "swap.used_pct" // derived: swap.used / swap.total102 MetricCertDays = "mcc.agent.cert_days_left" // days before the agent certificate expires103 MetricDenied10m = "mcc.audit.denied_10m" // denied audit outcomes in the last 10 minutes, per actor and per IP104 MetricNewDevice = "mcc.auth.new_device_login" // sign-ins of an owner/admin from a device first seen at that sign-in (last hour)105 deniedWindowMin = 10106 newDeviceWindowMn = 60107)108109var serverMetrics = []string{MetricDiskPct, MetricSwapPct, MetricCertDays, MetricDenied10m, MetricNewDevice}110111// KnownMetric reports whether a threshold rule may use the metric.112func KnownMetric(name string) bool {113 if _, ok := protocol.MetricByName(name); ok {114 return true115 }116 return slices.Contains(serverMetrics, name)117}118119var serviceStates = []string{"online", "stopped", "errored", "unknown"}120121// ServiceStates returns the states a service_state rule fires on.122func (r *Rule) ServiceStates() []string {123 if strings.TrimSpace(r.Metric) == "" {124 return []string{"errored", "stopped"}125 }126 var out []string127 for _, s := range strings.Split(r.Metric, ",") {128 out = append(out, strings.TrimSpace(s))129 }130 return out131}132133func (r *Rule) Validate() error {134 if n := len(strings.TrimSpace(r.Name)); n < 1 || len(r.Name) > 120 {135 return errors.New("name must be 1 to 120 characters")136 }137 if len(r.Description) > 1000 || len(r.LabelMatch) > 64 {138 return errors.New("description or label_match is too long")139 }140 if !slices.Contains([]string{"info", "warning", "critical"}, r.Severity) {141 return errors.New("severity must be info, warning or critical")142 }143 if r.ForSeconds < 0 || r.ForSeconds > 86400 || r.ClearSeconds < 0 || r.ClearSeconds > 86400 {144 return errors.New("for_seconds and clear_seconds must be between 0 and 86400")145 }146 if err := r.Scope.Validate(); err != nil {147 return err148 }149 switch r.Kind {150 case "threshold":151 if !KnownMetric(r.Metric) {152 return fmt.Errorf("unknown metric %q", r.Metric)153 }154 if !slices.Contains([]string{">", ">=", "<", "<="}, r.Op) || r.Threshold == nil {155 return errors.New("threshold rules require metric, op and threshold")156 }157 if c := r.ClearThreshold; c != nil {158 up := r.Op == ">" || r.Op == ">="159 if up && *c > *r.Threshold || !up && *c < *r.Threshold {160 return errors.New("clear_threshold must be on the clear side of threshold")161 }162 }163 case "absence":164 if r.Metric != "" || r.Op != "" || r.Threshold != nil || r.ClearThreshold != nil {165 return errors.New("absence rules take no metric, op or threshold")166 }167 case "service_state":168 if r.Op != "" || r.Threshold != nil || r.ClearThreshold != nil {169 return errors.New("service_state rules take no op or threshold")170 }171 for _, s := range r.ServiceStates() {172 if !slices.Contains(serviceStates, s) {173 return fmt.Errorf("unknown service state %q", s)174 }175 }176 default:177 return errors.New("kind must be threshold, absence or service_state")178 }179 return nil180}181182func compare(op string, v, t float64) bool {183 switch op {184 case ">":185 return v > t186 case ">=":187 return v >= t188 case "<":189 return v < t190 case "<=":191 return v <= t192 }193 return false194}195196// Firing: the value breaches the threshold.197func (r *Rule) Firing(v float64) bool { return r.Threshold != nil && compare(r.Op, v, *r.Threshold) }198199// Clear: the value is back beyond clear_threshold (hysteresis). Between the200// two thresholds a firing alert neither clears nor re-arms.201func (r *Rule) Clear(v float64) bool {202 c := r.Threshold203 if r.ClearThreshold != nil {204 c = r.ClearThreshold205 }206 return c != nil && !compare(r.Op, v, *c)207}208209// RuleColumns is the column list read by ScanRule.210const RuleColumns = `id, name, description, enabled, builtin, kind, coalesce(metric,''), coalesce(label_match,''), coalesce(op,''), threshold, clear_threshold,211 for_seconds, clear_seconds, severity::text, scope, notify`212213// ScanRule reads RuleColumns, then any extra destinations.214func ScanRule(row pgx.Row, extra ...any) (Rule, error) {215 var r Rule216 err := row.Scan(append([]any{&r.ID, &r.Name, &r.Description, &r.Enabled, &r.Builtin, &r.Kind, &r.Metric, &r.LabelMatch, &r.Op, &r.Threshold, &r.ClearThreshold,217 &r.ForSeconds, &r.ClearSeconds, &r.Severity, &r.Scope, &r.Notify}, extra...)...)218 return r, err219}220221func f(v float64) *float64 { return &v }222223// Builtins are the default rules. They are inserted once, by name; a restart224// never overwrites, re-enables or re-creates-with-defaults a rule that exists.225var Builtins = []Rule{226 {Name: "Node offline", Description: "The node has not reported for more than 2 minutes.", Kind: "absence", ForSeconds: 120, ClearSeconds: 30, Severity: "critical"},227 {Name: "Disk volume above 90 %", Description: "A volume is more than 90 % full.", Kind: "threshold", Metric: MetricDiskPct, LabelMatch: "*", Op: ">", Threshold: f(90), ClearThreshold: f(88), ForSeconds: 300, ClearSeconds: 300, Severity: "warning"},228 {Name: "Disk volume above 97 %", Description: "A volume is more than 97 % full.", Kind: "threshold", Metric: MetricDiskPct, LabelMatch: "*", Op: ">", Threshold: f(97), ClearThreshold: f(96), ForSeconds: 120, ClearSeconds: 300, Severity: "critical"},229 {Name: "Memory pressure (warning)", Description: "Kernel memory pressure level at warning or above for 5 minutes.", Kind: "threshold", Metric: "mem.pressure.level", Op: ">=", Threshold: f(2), ForSeconds: 300, ClearSeconds: 120, Severity: "warning"},230 {Name: "Memory pressure (critical)", Description: "Kernel memory pressure level critical for 2 minutes.", Kind: "threshold", Metric: "mem.pressure.level", Op: ">=", Threshold: f(4), ForSeconds: 120, ClearSeconds: 120, Severity: "critical"},231 {Name: "Swap above 80 %", Description: "More than 80 % of the allocated swap is in use for 10 minutes.", Kind: "threshold", Metric: MetricSwapPct, Op: ">", Threshold: f(80), ClearThreshold: f(70), ForSeconds: 600, ClearSeconds: 300, Severity: "warning"},232 {Name: "CPU saturated", Description: "CPU usage above 95 % for 15 minutes.", Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: f(95), ClearThreshold: f(85), ForSeconds: 900, ClearSeconds: 300, Severity: "warning"},233 {Name: "Managed service errored", Description: "A managed service has been in the errored state for 2 minutes.", Kind: "service_state", Metric: "errored", ForSeconds: 120, ClearSeconds: 60, Severity: "critical"},234 {Name: "Agent certificate expiring", Description: "The agent certificate expires in less than 7 days (renewal is not happening).", Kind: "threshold", Metric: MetricCertDays, Op: "<", Threshold: f(7), ForSeconds: 0, ClearSeconds: 0, Severity: "warning"},235 {Name: "Packet loss to gateway", Description: "Sustained packet loss of 34 % or more to the gateway for 5 minutes.", Kind: "threshold", Metric: "net.loss", LabelMatch: "gateway", Op: ">=", Threshold: f(34), ClearThreshold: f(1), ForSeconds: 300, ClearSeconds: 120, Severity: "warning"},236 {Name: "Burst of denied requests", Description: "10 or more denied requests from one account or one address within 10 minutes.", Kind: "threshold", Metric: MetricDenied10m, LabelMatch: "*", Op: ">=", Threshold: f(10), ForSeconds: 0, ClearSeconds: 300, Severity: "warning"},237 {Name: "Sign-in from a new device", Description: "An owner or administrator signed in from a device never seen before.", Kind: "threshold", Metric: MetricNewDevice, LabelMatch: "*", Op: ">=", Threshold: f(1), ForSeconds: 0, ClearSeconds: 0, Severity: "info"},238}239240var builtinNS = uuid.MustParse("6d636300-0000-4000-8000-616c65727473")241242// SeedBuiltins inserts the missing default rules. Idempotent.243func SeedBuiltins(ctx context.Context, q interface {244 Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)245}) error {246 for _, r := range Builtins {247 r.Scope, r.Notify = Scope{All: true}, true248 if err := r.Validate(); err != nil {249 return fmt.Errorf("alerts: builtin %q: %w", r.Name, err)250 }251 if _, err := q.Exec(ctx, `insert into alert_rules(id, name, description, enabled, builtin, kind, metric, label_match, op, threshold, clear_threshold,252 for_seconds, clear_seconds, severity, scope, notify)253 values ($1,$2,$3,true,true,$4,nullif($5,''),nullif($6,''),nullif($7,''),$8,$9,$10,$11,$12::severity,$13,true)254 on conflict do nothing`, uuid.NewSHA1(builtinNS, []byte(r.Name)), r.Name, r.Description, r.Kind, r.Metric, r.LabelMatch, r.Op,255 r.Threshold, r.ClearThreshold, r.ForSeconds, r.ClearSeconds, r.Severity, r.Scope); err != nil {256 return err257 }258 }259 return nil260}261262// Save inserts or updates a validated rule.263func Save(ctx context.Context, q interface {264 Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)265}, r Rule, createdBy *uuid.UUID, insert bool) error {266 if err := r.Validate(); err != nil {267 return err268 }269 if insert {270 _, err := q.Exec(ctx, `insert into alert_rules(id, name, description, enabled, builtin, kind, metric, label_match, op, threshold, clear_threshold,271 for_seconds, clear_seconds, severity, scope, notify, created_by)272 values ($1,$2,$3,$4,false,$5,nullif($6,''),nullif($7,''),nullif($8,''),$9,$10,$11,$12,$13::severity,$14,$15,$16)`,273 r.ID, r.Name, r.Description, r.Enabled, r.Kind, r.Metric, r.LabelMatch, r.Op, r.Threshold, r.ClearThreshold, r.ForSeconds, r.ClearSeconds, r.Severity, r.Scope, r.Notify, createdBy)274 return err275 }276 _, err := q.Exec(ctx, `update alert_rules set name=$2, description=$3, enabled=$4, metric=nullif($5,''), label_match=nullif($6,''), op=nullif($7,''),277 threshold=$8, clear_threshold=$9, for_seconds=$10, clear_seconds=$11, severity=$12::severity, scope=$13, notify=$14, updated_at=now() where id=$1`,278 r.ID, r.Name, r.Description, r.Enabled, r.Metric, r.LabelMatch, r.Op, r.Threshold, r.ClearThreshold, r.ForSeconds, r.ClearSeconds, r.Severity, r.Scope, r.Notify)279 return err280}281