SPB Git

Repository Public

rules.go 12.2 KB 281 lines go

1// Package alerts evaluates alert rules from real data and maintains the2// alert state machine:3//4//	pending ──(condition held for_seconds)──▶ firing ──(clear for clear_seconds)──▶ resolved5//	   └─(condition gone, or no data)──▶ resolved (silently)6//7// Rules of honesty:8//   - a threshold or service rule is only evaluated on FRESH data. The last9//     value of a node that stopped reporting neither opens nor keeps an alert:10//     without data an open alert is closed after clear_seconds with the reason11//     "no recent data"; the absence rule is what reports the silent node;12//   - an alert suppressed by a silence or by maintenance mode is still tracked13//     (suppressed_by is set) but produces no notification;14//   - one open alert per (rule, node, label): the unique partial index on15//     alerts.fingerprint makes concurrent evaluators converge on one row.16package alerts1718import (19	"context"20	"errors"21	"fmt"22	"slices"23	"strings"2425	"github.com/google/uuid"26	"github.com/jackc/pgx/v5"27	"github.com/jackc/pgx/v5/pgconn"2829	"clustr.spboucher.ai/mcc/internal/protocol"30)3132// Scope selects the nodes a rule or a silence applies to: exactly one form.33type Scope struct {34	All     bool        `json:"all,omitempty"`35	NodeIDs []uuid.UUID `json:"node_ids,omitempty"`36	Tags    []string    `json:"tags,omitempty"`37}3839func (s Scope) Validate() error {40	forms := 041	if s.All {42		forms++43	}44	if len(s.NodeIDs) > 0 {45		forms++46	}47	if len(s.Tags) > 0 {48		forms++49	}50	if forms != 1 {51		return errors.New(`scope must be exactly one of {"all":true}, {"node_ids":[…]} or {"tags":[…]}`)52	}53	if len(s.NodeIDs) > 200 || len(s.Tags) > 32 {54		return errors.New("scope is too large")55	}56	return nil57}5859// Matches reports whether a node is in scope. A nil node (cluster-wide60// alerts) is only matched by {"all":true}.61func (s Scope) Matches(node *uuid.UUID, tags []string) bool {62	if s.All {63		return true64	}65	if node == nil {66		return false67	}68	if slices.Contains(s.NodeIDs, *node) {69		return true70	}71	for _, t := range s.Tags {72		if slices.Contains(tags, t) {73			return true74		}75	}76	return false77}7879type Rule struct {80	ID             uuid.UUID81	Name           string82	Description    string83	Enabled        bool84	Builtin        bool85	Kind           string // threshold | absence | service_state86	Metric         string87	LabelMatch     string88	Op             string // > >= < <=89	Threshold      *float6490	ClearThreshold *float6491	ForSeconds     int92	ClearSeconds   int93	Severity       string94	Scope          Scope95	Notify         bool96}9798// Server-side metrics usable in threshold rules besides the agent catalog.99const (100	MetricDiskPct     = "disk.used_pct"             // derived: disk.used / disk.total per volume101	MetricSwapPct     = "swap.used_pct"             // derived: swap.used / swap.total102	MetricCertDays    = "mcc.agent.cert_days_left"  // days before the agent certificate expires103	MetricDenied10m   = "mcc.audit.denied_10m"      // denied audit outcomes in the last 10 minutes, per actor and per IP104	MetricNewDevice   = "mcc.auth.new_device_login" // sign-ins of an owner/admin from a device first seen at that sign-in (last hour)105	deniedWindowMin   = 10106	newDeviceWindowMn = 60107)108109var serverMetrics = []string{MetricDiskPct, MetricSwapPct, MetricCertDays, MetricDenied10m, MetricNewDevice}110111// KnownMetric reports whether a threshold rule may use the metric.112func KnownMetric(name string) bool {113	if _, ok := protocol.MetricByName(name); ok {114		return true115	}116	return slices.Contains(serverMetrics, name)117}118119var serviceStates = []string{"online", "stopped", "errored", "unknown"}120121// ServiceStates returns the states a service_state rule fires on.122func (r *Rule) ServiceStates() []string {123	if strings.TrimSpace(r.Metric) == "" {124		return []string{"errored", "stopped"}125	}126	var out []string127	for _, s := range strings.Split(r.Metric, ",") {128		out = append(out, strings.TrimSpace(s))129	}130	return out131}132133func (r *Rule) Validate() error {134	if n := len(strings.TrimSpace(r.Name)); n < 1 || len(r.Name) > 120 {135		return errors.New("name must be 1 to 120 characters")136	}137	if len(r.Description) > 1000 || len(r.LabelMatch) > 64 {138		return errors.New("description or label_match is too long")139	}140	if !slices.Contains([]string{"info", "warning", "critical"}, r.Severity) {141		return errors.New("severity must be info, warning or critical")142	}143	if r.ForSeconds < 0 || r.ForSeconds > 86400 || r.ClearSeconds < 0 || r.ClearSeconds > 86400 {144		return errors.New("for_seconds and clear_seconds must be between 0 and 86400")145	}146	if err := r.Scope.Validate(); err != nil {147		return err148	}149	switch r.Kind {150	case "threshold":151		if !KnownMetric(r.Metric) {152			return fmt.Errorf("unknown metric %q", r.Metric)153		}154		if !slices.Contains([]string{">", ">=", "<", "<="}, r.Op) || r.Threshold == nil {155			return errors.New("threshold rules require metric, op and threshold")156		}157		if c := r.ClearThreshold; c != nil {158			up := r.Op == ">" || r.Op == ">="159			if up && *c > *r.Threshold || !up && *c < *r.Threshold {160				return errors.New("clear_threshold must be on the clear side of threshold")161			}162		}163	case "absence":164		if r.Metric != "" || r.Op != "" || r.Threshold != nil || r.ClearThreshold != nil {165			return errors.New("absence rules take no metric, op or threshold")166		}167	case "service_state":168		if r.Op != "" || r.Threshold != nil || r.ClearThreshold != nil {169			return errors.New("service_state rules take no op or threshold")170		}171		for _, s := range r.ServiceStates() {172			if !slices.Contains(serviceStates, s) {173				return fmt.Errorf("unknown service state %q", s)174			}175		}176	default:177		return errors.New("kind must be threshold, absence or service_state")178	}179	return nil180}181182func compare(op string, v, t float64) bool {183	switch op {184	case ">":185		return v > t186	case ">=":187		return v >= t188	case "<":189		return v < t190	case "<=":191		return v <= t192	}193	return false194}195196// Firing: the value breaches the threshold.197func (r *Rule) Firing(v float64) bool { return r.Threshold != nil && compare(r.Op, v, *r.Threshold) }198199// Clear: the value is back beyond clear_threshold (hysteresis). Between the200// two thresholds a firing alert neither clears nor re-arms.201func (r *Rule) Clear(v float64) bool {202	c := r.Threshold203	if r.ClearThreshold != nil {204		c = r.ClearThreshold205	}206	return c != nil && !compare(r.Op, v, *c)207}208209// RuleColumns is the column list read by ScanRule.210const RuleColumns = `id, name, description, enabled, builtin, kind, coalesce(metric,''), coalesce(label_match,''), coalesce(op,''), threshold, clear_threshold,211	for_seconds, clear_seconds, severity::text, scope, notify`212213// ScanRule reads RuleColumns, then any extra destinations.214func ScanRule(row pgx.Row, extra ...any) (Rule, error) {215	var r Rule216	err := row.Scan(append([]any{&r.ID, &r.Name, &r.Description, &r.Enabled, &r.Builtin, &r.Kind, &r.Metric, &r.LabelMatch, &r.Op, &r.Threshold, &r.ClearThreshold,217		&r.ForSeconds, &r.ClearSeconds, &r.Severity, &r.Scope, &r.Notify}, extra...)...)218	return r, err219}220221func f(v float64) *float64 { return &v }222223// Builtins are the default rules. They are inserted once, by name; a restart224// never overwrites, re-enables or re-creates-with-defaults a rule that exists.225var Builtins = []Rule{226	{Name: "Node offline", Description: "The node has not reported for more than 2 minutes.", Kind: "absence", ForSeconds: 120, ClearSeconds: 30, Severity: "critical"},227	{Name: "Disk volume above 90 %", Description: "A volume is more than 90 % full.", Kind: "threshold", Metric: MetricDiskPct, LabelMatch: "*", Op: ">", Threshold: f(90), ClearThreshold: f(88), ForSeconds: 300, ClearSeconds: 300, Severity: "warning"},228	{Name: "Disk volume above 97 %", Description: "A volume is more than 97 % full.", Kind: "threshold", Metric: MetricDiskPct, LabelMatch: "*", Op: ">", Threshold: f(97), ClearThreshold: f(96), ForSeconds: 120, ClearSeconds: 300, Severity: "critical"},229	{Name: "Memory pressure (warning)", Description: "Kernel memory pressure level at warning or above for 5 minutes.", Kind: "threshold", Metric: "mem.pressure.level", Op: ">=", Threshold: f(2), ForSeconds: 300, ClearSeconds: 120, Severity: "warning"},230	{Name: "Memory pressure (critical)", Description: "Kernel memory pressure level critical for 2 minutes.", Kind: "threshold", Metric: "mem.pressure.level", Op: ">=", Threshold: f(4), ForSeconds: 120, ClearSeconds: 120, Severity: "critical"},231	{Name: "Swap above 80 %", Description: "More than 80 % of the allocated swap is in use for 10 minutes.", Kind: "threshold", Metric: MetricSwapPct, Op: ">", Threshold: f(80), ClearThreshold: f(70), ForSeconds: 600, ClearSeconds: 300, Severity: "warning"},232	{Name: "CPU saturated", Description: "CPU usage above 95 % for 15 minutes.", Kind: "threshold", Metric: "cpu.usage", Op: ">", Threshold: f(95), ClearThreshold: f(85), ForSeconds: 900, ClearSeconds: 300, Severity: "warning"},233	{Name: "Managed service errored", Description: "A managed service has been in the errored state for 2 minutes.", Kind: "service_state", Metric: "errored", ForSeconds: 120, ClearSeconds: 60, Severity: "critical"},234	{Name: "Agent certificate expiring", Description: "The agent certificate expires in less than 7 days (renewal is not happening).", Kind: "threshold", Metric: MetricCertDays, Op: "<", Threshold: f(7), ForSeconds: 0, ClearSeconds: 0, Severity: "warning"},235	{Name: "Packet loss to gateway", Description: "Sustained packet loss of 34 % or more to the gateway for 5 minutes.", Kind: "threshold", Metric: "net.loss", LabelMatch: "gateway", Op: ">=", Threshold: f(34), ClearThreshold: f(1), ForSeconds: 300, ClearSeconds: 120, Severity: "warning"},236	{Name: "Burst of denied requests", Description: "10 or more denied requests from one account or one address within 10 minutes.", Kind: "threshold", Metric: MetricDenied10m, LabelMatch: "*", Op: ">=", Threshold: f(10), ForSeconds: 0, ClearSeconds: 300, Severity: "warning"},237	{Name: "Sign-in from a new device", Description: "An owner or administrator signed in from a device never seen before.", Kind: "threshold", Metric: MetricNewDevice, LabelMatch: "*", Op: ">=", Threshold: f(1), ForSeconds: 0, ClearSeconds: 0, Severity: "info"},238}239240var builtinNS = uuid.MustParse("6d636300-0000-4000-8000-616c65727473")241242// SeedBuiltins inserts the missing default rules. Idempotent.243func SeedBuiltins(ctx context.Context, q interface {244	Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)245}) error {246	for _, r := range Builtins {247		r.Scope, r.Notify = Scope{All: true}, true248		if err := r.Validate(); err != nil {249			return fmt.Errorf("alerts: builtin %q: %w", r.Name, err)250		}251		if _, err := q.Exec(ctx, `insert into alert_rules(id, name, description, enabled, builtin, kind, metric, label_match, op, threshold, clear_threshold,252				for_seconds, clear_seconds, severity, scope, notify)253			values ($1,$2,$3,true,true,$4,nullif($5,''),nullif($6,''),nullif($7,''),$8,$9,$10,$11,$12::severity,$13,true)254			on conflict do nothing`, uuid.NewSHA1(builtinNS, []byte(r.Name)), r.Name, r.Description, r.Kind, r.Metric, r.LabelMatch, r.Op,255			r.Threshold, r.ClearThreshold, r.ForSeconds, r.ClearSeconds, r.Severity, r.Scope); err != nil {256			return err257		}258	}259	return nil260}261262// Save inserts or updates a validated rule.263func Save(ctx context.Context, q interface {264	Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)265}, r Rule, createdBy *uuid.UUID, insert bool) error {266	if err := r.Validate(); err != nil {267		return err268	}269	if insert {270		_, err := q.Exec(ctx, `insert into alert_rules(id, name, description, enabled, builtin, kind, metric, label_match, op, threshold, clear_threshold,271				for_seconds, clear_seconds, severity, scope, notify, created_by)272			values ($1,$2,$3,$4,false,$5,nullif($6,''),nullif($7,''),nullif($8,''),$9,$10,$11,$12,$13::severity,$14,$15,$16)`,273			r.ID, r.Name, r.Description, r.Enabled, r.Kind, r.Metric, r.LabelMatch, r.Op, r.Threshold, r.ClearThreshold, r.ForSeconds, r.ClearSeconds, r.Severity, r.Scope, r.Notify, createdBy)274		return err275	}276	_, err := q.Exec(ctx, `update alert_rules set name=$2, description=$3, enabled=$4, metric=nullif($5,''), label_match=nullif($6,''), op=nullif($7,''),277			threshold=$8, clear_threshold=$9, for_seconds=$10, clear_seconds=$11, severity=$12::severity, scope=$13, notify=$14, updated_at=now() where id=$1`,278		r.ID, r.Name, r.Description, r.Enabled, r.Metric, r.LabelMatch, r.Op, r.Threshold, r.ClearThreshold, r.ForSeconds, r.ClearSeconds, r.Severity, r.Scope, r.Notify)279	return err280}281
Files main
Menu

Theme