SPB Git

Repository Public

engine.go 18.7 KB 639 lines go

1package alerts23import (4	"context"5	"errors"6	"fmt"7	"math"8	"sort"9	"strings"10	"time"1112	"github.com/google/uuid"13	"github.com/jackc/pgx/v5"14	"github.com/jackc/pgx/v5/pgxpool"1516	"clustr.spboucher.ai/mcc/internal/events"17	"clustr.spboucher.ai/mcc/internal/ids"18)1920// Engine evaluates every enabled rule. It is stateless: everything it needs21// to continue after a restart is in the alerts table.22type Engine struct {23	Pool       *pgxpool.Pool24	Now        func() time.Time25	FreshAfter time.Duration // data older than this is not evaluated (default 30 s)26}2728func (e *Engine) now() time.Time {29	if e.Now != nil {30		return e.Now()31	}32	return time.Now().UTC()33}3435func (e *Engine) fresh() time.Duration {36	if e.FreshAfter > 0 {37		return e.FreshAfter38	}39	return 30 * time.Second40}4142type nodeInfo struct {43	id          uuid.UUID44	alias       string45	tags        []string46	maintenance bool47	hasAgent    bool48	enrolledAt  *time.Time49	certAfter   *time.Time50	lastReport  *time.Time // greatest(agent last_seen, node_state.received_at)51	stateAt     *time.Time52	values      map[string]any53}5455type serviceInfo struct {56	node  uuid.UUID57	label string58	state string59}6061type silence struct {62	rule  *uuid.UUID63	scope Scope64}6566type snapshot struct {67	now      time.Time68	nodes    map[uuid.UUID]*nodeInfo69	order    []uuid.UUID70	services []serviceInfo71	silences []silence72	denied   map[string]float64 // lazily loaded73	newDev   map[string]float6474}7576// instance is the current observation of one (rule, node, label).77type instance struct {78	node      *nodeInfo79	label     string80	value     float6481	evaluable bool       // false: no fresh data — never opens, never confirms82	firing    bool       // condition holds83	clear     bool       // condition is clear (beyond the hysteresis band)84	since     *time.Time // absence rules: when the condition really started85	what      string     // human description of the observation86}8788func fingerprint(rule uuid.UUID, node *nodeInfo, label string) string {89	n := "-"90	if node != nil {91		n = node.id.String()92	}93	return rule.String() + "/" + n + "/" + label94}9596func (e *Engine) load(ctx context.Context, now time.Time) (*snapshot, error) {97	s := &snapshot{now: now, nodes: map[uuid.UUID]*nodeInfo{}}98	rows, err := e.Pool.Query(ctx, `select n.id, n.alias::text, n.tags, n.maintenance, a.id is not null, a.enrolled_at, a.cert_not_after,99			greatest(a.last_seen_at, ns.received_at), ns.received_at, ns.values100		from nodes n left join agents a on a.node_id=n.id and a.revoked_at is null left join node_state ns on ns.node_id=n.id101		where n.retired_at is null order by n.alias`)102	if err != nil {103		return nil, err104	}105	defer rows.Close()106	for rows.Next() {107		n := &nodeInfo{}108		if err := rows.Scan(&n.id, &n.alias, &n.tags, &n.maintenance, &n.hasAgent, &n.enrolledAt, &n.certAfter, &n.lastReport, &n.stateAt, &n.values); err != nil {109			return nil, err110		}111		s.nodes[n.id] = n112		s.order = append(s.order, n.id)113	}114	if err := rows.Err(); err != nil {115		return nil, err116	}117	srows, err := e.Pool.Query(ctx, `select node_id, manager::text || ':' || name, state from services where managed and gone_at is null`)118	if err != nil {119		return nil, err120	}121	if s.services, err = pgx.CollectRows(srows, func(r pgx.CollectableRow) (serviceInfo, error) {122		var v serviceInfo123		return v, r.Scan(&v.node, &v.label, &v.state)124	}); err != nil {125		return nil, err126	}127	sl, err := e.Pool.Query(ctx, `select rule_id, scope from silences where starts_at <= $1 and ends_at > $1`, now)128	if err != nil {129		return nil, err130	}131	s.silences, err = pgx.CollectRows(sl, func(r pgx.CollectableRow) (silence, error) {132		var v silence133		return v, r.Scan(&v.rule, &v.scope)134	})135	return s, err136}137138func (s *snapshot) isFresh(n *nodeInfo, max time.Duration) bool {139	return n.stateAt != nil && s.now.Sub(*n.stateAt) <= max140}141142// suppression says why notifications are withheld for this alert, if they are.143func (s *snapshot) suppression(rule uuid.UUID, n *nodeInfo) string {144	if n != nil && n.maintenance {145		return "maintenance"146	}147	for _, sl := range s.silences {148		if sl.rule != nil && *sl.rule != rule {149			continue150		}151		var id *uuid.UUID152		var tags []string153		if n != nil {154			id, tags = &n.id, n.tags155		}156		if sl.scope.Matches(id, tags) {157			return "silence"158		}159	}160	return ""161}162163func num(v any) (float64, bool) {164	f, ok := v.(float64)165	return f, ok && !math.IsNaN(f) && !math.IsInf(f, 0)166}167168// series returns label -> value for a metric in a node snapshot ("name" or "name{label}" keys).169func series(values map[string]any, metric string) map[string]float64 {170	out := map[string]float64{}171	if v, ok := num(values[metric]); ok {172		out[""] = v173	}174	// telemetry-ingest stores labeled metrics nested: "net.rx": {"en0": 1.0}.175	if sub, ok := values[metric].(map[string]any); ok {176		for label, raw := range sub {177			if v, ok := num(raw); ok {178				out[label] = v179			}180		}181	}182	prefix := metric + "{"183	for k, raw := range values {184		if strings.HasPrefix(k, prefix) && strings.HasSuffix(k, "}") {185			if v, ok := num(raw); ok {186				out[k[len(prefix):len(k)-1]] = v187			}188		}189	}190	return out191}192193// nodeSeries resolves catalog and derived node metrics.194func nodeSeries(values map[string]any, metric string) map[string]float64 {195	ratio := func(used, total string) map[string]float64 {196		u, t := series(values, used), series(values, total)197		out := map[string]float64{}198		for l, uv := range u {199			if tv, ok := t[l]; ok && tv > 0 {200				out[l] = 100 * uv / tv201			}202		}203		return out204	}205	switch metric {206	case MetricDiskPct:207		return ratio("disk.used", "disk.total")208	case MetricSwapPct:209		return ratio("swap.used", "swap.total")210	}211	return series(values, metric)212}213214func labelOK(match, label string) bool { return match == "" || match == "*" || match == label }215216func (e *Engine) instances(ctx context.Context, r Rule, s *snapshot) (map[string]instance, error) {217	out := map[string]instance{}218	put := func(i instance) { out[fingerprint(r.ID, i.node, i.label)] = i }219	inScope := func(n *nodeInfo) bool { return r.Scope.Matches(&n.id, n.tags) }220	switch r.Kind {221	case "absence":222		for _, id := range s.order {223			n := s.nodes[id]224			if !n.hasAgent || !inScope(n) { // a node that never had an agent cannot "stop reporting"225				continue226			}227			last := n.lastReport228			if last == nil {229				last = n.enrolledAt230			}231			if last == nil {232				continue233			}234			age := s.now.Sub(*last)235			silent := age > e.fresh()236			put(instance{node: n, value: math.Round(age.Seconds()), evaluable: true, firing: silent, clear: !silent, since: last,237				what: fmt.Sprintf("no report for %s", age.Round(time.Second))})238		}239	case "service_state":240		states := r.ServiceStates()241		for _, sv := range s.services {242			n := s.nodes[sv.node]243			if n == nil || !inScope(n) || !labelOK(r.LabelMatch, sv.label) {244				continue245			}246			bad := false247			for _, st := range states {248				bad = bad || st == sv.state249			}250			v := 0.0251			if bad {252				v = 1253			}254			put(instance{node: n, label: sv.label, value: v, evaluable: s.isFresh(n, e.fresh()), firing: bad, clear: !bad, what: "service " + sv.label + " is " + sv.state})255		}256	case "threshold":257		switch r.Metric {258		case MetricCertDays:259			for _, id := range s.order {260				n := s.nodes[id]261				if !n.hasAgent || n.certAfter == nil || !inScope(n) {262					continue263				}264				d := n.certAfter.Sub(s.now).Hours() / 24265				put(instance{node: n, value: math.Round(d*10) / 10, evaluable: true, firing: r.Firing(d), clear: r.Clear(d), what: fmt.Sprintf("agent certificate expires in %.1f days", d)})266			}267		case MetricDenied10m, MetricNewDevice:268			if !r.Scope.All { // cluster-wide metrics belong to no node269				return out, nil270			}271			m, err := e.clusterMetric(ctx, r.Metric, s)272			if err != nil {273				return nil, err274			}275			for label, v := range m {276				if labelOK(r.LabelMatch, label) {277					put(instance{label: label, value: v, evaluable: true, firing: r.Firing(v), clear: r.Clear(v), what: fmt.Sprintf("%s %s = %g", r.Metric, label, v)})278				}279			}280		default:281			for _, id := range s.order {282				n := s.nodes[id]283				if !inScope(n) {284					continue285				}286				fresh := s.isFresh(n, e.fresh())287				for label, v := range nodeSeries(n.values, r.Metric) {288					if !labelOK(r.LabelMatch, label) {289						continue290					}291					what := fmt.Sprintf("%s = %.4g", r.Metric, v)292					if label != "" {293						what = fmt.Sprintf("%s{%s} = %.4g", r.Metric, label, v)294					}295					put(instance{node: n, label: label, value: v, evaluable: fresh, firing: fresh && r.Firing(v), clear: r.Clear(v), what: what})296				}297			}298		}299	}300	return out, nil301}302303// clusterMetric computes the audit-derived metrics (simple anomaly detection).304func (e *Engine) clusterMetric(ctx context.Context, metric string, s *snapshot) (map[string]float64, error) {305	switch metric {306	case MetricDenied10m:307		if s.denied == nil {308			s.denied = map[string]float64{}309			rows, err := e.Pool.Query(ctx, `select 'actor:' || actor_label, count(*) from audit_log310					where outcome='denied' and ts > $1 and ts <= $2 and actor_label <> '' group by 1311				union all312				select 'ip:' || host(ip), count(*) from audit_log where outcome='denied' and ts > $1 and ts <= $2 and ip is not null group by 1`,313				s.now.Add(-deniedWindowMin*time.Minute), s.now)314			if err != nil {315				return nil, err316			}317			defer rows.Close()318			for rows.Next() {319				var l string320				var c float64321				if err := rows.Scan(&l, &c); err != nil {322					return nil, err323				}324				s.denied[l] = c325			}326			if err := rows.Err(); err != nil {327				return nil, err328			}329		}330		return s.denied, nil331	case MetricNewDevice:332		if s.newDev == nil {333			s.newDev = map[string]float64{}334			// A sign-in whose device was created by that very sign-in, for an account that already had another device.335			rows, err := e.Pool.Query(ctx, `select u.handle::text, count(*) from audit_log l336					join users u on u.id=l.actor_id and u.role in ('owner','admin')337					join devices d on d.id=l.device_id338				where l.action='auth.login' and l.outcome='success' and l.ts > $1 and l.ts <= $2339				  and d.created_at >= l.ts - interval '1 minute'340				  and exists (select 1 from devices o where o.user_id=u.id and o.id<>d.id and o.created_at < d.created_at)341				group by 1`, s.now.Add(-newDeviceWindowMn*time.Minute), s.now)342			if err != nil {343				return nil, err344			}345			defer rows.Close()346			for rows.Next() {347				var l string348				var c float64349				if err := rows.Scan(&l, &c); err != nil {350					return nil, err351				}352				s.newDev[l] = c353			}354			if err := rows.Err(); err != nil {355				return nil, err356			}357		}358		return s.newDev, nil359	}360	return nil, nil361}362363// Evaluate runs one evaluation of every rule. Safe to call concurrently.364func (e *Engine) Evaluate(ctx context.Context) error {365	now := e.now()366	s, err := e.load(ctx, now)367	if err != nil {368		return err369	}370	rr, err := e.Pool.Query(ctx, `select `+RuleColumns+` from alert_rules where deleted_at is null order by name`)371	if err != nil {372		return err373	}374	rules, err := pgx.CollectRows(rr, func(r pgx.CollectableRow) (Rule, error) { return ScanRule(r) })375	if err != nil {376		return err377	}378	type open struct {379		fp   string380		rule uuid.UUID381	}382	or, err := e.Pool.Query(ctx, `select fingerprint, rule_id from alerts where state <> 'resolved'`)383	if err != nil {384		return err385	}386	opens, err := pgx.CollectRows(or, func(r pgx.CollectableRow) (open, error) {387		var o open388		return o, r.Scan(&o.fp, &o.rule)389	})390	if err != nil {391		return err392	}393	for _, r := range rules {394		insts := map[string]instance{}395		if r.Enabled {396			if insts, err = e.instances(ctx, r, s); err != nil {397				return err398			}399		}400		fps := make([]string, 0, len(insts))401		for fp := range insts {402			fps = append(fps, fp)403		}404		sort.Strings(fps)405		for _, fp := range fps {406			if err := e.apply(ctx, r, fp, insts[fp], s, ""); err != nil {407				return err408			}409		}410		// Open alerts the rule no longer produces.411		for _, o := range opens {412			if o.rule != r.ID {413				continue414			}415			if _, ok := insts[o.fp]; ok {416				continue417			}418			gone := instance{}419			why := "no recent data"420			switch {421			case !r.Enabled:422				why = "rule disabled"423			case r.Metric == MetricDenied10m || r.Metric == MetricNewDevice:424				gone = instance{evaluable: true, clear: true} // nothing counted any more: the value is zero425				why = ""426			}427			if err := e.apply(ctx, r, o.fp, gone, s, why); err != nil {428				return err429			}430		}431	}432	return nil433}434435type alertRow struct {436	id         uuid.UUID437	state      string438	nodeID     *uuid.UUID439	started    time.Time440	clearSince *time.Time441	suppressed *string442	flaps      int443	summary    string444	incident   *uuid.UUID445	lastValue  *float64446}447448// apply advances one alert. goneWhy is set when the rule produced no observation for an open alert.449func (e *Engine) apply(ctx context.Context, r Rule, fp string, in instance, s *snapshot, goneWhy string) error {450	now := s.now451	return pgx.BeginFunc(ctx, e.Pool, func(tx pgx.Tx) error {452		var a alertRow453		err := tx.QueryRow(ctx, `select id, state::text, node_id, started_at, clear_since, suppressed_by, flap_count, summary, incident_id, last_value454			from alerts where fingerprint=$1 and state <> 'resolved' for update`, fp).455			Scan(&a.id, &a.state, &a.nodeID, &a.started, &a.clearSince, &a.suppressed, &a.flaps, &a.summary, &a.incident, &a.lastValue)456		exists := err == nil457		if err != nil && !errors.Is(err, pgx.ErrNoRows) {458			return err459		}460		node := in.node461		if node == nil && a.nodeID != nil {462			node = s.nodes[*a.nodeID]463		}464		supp := s.suppression(r.ID, node)465		if goneWhy == "rule disabled" {466			supp = deref(a.suppressed)467		}468		timeline := func(kind, body string) error {469			_, err := tx.Exec(ctx, `insert into timeline_events(id, alert_id, ts, kind, body) values ($1,$2,$3,$4,$5)`, ids.New(), a.id, now, kind, body)470			return err471		}472		publish := func(state string) error {473			p := map[string]any{"alert_id": a.id, "rule_id": r.ID, "state": state, "severity": r.Severity, "summary": a.summary, "suppressed": supp != ""}474			if node != nil {475				p["node_id"] = node.id476			}477			if a.incident != nil {478				p["incident_id"] = *a.incident479			}480			return events.Publish(ctx, tx, "alert.changed", p)481		}482		fire := func() error {483			if _, err := tx.Exec(ctx, `update alerts set state='firing', fired_at=$2 where id=$1`, a.id, now); err != nil {484				return err485			}486			if err := timeline("fired", in.what); err != nil {487				return err488			}489			switch {490			case supp != "":491				if err := timeline("suppressed", "notification withheld: "+supp); err != nil {492					return err493				}494			case r.Notify:495				if _, err := tx.Exec(ctx, `insert into alert_notifications(alert_id, transition, created_at, next_try_at) values ($1,'firing',$2,$2)`, a.id, now); err != nil {496					return err497				}498			}499			return publish("firing")500		}501502		if !exists {503			if !in.evaluable || !in.firing {504				return nil505			}506			a = alertRow{id: ids.New(), started: now}507			if in.since != nil && in.since.Before(now) {508				a.started = *in.since509			}510			a.summary = r.Name + ": " + in.what511			if node != nil {512				a.summary = r.Name + " on " + node.alias + ": " + in.what513			}514			// A fingerprint that re-opens shortly after resolving is flapping.515			var prevFlaps *int516			if err := tx.QueryRow(ctx, `select flap_count from alerts where fingerprint=$1 and state='resolved' and fired_at is not null and resolved_at > $2517				order by resolved_at desc limit 1`, fp, now.Add(-15*time.Minute)).Scan(&prevFlaps); err != nil && !errors.Is(err, pgx.ErrNoRows) {518				return err519			}520			if prevFlaps != nil {521				a.flaps = *prevFlaps + 1522			}523			var nodeID *uuid.UUID524			if node != nil {525				nodeID = &node.id526			}527			tag, err := tx.Exec(ctx, `insert into alerts(id, rule_id, node_id, label, fingerprint, state, severity, summary, last_value, started_at, last_eval_at, flap_count, suppressed_by)528				values ($1,$2,$3,$4,$5,'pending',$6::severity,$7,$8,$9,$10,$11,nullif($12,''))529				on conflict (fingerprint) where state <> 'resolved' do nothing`,530				a.id, r.ID, nodeID, in.label, fp, r.Severity, a.summary, in.value, a.started, now, a.flaps, supp)531			if err != nil || tag.RowsAffected() == 0 {532				return err // 0 rows: another evaluator opened it first533			}534			kind := "opened"535			if prevFlaps != nil {536				kind = "reopened"537			}538			if err := timeline(kind, in.what); err != nil {539				return err540			}541			if now.Sub(a.started) >= time.Duration(r.ForSeconds)*time.Second {542				return fire()543			}544			return publish("pending")545		}546547		resolve := func(body string, notify bool) error {548			if _, err := tx.Exec(ctx, `update alerts set state='resolved', resolved_at=$2, last_eval_at=$2, suppressed_by=nullif($3,'') where id=$1`, a.id, now, supp); err != nil {549				return err550			}551			if err := timeline("resolved", body); err != nil {552				return err553			}554			if notify && r.Notify && supp == "" {555				if _, err := tx.Exec(ctx, `insert into alert_notifications(alert_id, transition, created_at, next_try_at) values ($1,'resolved',$2,$2)`, a.id, now); err != nil {556					return err557				}558			}559			return publish("resolved")560		}561		noData := "no recent data: the condition can no longer be verified"562		if goneWhy != "" && goneWhy != "no recent data" {563			noData = goneWhy564		}565		if goneWhy == "rule disabled" {566			return resolve("rule disabled", false)567		}568		switch a.state {569		case "pending":570			switch {571			case !in.evaluable:572				return resolve(noData, false)573			case !in.firing:574				return resolve("condition cleared before firing", false)575			case now.Sub(a.started) >= time.Duration(r.ForSeconds)*time.Second:576				if _, err := tx.Exec(ctx, `update alerts set last_value=$2, last_eval_at=$3, suppressed_by=nullif($4,'') where id=$1`, a.id, in.value, now, supp); err != nil {577					return err578				}579				return fire()580			}581		case "firing", "acknowledged":582			clearing := !in.evaluable || in.clear583			switch {584			case clearing:585				since := now586				if a.clearSince != nil {587					since = *a.clearSince588				}589				if now.Sub(since) >= time.Duration(r.ClearSeconds)*time.Second {590					body := "condition cleared"591					if !in.evaluable {592						body = noData593					}594					return resolve(body, true)595				}596				if a.clearSince == nil {597					if _, err := tx.Exec(ctx, `update alerts set clear_since=$2 where id=$1`, a.id, now); err != nil {598						return err599					}600				}601			case a.clearSince != nil:602				// It started to clear and came back: one flap.603				if _, err := tx.Exec(ctx, `update alerts set clear_since=null, flap_count=flap_count+1 where id=$1`, a.id); err != nil {604					return err605				}606				if err := timeline("state_changed", "condition returned before the clear delay (flap)"); err != nil {607					return err608				}609			}610		}611		// Still open: refresh the observation and the suppression.612		var val *float64613		if in.evaluable {614			val = &in.value615		}616		if _, err := tx.Exec(ctx, `update alerts set last_value=coalesce($2,last_value), last_eval_at=$3, suppressed_by=nullif($4,'') where id=$1`, a.id, val, now, supp); err != nil {617			return err618		}619		if supp != deref(a.suppressed) {620			body := "notifications restored"621			if supp != "" {622				body = "notifications withheld: " + supp623			}624			if err := timeline("suppressed", body); err != nil {625				return err626			}627			return publish(a.state)628		}629		return nil630	})631}632633func deref[T any](p *T) (v T) {634	if p != nil {635		v = *p636	}637	return v638}639
Files main
Menu

Theme