SPB Git

Repository Public

ops_alerts.go 33.8 KB 987 lines go

1package api23import (4	"context"5	"errors"6	"strings"7	"time"89	"github.com/google/uuid"10	"github.com/jackc/pgx/v5"1112	"clustr.spboucher.ai/mcc/internal/alerts"13	"clustr.spboucher.ai/mcc/internal/audit"14	"clustr.spboucher.ai/mcc/internal/authz"15	"clustr.spboucher.ai/mcc/internal/events"16	"clustr.spboucher.ai/mcc/internal/httpx"17	"clustr.spboucher.ai/mcc/internal/ids"18)1920// Alerts, rules, silences and incidents. The alert engine (internal/alerts)21// owns the automatic transitions; this file holds the human ones.2223// ── Mapping ────────────────────────────────────────────────────────────────2425const alertSelect = `select a.id, a.rule_id, r.name, a.node_id, n.alias::text, a.label, a.fingerprint, a.state::text, a.severity::text, a.summary, a.last_value,26	a.started_at, a.fired_at, a.acknowledged_at, a.acknowledged_by, u.display_name, a.resolved_at, a.last_eval_at, a.flap_count, a.suppressed_by, a.incident_id27	from alerts a join alert_rules r on r.id=a.rule_id left join nodes n on n.id=a.node_id left join users u on u.id=a.acknowledged_by`2829func scanAlert(r pgx.Row) (Alert, error) {30	var a Alert31	var state, sev string32	var supp *string33	var val *float6434	err := r.Scan(&a.Id, &a.RuleId, &a.RuleName, &a.NodeId, &a.NodeAlias, &a.Label, &a.Fingerprint, &state, &sev, &a.Summary, &val,35		&a.StartedAt, &a.FiredAt, &a.AcknowledgedAt, &a.AcknowledgedBy, &a.AcknowledgedByDisplayName, &a.ResolvedAt, &a.LastEvalAt, &a.FlapCount, &supp, &a.IncidentId)36	a.State, a.Severity = AlertState(state), Severity(sev)37	if supp != nil {38		a.SuppressedBy = ptr(AlertSuppressedBy(*supp))39	}40	if val != nil {41		a.LastValue = ptr(float32(*val))42	}43	return a, err44}4546func alertNodeRes(a Alert) authz.Resource {47	if a.NodeId == nil {48		return authz.Resource{}49	}50	return authz.Resource{Kind: "node", ID: a.NodeId.String()}51}5253func (h *opsHandlers) timeline(ctx context.Context, col string, id uuid.UUID) ([]TimelineEvent, error) {54	rows, err := h.Pool.Query(ctx, `select t.id, t.ts, t.kind, t.actor_id, u.display_name, t.body from timeline_events t left join users u on u.id=t.actor_id55		where t.`+col+`=$1 order by t.ts, t.id limit 1000`, id)56	if err != nil {57		return nil, err58	}59	out, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (TimelineEvent, error) {60		var t TimelineEvent61		return t, r.Scan(&t.Id, &t.Ts, &t.Kind, &t.ActorId, &t.ActorDisplayName, &t.Body)62	})63	if out == nil {64		out = []TimelineEvent{}65	}66	return out, err67}6869func (h *opsHandlers) addTimeline(ctx context.Context, tx pgx.Tx, alert, incident *uuid.UUID, kind, body string) (TimelineEvent, error) {70	p := principalFrom(ctx)71	t := TimelineEvent{Id: ids.New(), Ts: nowUTC(), Kind: kind, Body: body, ActorId: &p.Session.User.ID, ActorDisplayName: &p.Session.User.DisplayName}72	_, err := tx.Exec(ctx, `insert into timeline_events(id, alert_id, incident_id, ts, kind, actor_id, body) values ($1,$2,$3,$4,$5,$6,$7)`,73		t.Id, alert, incident, t.Ts, kind, t.ActorId, body)74	return t, err75}7677func publishAlert(ctx context.Context, tx pgx.Tx, a Alert) error {78	p := map[string]any{"alert_id": a.Id, "rule_id": a.RuleId, "state": a.State, "severity": a.Severity, "summary": a.Summary, "suppressed": a.SuppressedBy != nil}79	if a.NodeId != nil {80		p["node_id"] = *a.NodeId81	}82	if a.IncidentId != nil {83		p["incident_id"] = *a.IncidentId84	}85	return events.Publish(ctx, tx, "alert.changed", p)86}8788// ── Alerts ─────────────────────────────────────────────────────────────────8990func (h *opsHandlers) ListAlerts(ctx context.Context, req ListAlertsRequestObject) (ListAlertsResponseObject, error) {91	p := req.Params92	var w where93	if p.State != nil {94		w.add("a.state = ?::alert_state", string(*p.State))95	}96	if p.Open != nil && *p.Open {97		w.add("a.state <> 'resolved'")98	}99	if p.Severity != nil {100		w.add("a.severity = ?::severity", string(*p.Severity))101	}102	if p.NodeId != nil {103		w.add("a.node_id = ?", *p.NodeId)104	}105	if p.RuleId != nil {106		w.add("a.rule_id = ?", *p.RuleId)107	}108	if p.IncidentId != nil {109		w.add("a.incident_id = ?", *p.IncidentId)110	}111	cur, err := uuidCursor(p.Cursor)112	if err != nil {113		return nil, err114	}115	if cur != nil {116		w.add("a.id < ?", *cur)117	}118	limit := pageLimit(p.Limit)119	rows, err := h.Pool.Query(ctx, alertSelect+w.sql()+" order by a.id desc limit "+w.arg(limit+1), w.args...)120	if err != nil {121		return nil, err122	}123	items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Alert, error) { return scanAlert(r) })124	if err != nil {125		return nil, err126	}127	out := AlertList{Items: items}128	if len(items) > limit {129		out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())130	}131	if out.Items == nil {132		out.Items = []Alert{}133	}134	return ListAlerts200JSONResponse{Body: out}, nil135}136137func (h *opsHandlers) GetAlert(ctx context.Context, req GetAlertRequestObject) (GetAlertResponseObject, error) {138	a, err := scanAlert(h.Pool.QueryRow(ctx, alertSelect+` where a.id=$1`, req.Id))139	if err != nil {140		return nil, notFound(err)141	}142	tl, err := h.timeline(ctx, "alert_id", req.Id)143	if err != nil {144		return nil, err145	}146	return GetAlert200JSONResponse{Body: AlertDetail{Alert: a, Timeline: tl}}, nil147}148149// transition is acknowledge and manual resolve.150func (h *opsHandlers) transition(ctx context.Context, id uuid.UUID, to string, body *AlertTransitionRequest) (Alert, error) {151	comment := ""152	if body != nil {153		comment = strings.TrimSpace(deref(body.Comment))154	}155	if len(comment) > 4000 {156		return Alert{}, httpx.Validation("comment is too long.")157	}158	var a Alert159	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {160		var err error161		if a, err = scanAlert(tx.QueryRow(ctx, alertSelect+` where a.id=$1 for update of a`, id)); err != nil {162			return notFound(err)163		}164		if err := h.authorize(ctx, authz.AlertsAck, alertNodeRes(a)); err != nil {165			return err166		}167		now, me := nowUTC(), h.userID(ctx)168		switch to {169		case "acknowledged":170			if a.State != AlertStateFiring {171				return httpx.Conflict("conflict", "Only a firing alert can be acknowledged.")172			}173			_, err = tx.Exec(ctx, `update alerts set state='acknowledged', acknowledged_at=$2, acknowledged_by=$3 where id=$1`, id, now, me)174		case "resolved":175			if a.State == AlertStateResolved {176				return httpx.Conflict("conflict", "The alert is already resolved.")177			}178			_, err = tx.Exec(ctx, `update alerts set state='resolved', resolved_at=$2 where id=$1`, id, now)179			if comment == "" {180				comment = "resolved manually"181			}182		}183		if err != nil {184			return err185		}186		if _, err := h.addTimeline(ctx, tx, &id, nil, to, comment); err != nil {187			return err188		}189		if a, err = scanAlert(tx.QueryRow(ctx, alertSelect+` where a.id=$1`, id)); err != nil {190			return err191		}192		if err := publishAlert(ctx, tx, a); err != nil {193			return err194		}195		return h.audit(ctx, tx, "alert."+map[string]string{"acknowledged": "acknowledge", "resolved": "resolve"}[to], "alert", id.String(), audit.Success, "",196			map[string]any{"rule": a.RuleName, "node": deref(a.NodeAlias)})197	})198	return a, err199}200201func (h *opsHandlers) AcknowledgeAlert(ctx context.Context, req AcknowledgeAlertRequestObject) (AcknowledgeAlertResponseObject, error) {202	a, err := h.transition(ctx, req.Id, "acknowledged", req.Body)203	if err != nil {204		return nil, err205	}206	return AcknowledgeAlert200JSONResponse{Body: a}, nil207}208209func (h *opsHandlers) ResolveAlert(ctx context.Context, req ResolveAlertRequestObject) (ResolveAlertResponseObject, error) {210	a, err := h.transition(ctx, req.Id, "resolved", req.Body)211	if err != nil {212		return nil, err213	}214	return ResolveAlert200JSONResponse{Body: a}, nil215}216217func commentBody(b *CommentRequest) (string, error) {218	if b == nil || strings.TrimSpace(b.Body) == "" || len(b.Body) > 4000 {219		return "", httpx.Validation("body is required (at most 4000 characters).")220	}221	return strings.TrimSpace(b.Body), nil222}223224func (h *opsHandlers) CommentAlert(ctx context.Context, req CommentAlertRequestObject) (CommentAlertResponseObject, error) {225	body, err := commentBody(req.Body)226	if err != nil {227		return nil, err228	}229	a, err := scanAlert(h.Pool.QueryRow(ctx, alertSelect+` where a.id=$1`, req.Id))230	if err != nil {231		return nil, notFound(err)232	}233	if err := h.authorize(ctx, authz.AlertsAck, alertNodeRes(a)); err != nil {234		return nil, err235	}236	var t TimelineEvent237	err = pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {238		if t, err = h.addTimeline(ctx, tx, &req.Id, nil, "comment", body); err != nil {239			return err240		}241		return h.audit(ctx, tx, "alert.comment", "alert", req.Id.String(), audit.Success, "", nil)242	})243	if err != nil {244		return nil, err245	}246	return CommentAlert201JSONResponse{Body: t}, nil247}248249// ── Rules ──────────────────────────────────────────────────────────────────250251var opToSQL = map[AlertOp]string{AlertOpGt: ">", AlertOpGte: ">=", AlertOpLt: "<", AlertOpLte: "<="}252var opFromSQL = map[string]AlertOp{">": AlertOpGt, ">=": AlertOpGte, "<": AlertOpLt, "<=": AlertOpLte}253254func f64(p *float32) *float64 {255	if p == nil {256		return nil257	}258	v := float64(*p)259	return &v260}261262func f32(p *float64) *float32 {263	if p == nil {264		return nil265	}266	v := float32(*p)267	return &v268}269270func scopeIn(s *AlertScope) alerts.Scope {271	if s == nil {272		return alerts.Scope{All: true}273	}274	return alerts.Scope{All: deref(s.All), NodeIDs: deref(s.NodeIds), Tags: deref(s.Tags)}275}276277func scopeOut(s alerts.Scope) AlertScope {278	var out AlertScope279	switch {280	case s.All:281		out.All = ptr(true)282	case len(s.NodeIDs) > 0:283		out.NodeIds = &s.NodeIDs284	case len(s.Tags) > 0:285		out.Tags = &s.Tags286	}287	return out288}289290func ruleFromInput(in AlertRuleInput) (alerts.Rule, error) {291	r := alerts.Rule{Name: strings.TrimSpace(in.Name), Description: deref(in.Description), Enabled: in.Enabled == nil || *in.Enabled, Kind: string(in.Kind),292		Metric: deref(in.Metric), LabelMatch: deref(in.LabelMatch), Threshold: f64(in.Threshold), ClearThreshold: f64(in.ClearThreshold),293		ForSeconds: 300, ClearSeconds: 120, Severity: "warning", Scope: scopeIn(in.Scope), Notify: in.Notify == nil || *in.Notify}294	if in.Op != nil {295		op, ok := opToSQL[*in.Op]296		if !ok {297			return r, errors.New("op must be gt, gte, lt or lte")298		}299		r.Op = op300	}301	if in.ForSeconds != nil {302		r.ForSeconds = *in.ForSeconds303	}304	if in.ClearSeconds != nil {305		r.ClearSeconds = *in.ClearSeconds306	}307	if in.Severity != nil {308		r.Severity = string(*in.Severity)309	}310	return r, r.Validate()311}312313type ruleMeta struct {314	createdBy *uuid.UUID315	created   time.Time316	updated   time.Time317}318319func ruleOut(r alerts.Rule, m ruleMeta) AlertRule {320	out := AlertRule{Id: r.ID, Name: r.Name, Description: r.Description, Enabled: r.Enabled, Builtin: r.Builtin, Kind: AlertRuleKind(r.Kind),321		Metric: nonEmpty(r.Metric), LabelMatch: nonEmpty(r.LabelMatch), Threshold: f32(r.Threshold), ClearThreshold: f32(r.ClearThreshold),322		ForSeconds: r.ForSeconds, ClearSeconds: r.ClearSeconds, Severity: Severity(r.Severity), Scope: scopeOut(r.Scope), Notify: r.Notify,323		CreatedBy: m.createdBy, CreatedAt: m.created, UpdatedAt: m.updated}324	if op, ok := opFromSQL[r.Op]; ok {325		out.Op = &op326	}327	return out328}329330const ruleSelect = `select ` + alerts.RuleColumns + `, created_by, created_at, updated_at from alert_rules`331332func (h *opsHandlers) loadRule(ctx context.Context, q interface {333	QueryRow(context.Context, string, ...any) pgx.Row334}, id uuid.UUID, lock string) (alerts.Rule, ruleMeta, error) {335	var m ruleMeta336	r, err := alerts.ScanRule(q.QueryRow(ctx, ruleSelect+` where id=$1 and deleted_at is null`+lock, id), &m.createdBy, &m.created, &m.updated)337	return r, m, notFound(err)338}339340// scopeNodesExist refuses a scope that names unknown nodes.341func (h *opsHandlers) scopeNodesExist(ctx context.Context, nodes []uuid.UUID) error {342	if len(nodes) == 0 {343		return nil344	}345	var n int346	if err := h.Pool.QueryRow(ctx, `select count(distinct id) from nodes where id = any($1) and retired_at is null`, nodes).Scan(&n); err != nil {347		return err348	}349	if n != len(nodes) {350		return httpx.Validation("The scope names a node that does not exist (or repeats one).")351	}352	return nil353}354355func (h *opsHandlers) ListAlertRules(ctx context.Context, req ListAlertRulesRequestObject) (ListAlertRulesResponseObject, error) {356	var w where357	w.add("deleted_at is null")358	if req.Params.Enabled != nil {359		w.add("enabled = ?", *req.Params.Enabled)360	}361	cur, err := uuidCursor(req.Params.Cursor)362	if err != nil {363		return nil, err364	}365	if cur != nil {366		w.add("id > ?", *cur)367	}368	limit := pageLimit(req.Params.Limit)369	rows, err := h.Pool.Query(ctx, ruleSelect+w.sql()+" order by id limit "+w.arg(limit+1), w.args...)370	if err != nil {371		return nil, err372	}373	items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (AlertRule, error) {374		var m ruleMeta375		rule, err := alerts.ScanRule(r, &m.createdBy, &m.created, &m.updated)376		return ruleOut(rule, m), err377	})378	if err != nil {379		return nil, err380	}381	out := AlertRuleList{Items: items}382	if len(items) > limit {383		out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())384	}385	if out.Items == nil {386		out.Items = []AlertRule{}387	}388	return ListAlertRules200JSONResponse{Body: out}, nil389}390391func (h *opsHandlers) GetAlertRule(ctx context.Context, req GetAlertRuleRequestObject) (GetAlertRuleResponseObject, error) {392	r, m, err := h.loadRule(ctx, h.Pool, req.Id, "")393	if err != nil {394		return nil, err395	}396	return GetAlertRule200JSONResponse{Body: ruleOut(r, m)}, nil397}398399func (h *opsHandlers) CreateAlertRule(ctx context.Context, req CreateAlertRuleRequestObject) (CreateAlertRuleResponseObject, error) {400	if req.Body == nil {401		return nil, httpx.Validation("Body required.")402	}403	r, err := ruleFromInput(*req.Body)404	if err != nil {405		return nil, httpx.Validation("Invalid rule: " + err.Error() + ".")406	}407	if err := h.scopeNodesExist(ctx, r.Scope.NodeIDs); err != nil {408		return nil, err409	}410	r.ID = ids.New()411	me := h.userID(ctx)412	err = pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {413		if err := alerts.Save(ctx, tx, r, &me, true); err != nil {414			return err415		}416		return h.audit(ctx, tx, "alert_rule.create", "alert_rule", r.ID.String(), audit.Success, "", map[string]any{"name": r.Name, "kind": r.Kind, "severity": r.Severity})417	})418	if uniqueViolation(err) {419		return nil, httpx.Conflict("conflict", "A rule with this name already exists.")420	}421	if err != nil {422		return nil, err423	}424	r, m, err := h.loadRule(ctx, h.Pool, r.ID, "")425	if err != nil {426		return nil, err427	}428	return CreateAlertRule201JSONResponse{Body: ruleOut(r, m)}, nil429}430431func (h *opsHandlers) UpdateAlertRule(ctx context.Context, req UpdateAlertRuleRequestObject) (UpdateAlertRuleResponseObject, error) {432	if req.Body == nil {433		return nil, httpx.Validation("Body required.")434	}435	b := req.Body436	var out AlertRule437	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {438		r, _, err := h.loadRule(ctx, tx, req.Id, " for update")439		if err != nil {440			return err441		}442		if r.Builtin && (b.Name != nil || b.Description != nil || b.Metric != nil || b.LabelMatch != nil || b.Op != nil) {443			return httpx.Conflict("conflict", "A built-in rule only accepts changes to enabled, thresholds, durations, severity, scope and notify.")444		}445		changed := map[string]any{}446		set := func(k string, v any) { changed[k] = v }447		if b.Name != nil {448			r.Name = strings.TrimSpace(*b.Name)449			set("name", r.Name)450		}451		if b.Description != nil {452			r.Description = *b.Description453		}454		if b.Enabled != nil {455			r.Enabled = *b.Enabled456			set("enabled", r.Enabled)457		}458		if b.Metric != nil {459			r.Metric = *b.Metric460			set("metric", r.Metric)461		}462		if b.LabelMatch != nil {463			r.LabelMatch = *b.LabelMatch464		}465		if b.Op != nil {466			op, ok := opToSQL[*b.Op]467			if !ok {468				return httpx.Validation("op must be gt, gte, lt or lte.")469			}470			r.Op = op471		}472		if b.Threshold != nil {473			r.Threshold = f64(b.Threshold)474			set("threshold", *r.Threshold)475		}476		if b.ClearThreshold != nil {477			r.ClearThreshold = f64(b.ClearThreshold)478		}479		if b.ForSeconds != nil {480			r.ForSeconds = *b.ForSeconds481			set("for_seconds", r.ForSeconds)482		}483		if b.ClearSeconds != nil {484			r.ClearSeconds = *b.ClearSeconds485		}486		if b.Severity != nil {487			r.Severity = string(*b.Severity)488			set("severity", r.Severity)489		}490		if b.Scope != nil {491			r.Scope = scopeIn(b.Scope)492			if err := h.scopeNodesExist(ctx, r.Scope.NodeIDs); err != nil {493				return err494			}495		}496		if b.Notify != nil {497			r.Notify = *b.Notify498			set("notify", r.Notify)499		}500		if err := r.Validate(); err != nil {501			return httpx.Validation("Invalid rule: " + err.Error() + ".")502		}503		if err := alerts.Save(ctx, tx, r, nil, false); err != nil {504			return err505		}506		if err := h.audit(ctx, tx, "alert_rule.update", "alert_rule", r.ID.String(), audit.Success, "", changed); err != nil {507			return err508		}509		nr, m, err := h.loadRule(ctx, tx, req.Id, "")510		out = ruleOut(nr, m)511		return err512	})513	if uniqueViolation(err) {514		return nil, httpx.Conflict("conflict", "A rule with this name already exists.")515	}516	if err != nil {517		return nil, err518	}519	return UpdateAlertRule200JSONResponse{Body: out}, nil520}521522// DeleteAlertRule removes a rule logically: its alert history is kept, its open alerts are resolved.523func (h *opsHandlers) DeleteAlertRule(ctx context.Context, req DeleteAlertRuleRequestObject) (DeleteAlertRuleResponseObject, error) {524	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {525		r, _, err := h.loadRule(ctx, tx, req.Id, " for update")526		if err != nil {527			return err528		}529		if r.Builtin {530			return httpx.Conflict("conflict", "Built-in rules cannot be deleted; disable them instead.")531		}532		now := nowUTC()533		rows, err := tx.Query(ctx, alertSelect+` where a.rule_id=$1 and a.state <> 'resolved' for update of a`, req.Id)534		if err != nil {535			return err536		}537		open, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Alert, error) { return scanAlert(r) })538		if err != nil {539			return err540		}541		for _, a := range open {542			if _, err := tx.Exec(ctx, `update alerts set state='resolved', resolved_at=$2 where id=$1`, a.Id, now); err != nil {543				return err544			}545			if _, err := h.addTimeline(ctx, tx, &a.Id, nil, "resolved", "rule deleted"); err != nil {546				return err547			}548			a.State = AlertStateResolved549			if err := publishAlert(ctx, tx, a); err != nil {550				return err551			}552		}553		if _, err := tx.Exec(ctx, `update alert_rules set deleted_at=$2, enabled=false, name=left(name,60) || ' (deleted ' || id::text || ')', updated_at=$2 where id=$1`, req.Id, now); err != nil {554			return err555		}556		if _, err := tx.Exec(ctx, `delete from silences where rule_id=$1`, req.Id); err != nil {557			return err558		}559		return h.audit(ctx, tx, "alert_rule.delete", "alert_rule", req.Id.String(), audit.Success, "", map[string]any{"name": r.Name, "resolved_alerts": len(open)})560	})561	if err != nil {562		return nil, err563	}564	return DeleteAlertRule204Response{}, nil565}566567// ValidateAlertRule replays a draft on history. Read-only.568func (h *opsHandlers) ValidateAlertRule(ctx context.Context, req ValidateAlertRuleRequestObject) (ValidateAlertRuleResponseObject, error) {569	if req.Body == nil {570		return nil, httpx.Validation("Body required.")571	}572	r, err := ruleFromInput(req.Body.Rule)573	if err != nil {574		return nil, httpx.Validation("Invalid rule: " + err.Error() + ".")575	}576	to := nowUTC()577	if req.Body.To != nil {578		to = *req.Body.To579	}580	from := to.Add(-7 * 24 * time.Hour)581	if req.Body.From != nil {582		from = *req.Body.From583	}584	if !to.After(from) || to.Sub(from) > alerts.MaxReplaySpan {585		return nil, httpx.Validation("The window must be positive and at most 31 days.")586	}587	rctx, cancel := context.WithTimeout(ctx, 20*time.Second)588	defer cancel()589	res, err := alerts.Replay(rctx, h.Pool, r, from, to)590	if errors.Is(err, alerts.ErrNotReplayable) {591		return nil, httpx.Err(422, "unprocessable", "This rule cannot be replayed: "+err.Error()+".")592	}593	if err != nil {594		return nil, err595	}596	out := AlertRuleValidation{EvaluatedFrom: res.From, EvaluatedTo: res.To, Resolution: Resolution(res.Resolution), NodesEvaluated: res.NodesEvaluated,597		NodesWithoutData: &res.NodesWithoutData, TotalFirings: len(res.Intervals), Intervals: []AlertFiringInterval{}, Warnings: res.Warnings}598	for _, iv := range res.Intervals {599		out.Intervals = append(out.Intervals, AlertFiringInterval{NodeId: iv.NodeID, NodeAlias: iv.NodeAlias, Label: iv.Label, StartedAt: iv.StartedAt,600			EndedAt: iv.EndedAt, PeakValue: ptr(float32(iv.Peak))})601	}602	return ValidateAlertRule200JSONResponse{Body: out}, nil603}604605// ── Silences ───────────────────────────────────────────────────────────────606607const silenceSelect = `select id, reason, scope, rule_id, starts_at, ends_at, maintenance_window_id, created_by, created_at from silences`608609func scanSilence(r pgx.Row, now time.Time) (Silence, error) {610	var s Silence611	var scope alerts.Scope612	err := r.Scan(&s.Id, &s.Reason, &scope, &s.RuleId, &s.StartsAt, &s.EndsAt, &s.MaintenanceWindowId, &s.CreatedBy, &s.CreatedAt)613	s.Scope = scopeOut(scope)614	s.Active = !now.Before(s.StartsAt) && now.Before(s.EndsAt)615	return s, err616}617618func (h *opsHandlers) ListSilences(ctx context.Context, req ListSilencesRequestObject) (ListSilencesResponseObject, error) {619	now := nowUTC()620	var w where621	if req.Params.Active != nil && *req.Params.Active {622		w.add("starts_at <= ? and ends_at > ?", now, now)623	}624	cur, err := uuidCursor(req.Params.Cursor)625	if err != nil {626		return nil, err627	}628	if cur != nil {629		w.add("id < ?", *cur)630	}631	limit := pageLimit(req.Params.Limit)632	rows, err := h.Pool.Query(ctx, silenceSelect+w.sql()+" order by id desc limit "+w.arg(limit+1), w.args...)633	if err != nil {634		return nil, err635	}636	items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Silence, error) { return scanSilence(r, now) })637	if err != nil {638		return nil, err639	}640	out := SilenceList{Items: items}641	if len(items) > limit {642		out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())643	}644	if out.Items == nil {645		out.Items = []Silence{}646	}647	return ListSilences200JSONResponse{Body: out}, nil648}649650func (h *opsHandlers) CreateSilence(ctx context.Context, req CreateSilenceRequestObject) (CreateSilenceResponseObject, error) {651	if req.Body == nil {652		return nil, httpx.Validation("Body required.")653	}654	b := req.Body655	now := nowUTC()656	reason := strings.TrimSpace(b.Reason)657	starts := now658	if b.StartsAt != nil {659		starts = *b.StartsAt660	}661	scope := scopeIn(&b.Scope)662	switch {663	case reason == "" || len(reason) > 500:664		return nil, httpx.Validation("reason is required (at most 500 characters).")665	case scope.Validate() != nil:666		return nil, httpx.Validation("Invalid scope: " + scope.Validate().Error() + ".")667	case !b.EndsAt.After(starts) || !b.EndsAt.After(now):668		return nil, httpx.Validation("ends_at must be in the future and after starts_at.")669	case b.EndsAt.Sub(starts) > 90*24*time.Hour:670		return nil, httpx.Validation("A silence lasts at most 90 days.")671	}672	if err := h.scopeNodesExist(ctx, scope.NodeIDs); err != nil {673		return nil, err674	}675	if b.RuleId != nil {676		if _, _, err := h.loadRule(ctx, h.Pool, *b.RuleId, ""); err != nil {677			return nil, httpx.Validation("rule_id does not exist.")678		}679	}680	id := ids.New()681	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {682		if _, err := tx.Exec(ctx, `insert into silences(id, reason, scope, rule_id, starts_at, ends_at, created_by, created_at) values ($1,$2,$3,$4,$5,$6,$7,$8)`,683			id, reason, scope, b.RuleId, starts, b.EndsAt, h.userID(ctx), now); err != nil {684			return err685		}686		return h.audit(ctx, tx, "silence.create", "silence", id.String(), audit.Success, "", map[string]any{"reason": reason, "ends_at": b.EndsAt, "all": scope.All})687	})688	if err != nil {689		return nil, err690	}691	s, err := scanSilence(h.Pool.QueryRow(ctx, silenceSelect+` where id=$1`, id), now)692	if err != nil {693		return nil, err694	}695	return CreateSilence201JSONResponse{Body: s}, nil696}697698func (h *opsHandlers) DeleteSilence(ctx context.Context, req DeleteSilenceRequestObject) (DeleteSilenceResponseObject, error) {699	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {700		var window *uuid.UUID701		if err := tx.QueryRow(ctx, `select maintenance_window_id from silences where id=$1 for update`, req.Id).Scan(&window); err != nil {702			return notFound(err)703		}704		if window != nil {705			return httpx.Conflict("conflict", "This silence belongs to a maintenance window; complete or cancel the window instead.")706		}707		if _, err := tx.Exec(ctx, `delete from silences where id=$1`, req.Id); err != nil {708			return err709		}710		return h.audit(ctx, tx, "silence.delete", "silence", req.Id.String(), audit.Success, "", nil)711	})712	if err != nil {713		return nil, err714	}715	return DeleteSilence204Response{}, nil716}717718// ── Incidents ──────────────────────────────────────────────────────────────719720const incidentSelect = `select i.id, i.title, i.severity::text, i.state, i.summary, i.owner_id, u.display_name, i.opened_at, i.resolved_at, i.created_by,721	(select count(*) from alerts a where a.incident_id=i.id) from incidents i left join users u on u.id=i.owner_id`722723func scanIncident(r pgx.Row) (Incident, error) {724	var i Incident725	var sev, state string726	err := r.Scan(&i.Id, &i.Title, &sev, &state, &i.Summary, &i.OwnerId, &i.OwnerDisplayName, &i.OpenedAt, &i.ResolvedAt, &i.CreatedBy, &i.AlertCount)727	i.Severity, i.State = Severity(sev), IncidentState(state)728	return i, err729}730731func publishIncident(ctx context.Context, tx pgx.Tx, i Incident) error {732	return events.Publish(ctx, tx, "incident.changed", map[string]any{"incident_id": i.Id, "state": i.State, "severity": i.Severity, "title": i.Title})733}734735func (h *opsHandlers) ListIncidents(ctx context.Context, req ListIncidentsRequestObject) (ListIncidentsResponseObject, error) {736	var w where737	if req.Params.State != nil {738		w.add("i.state = ?", string(*req.Params.State))739	}740	if req.Params.Severity != nil {741		w.add("i.severity = ?::severity", string(*req.Params.Severity))742	}743	cur, err := uuidCursor(req.Params.Cursor)744	if err != nil {745		return nil, err746	}747	if cur != nil {748		w.add("i.id < ?", *cur)749	}750	limit := pageLimit(req.Params.Limit)751	rows, err := h.Pool.Query(ctx, incidentSelect+w.sql()+" order by i.id desc limit "+w.arg(limit+1), w.args...)752	if err != nil {753		return nil, err754	}755	items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Incident, error) { return scanIncident(r) })756	if err != nil {757		return nil, err758	}759	out := IncidentList{Items: items}760	if len(items) > limit {761		out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())762	}763	if out.Items == nil {764		out.Items = []Incident{}765	}766	return ListIncidents200JSONResponse{Body: out}, nil767}768769func (h *opsHandlers) GetIncident(ctx context.Context, req GetIncidentRequestObject) (GetIncidentResponseObject, error) {770	i, err := scanIncident(h.Pool.QueryRow(ctx, incidentSelect+` where i.id=$1`, req.Id))771	if err != nil {772		return nil, notFound(err)773	}774	tl, err := h.timeline(ctx, "incident_id", req.Id)775	if err != nil {776		return nil, err777	}778	rows, err := h.Pool.Query(ctx, alertSelect+` where a.incident_id=$1 order by a.id desc limit 500`, req.Id)779	if err != nil {780		return nil, err781	}782	as, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Alert, error) { return scanAlert(r) })783	if err != nil {784		return nil, err785	}786	if as == nil {787		as = []Alert{}788	}789	return GetIncident200JSONResponse{Body: IncidentDetail{Incident: i, Timeline: tl, Alerts: as}}, nil790}791792// linkAlerts attaches (or detaches) alerts and records it on the incident timeline.793func (h *opsHandlers) linkAlerts(ctx context.Context, tx pgx.Tx, incident uuid.UUID, alertIDs []uuid.UUID, attach bool) error {794	for _, id := range alertIDs {795		a, err := scanAlert(tx.QueryRow(ctx, alertSelect+` where a.id=$1 for update of a`, id))796		if err != nil {797			return notFound(err)798		}799		kind := "alert_linked"800		if attach {801			_, err = tx.Exec(ctx, `update alerts set incident_id=$2 where id=$1`, id, incident)802			a.IncidentId = &incident803		} else {804			if a.IncidentId == nil || *a.IncidentId != incident {805				continue806			}807			kind = "alert_unlinked"808			_, err = tx.Exec(ctx, `update alerts set incident_id=null where id=$1`, id)809			a.IncidentId = nil810		}811		if err != nil {812			return err813		}814		if _, err := h.addTimeline(ctx, tx, nil, &incident, kind, a.Summary); err != nil {815			return err816		}817		if err := publishAlert(ctx, tx, a); err != nil {818			return err819		}820	}821	return nil822}823824func (h *opsHandlers) userExists(ctx context.Context, id uuid.UUID) error {825	var one int826	if err := h.Pool.QueryRow(ctx, `select 1 from users where id=$1 and disabled_at is null`, id).Scan(&one); err != nil {827		if errors.Is(err, pgx.ErrNoRows) {828			return httpx.Validation("owner_id is not an active user.")829		}830		return err831	}832	return nil833}834835func (h *opsHandlers) CreateIncident(ctx context.Context, req CreateIncidentRequestObject) (CreateIncidentResponseObject, error) {836	if req.Body == nil {837		return nil, httpx.Validation("Body required.")838	}839	b := req.Body840	title := strings.TrimSpace(b.Title)841	sev := string(b.Severity)842	if title == "" || len(title) > 160 || len(deref(b.Summary)) > 8000 || len(deref(b.AlertIds)) > 100 || (sev != "info" && sev != "warning" && sev != "critical") {843		return nil, httpx.Validation("title (1-160 characters) and severity are required; summary at most 8000 characters; at most 100 alerts.")844	}845	if b.OwnerId != nil {846		if err := h.userExists(ctx, *b.OwnerId); err != nil {847			return nil, err848		}849	}850	id := ids.New()851	var out Incident852	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {853		if _, err := tx.Exec(ctx, `insert into incidents(id, title, severity, summary, owner_id, opened_at, created_by) values ($1,$2,$3::severity,$4,$5,$6,$7)`,854			id, title, sev, deref(b.Summary), b.OwnerId, nowUTC(), h.userID(ctx)); err != nil {855			return err856		}857		if _, err := h.addTimeline(ctx, tx, nil, &id, "opened", title); err != nil {858			return err859		}860		if err := h.linkAlerts(ctx, tx, id, deref(b.AlertIds), true); err != nil {861			return err862		}863		var err error864		if out, err = scanIncident(tx.QueryRow(ctx, incidentSelect+` where i.id=$1`, id)); err != nil {865			return err866		}867		if err := publishIncident(ctx, tx, out); err != nil {868			return err869		}870		return h.audit(ctx, tx, "incident.create", "incident", id.String(), audit.Success, "", map[string]any{"title": title, "severity": sev, "alerts": len(deref(b.AlertIds))})871	})872	if err != nil {873		return nil, err874	}875	return CreateIncident201JSONResponse{Body: out}, nil876}877878func (h *opsHandlers) UpdateIncident(ctx context.Context, req UpdateIncidentRequestObject) (UpdateIncidentResponseObject, error) {879	if req.Body == nil {880		return nil, httpx.Validation("Body required.")881	}882	b := req.Body883	if b.Title != nil && (strings.TrimSpace(*b.Title) == "" || len(*b.Title) > 160) || len(deref(b.Summary)) > 8000 ||884		len(deref(b.AddAlertIds)) > 100 || len(deref(b.RemoveAlertIds)) > 100 {885		return nil, httpx.Validation("Invalid incident update.")886	}887	if b.OwnerId != nil {888		if err := h.userExists(ctx, *b.OwnerId); err != nil {889			return nil, err890		}891	}892	var out Incident893	err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {894		cur, err := scanIncident(tx.QueryRow(ctx, incidentSelect+` where i.id=$1 for update of i`, req.Id))895		if err != nil {896			return notFound(err)897		}898		now := nowUTC()899		changed := map[string]any{}900		if b.State != nil && *b.State != cur.State {901			st := string(*b.State)902			if st != "open" && st != "mitigated" && st != "resolved" {903				return httpx.Validation("state must be open, mitigated or resolved.")904			}905			var resolved *time.Time906			if st == "resolved" {907				resolved = &now908			}909			if _, err := tx.Exec(ctx, `update incidents set state=$2, resolved_at=$3 where id=$1`, req.Id, st, resolved); err != nil {910				return err911			}912			if _, err := h.addTimeline(ctx, tx, nil, &req.Id, "state_changed", string(cur.State)+" → "+st); err != nil {913				return err914			}915			changed["state"] = st916		}917		if b.OwnerId != nil && (cur.OwnerId == nil || *cur.OwnerId != *b.OwnerId) {918			if _, err := tx.Exec(ctx, `update incidents set owner_id=$2 where id=$1`, req.Id, *b.OwnerId); err != nil {919				return err920			}921			if _, err := h.addTimeline(ctx, tx, nil, &req.Id, "owner_changed", ""); err != nil {922				return err923			}924			changed["owner_id"] = b.OwnerId.String()925		}926		if b.Title != nil || b.Severity != nil || b.Summary != nil {927			var sev *string928			if b.Severity != nil {929				s := string(*b.Severity)930				if s != "info" && s != "warning" && s != "critical" {931					return httpx.Validation("Invalid severity.")932				}933				sev = &s934				changed["severity"] = s935			}936			var title *string937			if b.Title != nil {938				title = ptr(strings.TrimSpace(*b.Title))939				changed["title"] = *title940			}941			if _, err := tx.Exec(ctx, `update incidents set title=coalesce($2,title), severity=coalesce($3::severity,severity), summary=coalesce($4,summary) where id=$1`,942				req.Id, title, sev, b.Summary); err != nil {943				return err944			}945		}946		if err := h.linkAlerts(ctx, tx, req.Id, deref(b.AddAlertIds), true); err != nil {947			return err948		}949		if err := h.linkAlerts(ctx, tx, req.Id, deref(b.RemoveAlertIds), false); err != nil {950			return err951		}952		if out, err = scanIncident(tx.QueryRow(ctx, incidentSelect+` where i.id=$1`, req.Id)); err != nil {953			return err954		}955		if err := publishIncident(ctx, tx, out); err != nil {956			return err957		}958		return h.audit(ctx, tx, "incident.update", "incident", req.Id.String(), audit.Success, "", changed)959	})960	if err != nil {961		return nil, err962	}963	return UpdateIncident200JSONResponse{Body: out}, nil964}965966func (h *opsHandlers) CommentIncident(ctx context.Context, req CommentIncidentRequestObject) (CommentIncidentResponseObject, error) {967	body, err := commentBody(req.Body)968	if err != nil {969		return nil, err970	}971	var t TimelineEvent972	err = pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {973		var one int974		if err := tx.QueryRow(ctx, `select 1 from incidents where id=$1`, req.Id).Scan(&one); err != nil {975			return notFound(err)976		}977		if t, err = h.addTimeline(ctx, tx, nil, &req.Id, "comment", body); err != nil {978			return err979		}980		return h.audit(ctx, tx, "incident.comment", "incident", req.Id.String(), audit.Success, "", nil)981	})982	if err != nil {983		return nil, err984	}985	return CommentIncident201JSONResponse{Body: t}, nil986}987
Files main
Menu

Theme