Repository Public
1package api23import (4 "context"5 "errors"6 "strings"7 "time"89 "github.com/google/uuid"10 "github.com/jackc/pgx/v5"1112 "clustr.spboucher.ai/mcc/internal/alerts"13 "clustr.spboucher.ai/mcc/internal/audit"14 "clustr.spboucher.ai/mcc/internal/authz"15 "clustr.spboucher.ai/mcc/internal/events"16 "clustr.spboucher.ai/mcc/internal/httpx"17 "clustr.spboucher.ai/mcc/internal/ids"18)1920// Alerts, rules, silences and incidents. The alert engine (internal/alerts)21// owns the automatic transitions; this file holds the human ones.2223// ── Mapping ────────────────────────────────────────────────────────────────2425const alertSelect = `select a.id, a.rule_id, r.name, a.node_id, n.alias::text, a.label, a.fingerprint, a.state::text, a.severity::text, a.summary, a.last_value,26 a.started_at, a.fired_at, a.acknowledged_at, a.acknowledged_by, u.display_name, a.resolved_at, a.last_eval_at, a.flap_count, a.suppressed_by, a.incident_id27 from alerts a join alert_rules r on r.id=a.rule_id left join nodes n on n.id=a.node_id left join users u on u.id=a.acknowledged_by`2829func scanAlert(r pgx.Row) (Alert, error) {30 var a Alert31 var state, sev string32 var supp *string33 var val *float6434 err := r.Scan(&a.Id, &a.RuleId, &a.RuleName, &a.NodeId, &a.NodeAlias, &a.Label, &a.Fingerprint, &state, &sev, &a.Summary, &val,35 &a.StartedAt, &a.FiredAt, &a.AcknowledgedAt, &a.AcknowledgedBy, &a.AcknowledgedByDisplayName, &a.ResolvedAt, &a.LastEvalAt, &a.FlapCount, &supp, &a.IncidentId)36 a.State, a.Severity = AlertState(state), Severity(sev)37 if supp != nil {38 a.SuppressedBy = ptr(AlertSuppressedBy(*supp))39 }40 if val != nil {41 a.LastValue = ptr(float32(*val))42 }43 return a, err44}4546func alertNodeRes(a Alert) authz.Resource {47 if a.NodeId == nil {48 return authz.Resource{}49 }50 return authz.Resource{Kind: "node", ID: a.NodeId.String()}51}5253func (h *opsHandlers) timeline(ctx context.Context, col string, id uuid.UUID) ([]TimelineEvent, error) {54 rows, err := h.Pool.Query(ctx, `select t.id, t.ts, t.kind, t.actor_id, u.display_name, t.body from timeline_events t left join users u on u.id=t.actor_id55 where t.`+col+`=$1 order by t.ts, t.id limit 1000`, id)56 if err != nil {57 return nil, err58 }59 out, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (TimelineEvent, error) {60 var t TimelineEvent61 return t, r.Scan(&t.Id, &t.Ts, &t.Kind, &t.ActorId, &t.ActorDisplayName, &t.Body)62 })63 if out == nil {64 out = []TimelineEvent{}65 }66 return out, err67}6869func (h *opsHandlers) addTimeline(ctx context.Context, tx pgx.Tx, alert, incident *uuid.UUID, kind, body string) (TimelineEvent, error) {70 p := principalFrom(ctx)71 t := TimelineEvent{Id: ids.New(), Ts: nowUTC(), Kind: kind, Body: body, ActorId: &p.Session.User.ID, ActorDisplayName: &p.Session.User.DisplayName}72 _, err := tx.Exec(ctx, `insert into timeline_events(id, alert_id, incident_id, ts, kind, actor_id, body) values ($1,$2,$3,$4,$5,$6,$7)`,73 t.Id, alert, incident, t.Ts, kind, t.ActorId, body)74 return t, err75}7677func publishAlert(ctx context.Context, tx pgx.Tx, a Alert) error {78 p := map[string]any{"alert_id": a.Id, "rule_id": a.RuleId, "state": a.State, "severity": a.Severity, "summary": a.Summary, "suppressed": a.SuppressedBy != nil}79 if a.NodeId != nil {80 p["node_id"] = *a.NodeId81 }82 if a.IncidentId != nil {83 p["incident_id"] = *a.IncidentId84 }85 return events.Publish(ctx, tx, "alert.changed", p)86}8788// ── Alerts ─────────────────────────────────────────────────────────────────8990func (h *opsHandlers) ListAlerts(ctx context.Context, req ListAlertsRequestObject) (ListAlertsResponseObject, error) {91 p := req.Params92 var w where93 if p.State != nil {94 w.add("a.state = ?::alert_state", string(*p.State))95 }96 if p.Open != nil && *p.Open {97 w.add("a.state <> 'resolved'")98 }99 if p.Severity != nil {100 w.add("a.severity = ?::severity", string(*p.Severity))101 }102 if p.NodeId != nil {103 w.add("a.node_id = ?", *p.NodeId)104 }105 if p.RuleId != nil {106 w.add("a.rule_id = ?", *p.RuleId)107 }108 if p.IncidentId != nil {109 w.add("a.incident_id = ?", *p.IncidentId)110 }111 cur, err := uuidCursor(p.Cursor)112 if err != nil {113 return nil, err114 }115 if cur != nil {116 w.add("a.id < ?", *cur)117 }118 limit := pageLimit(p.Limit)119 rows, err := h.Pool.Query(ctx, alertSelect+w.sql()+" order by a.id desc limit "+w.arg(limit+1), w.args...)120 if err != nil {121 return nil, err122 }123 items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Alert, error) { return scanAlert(r) })124 if err != nil {125 return nil, err126 }127 out := AlertList{Items: items}128 if len(items) > limit {129 out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())130 }131 if out.Items == nil {132 out.Items = []Alert{}133 }134 return ListAlerts200JSONResponse{Body: out}, nil135}136137func (h *opsHandlers) GetAlert(ctx context.Context, req GetAlertRequestObject) (GetAlertResponseObject, error) {138 a, err := scanAlert(h.Pool.QueryRow(ctx, alertSelect+` where a.id=$1`, req.Id))139 if err != nil {140 return nil, notFound(err)141 }142 tl, err := h.timeline(ctx, "alert_id", req.Id)143 if err != nil {144 return nil, err145 }146 return GetAlert200JSONResponse{Body: AlertDetail{Alert: a, Timeline: tl}}, nil147}148149// transition is acknowledge and manual resolve.150func (h *opsHandlers) transition(ctx context.Context, id uuid.UUID, to string, body *AlertTransitionRequest) (Alert, error) {151 comment := ""152 if body != nil {153 comment = strings.TrimSpace(deref(body.Comment))154 }155 if len(comment) > 4000 {156 return Alert{}, httpx.Validation("comment is too long.")157 }158 var a Alert159 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {160 var err error161 if a, err = scanAlert(tx.QueryRow(ctx, alertSelect+` where a.id=$1 for update of a`, id)); err != nil {162 return notFound(err)163 }164 if err := h.authorize(ctx, authz.AlertsAck, alertNodeRes(a)); err != nil {165 return err166 }167 now, me := nowUTC(), h.userID(ctx)168 switch to {169 case "acknowledged":170 if a.State != AlertStateFiring {171 return httpx.Conflict("conflict", "Only a firing alert can be acknowledged.")172 }173 _, err = tx.Exec(ctx, `update alerts set state='acknowledged', acknowledged_at=$2, acknowledged_by=$3 where id=$1`, id, now, me)174 case "resolved":175 if a.State == AlertStateResolved {176 return httpx.Conflict("conflict", "The alert is already resolved.")177 }178 _, err = tx.Exec(ctx, `update alerts set state='resolved', resolved_at=$2 where id=$1`, id, now)179 if comment == "" {180 comment = "resolved manually"181 }182 }183 if err != nil {184 return err185 }186 if _, err := h.addTimeline(ctx, tx, &id, nil, to, comment); err != nil {187 return err188 }189 if a, err = scanAlert(tx.QueryRow(ctx, alertSelect+` where a.id=$1`, id)); err != nil {190 return err191 }192 if err := publishAlert(ctx, tx, a); err != nil {193 return err194 }195 return h.audit(ctx, tx, "alert."+map[string]string{"acknowledged": "acknowledge", "resolved": "resolve"}[to], "alert", id.String(), audit.Success, "",196 map[string]any{"rule": a.RuleName, "node": deref(a.NodeAlias)})197 })198 return a, err199}200201func (h *opsHandlers) AcknowledgeAlert(ctx context.Context, req AcknowledgeAlertRequestObject) (AcknowledgeAlertResponseObject, error) {202 a, err := h.transition(ctx, req.Id, "acknowledged", req.Body)203 if err != nil {204 return nil, err205 }206 return AcknowledgeAlert200JSONResponse{Body: a}, nil207}208209func (h *opsHandlers) ResolveAlert(ctx context.Context, req ResolveAlertRequestObject) (ResolveAlertResponseObject, error) {210 a, err := h.transition(ctx, req.Id, "resolved", req.Body)211 if err != nil {212 return nil, err213 }214 return ResolveAlert200JSONResponse{Body: a}, nil215}216217func commentBody(b *CommentRequest) (string, error) {218 if b == nil || strings.TrimSpace(b.Body) == "" || len(b.Body) > 4000 {219 return "", httpx.Validation("body is required (at most 4000 characters).")220 }221 return strings.TrimSpace(b.Body), nil222}223224func (h *opsHandlers) CommentAlert(ctx context.Context, req CommentAlertRequestObject) (CommentAlertResponseObject, error) {225 body, err := commentBody(req.Body)226 if err != nil {227 return nil, err228 }229 a, err := scanAlert(h.Pool.QueryRow(ctx, alertSelect+` where a.id=$1`, req.Id))230 if err != nil {231 return nil, notFound(err)232 }233 if err := h.authorize(ctx, authz.AlertsAck, alertNodeRes(a)); err != nil {234 return nil, err235 }236 var t TimelineEvent237 err = pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {238 if t, err = h.addTimeline(ctx, tx, &req.Id, nil, "comment", body); err != nil {239 return err240 }241 return h.audit(ctx, tx, "alert.comment", "alert", req.Id.String(), audit.Success, "", nil)242 })243 if err != nil {244 return nil, err245 }246 return CommentAlert201JSONResponse{Body: t}, nil247}248249// ── Rules ──────────────────────────────────────────────────────────────────250251var opToSQL = map[AlertOp]string{AlertOpGt: ">", AlertOpGte: ">=", AlertOpLt: "<", AlertOpLte: "<="}252var opFromSQL = map[string]AlertOp{">": AlertOpGt, ">=": AlertOpGte, "<": AlertOpLt, "<=": AlertOpLte}253254func f64(p *float32) *float64 {255 if p == nil {256 return nil257 }258 v := float64(*p)259 return &v260}261262func f32(p *float64) *float32 {263 if p == nil {264 return nil265 }266 v := float32(*p)267 return &v268}269270func scopeIn(s *AlertScope) alerts.Scope {271 if s == nil {272 return alerts.Scope{All: true}273 }274 return alerts.Scope{All: deref(s.All), NodeIDs: deref(s.NodeIds), Tags: deref(s.Tags)}275}276277func scopeOut(s alerts.Scope) AlertScope {278 var out AlertScope279 switch {280 case s.All:281 out.All = ptr(true)282 case len(s.NodeIDs) > 0:283 out.NodeIds = &s.NodeIDs284 case len(s.Tags) > 0:285 out.Tags = &s.Tags286 }287 return out288}289290func ruleFromInput(in AlertRuleInput) (alerts.Rule, error) {291 r := alerts.Rule{Name: strings.TrimSpace(in.Name), Description: deref(in.Description), Enabled: in.Enabled == nil || *in.Enabled, Kind: string(in.Kind),292 Metric: deref(in.Metric), LabelMatch: deref(in.LabelMatch), Threshold: f64(in.Threshold), ClearThreshold: f64(in.ClearThreshold),293 ForSeconds: 300, ClearSeconds: 120, Severity: "warning", Scope: scopeIn(in.Scope), Notify: in.Notify == nil || *in.Notify}294 if in.Op != nil {295 op, ok := opToSQL[*in.Op]296 if !ok {297 return r, errors.New("op must be gt, gte, lt or lte")298 }299 r.Op = op300 }301 if in.ForSeconds != nil {302 r.ForSeconds = *in.ForSeconds303 }304 if in.ClearSeconds != nil {305 r.ClearSeconds = *in.ClearSeconds306 }307 if in.Severity != nil {308 r.Severity = string(*in.Severity)309 }310 return r, r.Validate()311}312313type ruleMeta struct {314 createdBy *uuid.UUID315 created time.Time316 updated time.Time317}318319func ruleOut(r alerts.Rule, m ruleMeta) AlertRule {320 out := AlertRule{Id: r.ID, Name: r.Name, Description: r.Description, Enabled: r.Enabled, Builtin: r.Builtin, Kind: AlertRuleKind(r.Kind),321 Metric: nonEmpty(r.Metric), LabelMatch: nonEmpty(r.LabelMatch), Threshold: f32(r.Threshold), ClearThreshold: f32(r.ClearThreshold),322 ForSeconds: r.ForSeconds, ClearSeconds: r.ClearSeconds, Severity: Severity(r.Severity), Scope: scopeOut(r.Scope), Notify: r.Notify,323 CreatedBy: m.createdBy, CreatedAt: m.created, UpdatedAt: m.updated}324 if op, ok := opFromSQL[r.Op]; ok {325 out.Op = &op326 }327 return out328}329330const ruleSelect = `select ` + alerts.RuleColumns + `, created_by, created_at, updated_at from alert_rules`331332func (h *opsHandlers) loadRule(ctx context.Context, q interface {333 QueryRow(context.Context, string, ...any) pgx.Row334}, id uuid.UUID, lock string) (alerts.Rule, ruleMeta, error) {335 var m ruleMeta336 r, err := alerts.ScanRule(q.QueryRow(ctx, ruleSelect+` where id=$1 and deleted_at is null`+lock, id), &m.createdBy, &m.created, &m.updated)337 return r, m, notFound(err)338}339340// scopeNodesExist refuses a scope that names unknown nodes.341func (h *opsHandlers) scopeNodesExist(ctx context.Context, nodes []uuid.UUID) error {342 if len(nodes) == 0 {343 return nil344 }345 var n int346 if err := h.Pool.QueryRow(ctx, `select count(distinct id) from nodes where id = any($1) and retired_at is null`, nodes).Scan(&n); err != nil {347 return err348 }349 if n != len(nodes) {350 return httpx.Validation("The scope names a node that does not exist (or repeats one).")351 }352 return nil353}354355func (h *opsHandlers) ListAlertRules(ctx context.Context, req ListAlertRulesRequestObject) (ListAlertRulesResponseObject, error) {356 var w where357 w.add("deleted_at is null")358 if req.Params.Enabled != nil {359 w.add("enabled = ?", *req.Params.Enabled)360 }361 cur, err := uuidCursor(req.Params.Cursor)362 if err != nil {363 return nil, err364 }365 if cur != nil {366 w.add("id > ?", *cur)367 }368 limit := pageLimit(req.Params.Limit)369 rows, err := h.Pool.Query(ctx, ruleSelect+w.sql()+" order by id limit "+w.arg(limit+1), w.args...)370 if err != nil {371 return nil, err372 }373 items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (AlertRule, error) {374 var m ruleMeta375 rule, err := alerts.ScanRule(r, &m.createdBy, &m.created, &m.updated)376 return ruleOut(rule, m), err377 })378 if err != nil {379 return nil, err380 }381 out := AlertRuleList{Items: items}382 if len(items) > limit {383 out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())384 }385 if out.Items == nil {386 out.Items = []AlertRule{}387 }388 return ListAlertRules200JSONResponse{Body: out}, nil389}390391func (h *opsHandlers) GetAlertRule(ctx context.Context, req GetAlertRuleRequestObject) (GetAlertRuleResponseObject, error) {392 r, m, err := h.loadRule(ctx, h.Pool, req.Id, "")393 if err != nil {394 return nil, err395 }396 return GetAlertRule200JSONResponse{Body: ruleOut(r, m)}, nil397}398399func (h *opsHandlers) CreateAlertRule(ctx context.Context, req CreateAlertRuleRequestObject) (CreateAlertRuleResponseObject, error) {400 if req.Body == nil {401 return nil, httpx.Validation("Body required.")402 }403 r, err := ruleFromInput(*req.Body)404 if err != nil {405 return nil, httpx.Validation("Invalid rule: " + err.Error() + ".")406 }407 if err := h.scopeNodesExist(ctx, r.Scope.NodeIDs); err != nil {408 return nil, err409 }410 r.ID = ids.New()411 me := h.userID(ctx)412 err = pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {413 if err := alerts.Save(ctx, tx, r, &me, true); err != nil {414 return err415 }416 return h.audit(ctx, tx, "alert_rule.create", "alert_rule", r.ID.String(), audit.Success, "", map[string]any{"name": r.Name, "kind": r.Kind, "severity": r.Severity})417 })418 if uniqueViolation(err) {419 return nil, httpx.Conflict("conflict", "A rule with this name already exists.")420 }421 if err != nil {422 return nil, err423 }424 r, m, err := h.loadRule(ctx, h.Pool, r.ID, "")425 if err != nil {426 return nil, err427 }428 return CreateAlertRule201JSONResponse{Body: ruleOut(r, m)}, nil429}430431func (h *opsHandlers) UpdateAlertRule(ctx context.Context, req UpdateAlertRuleRequestObject) (UpdateAlertRuleResponseObject, error) {432 if req.Body == nil {433 return nil, httpx.Validation("Body required.")434 }435 b := req.Body436 var out AlertRule437 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {438 r, _, err := h.loadRule(ctx, tx, req.Id, " for update")439 if err != nil {440 return err441 }442 if r.Builtin && (b.Name != nil || b.Description != nil || b.Metric != nil || b.LabelMatch != nil || b.Op != nil) {443 return httpx.Conflict("conflict", "A built-in rule only accepts changes to enabled, thresholds, durations, severity, scope and notify.")444 }445 changed := map[string]any{}446 set := func(k string, v any) { changed[k] = v }447 if b.Name != nil {448 r.Name = strings.TrimSpace(*b.Name)449 set("name", r.Name)450 }451 if b.Description != nil {452 r.Description = *b.Description453 }454 if b.Enabled != nil {455 r.Enabled = *b.Enabled456 set("enabled", r.Enabled)457 }458 if b.Metric != nil {459 r.Metric = *b.Metric460 set("metric", r.Metric)461 }462 if b.LabelMatch != nil {463 r.LabelMatch = *b.LabelMatch464 }465 if b.Op != nil {466 op, ok := opToSQL[*b.Op]467 if !ok {468 return httpx.Validation("op must be gt, gte, lt or lte.")469 }470 r.Op = op471 }472 if b.Threshold != nil {473 r.Threshold = f64(b.Threshold)474 set("threshold", *r.Threshold)475 }476 if b.ClearThreshold != nil {477 r.ClearThreshold = f64(b.ClearThreshold)478 }479 if b.ForSeconds != nil {480 r.ForSeconds = *b.ForSeconds481 set("for_seconds", r.ForSeconds)482 }483 if b.ClearSeconds != nil {484 r.ClearSeconds = *b.ClearSeconds485 }486 if b.Severity != nil {487 r.Severity = string(*b.Severity)488 set("severity", r.Severity)489 }490 if b.Scope != nil {491 r.Scope = scopeIn(b.Scope)492 if err := h.scopeNodesExist(ctx, r.Scope.NodeIDs); err != nil {493 return err494 }495 }496 if b.Notify != nil {497 r.Notify = *b.Notify498 set("notify", r.Notify)499 }500 if err := r.Validate(); err != nil {501 return httpx.Validation("Invalid rule: " + err.Error() + ".")502 }503 if err := alerts.Save(ctx, tx, r, nil, false); err != nil {504 return err505 }506 if err := h.audit(ctx, tx, "alert_rule.update", "alert_rule", r.ID.String(), audit.Success, "", changed); err != nil {507 return err508 }509 nr, m, err := h.loadRule(ctx, tx, req.Id, "")510 out = ruleOut(nr, m)511 return err512 })513 if uniqueViolation(err) {514 return nil, httpx.Conflict("conflict", "A rule with this name already exists.")515 }516 if err != nil {517 return nil, err518 }519 return UpdateAlertRule200JSONResponse{Body: out}, nil520}521522// DeleteAlertRule removes a rule logically: its alert history is kept, its open alerts are resolved.523func (h *opsHandlers) DeleteAlertRule(ctx context.Context, req DeleteAlertRuleRequestObject) (DeleteAlertRuleResponseObject, error) {524 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {525 r, _, err := h.loadRule(ctx, tx, req.Id, " for update")526 if err != nil {527 return err528 }529 if r.Builtin {530 return httpx.Conflict("conflict", "Built-in rules cannot be deleted; disable them instead.")531 }532 now := nowUTC()533 rows, err := tx.Query(ctx, alertSelect+` where a.rule_id=$1 and a.state <> 'resolved' for update of a`, req.Id)534 if err != nil {535 return err536 }537 open, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Alert, error) { return scanAlert(r) })538 if err != nil {539 return err540 }541 for _, a := range open {542 if _, err := tx.Exec(ctx, `update alerts set state='resolved', resolved_at=$2 where id=$1`, a.Id, now); err != nil {543 return err544 }545 if _, err := h.addTimeline(ctx, tx, &a.Id, nil, "resolved", "rule deleted"); err != nil {546 return err547 }548 a.State = AlertStateResolved549 if err := publishAlert(ctx, tx, a); err != nil {550 return err551 }552 }553 if _, err := tx.Exec(ctx, `update alert_rules set deleted_at=$2, enabled=false, name=left(name,60) || ' (deleted ' || id::text || ')', updated_at=$2 where id=$1`, req.Id, now); err != nil {554 return err555 }556 if _, err := tx.Exec(ctx, `delete from silences where rule_id=$1`, req.Id); err != nil {557 return err558 }559 return h.audit(ctx, tx, "alert_rule.delete", "alert_rule", req.Id.String(), audit.Success, "", map[string]any{"name": r.Name, "resolved_alerts": len(open)})560 })561 if err != nil {562 return nil, err563 }564 return DeleteAlertRule204Response{}, nil565}566567// ValidateAlertRule replays a draft on history. Read-only.568func (h *opsHandlers) ValidateAlertRule(ctx context.Context, req ValidateAlertRuleRequestObject) (ValidateAlertRuleResponseObject, error) {569 if req.Body == nil {570 return nil, httpx.Validation("Body required.")571 }572 r, err := ruleFromInput(req.Body.Rule)573 if err != nil {574 return nil, httpx.Validation("Invalid rule: " + err.Error() + ".")575 }576 to := nowUTC()577 if req.Body.To != nil {578 to = *req.Body.To579 }580 from := to.Add(-7 * 24 * time.Hour)581 if req.Body.From != nil {582 from = *req.Body.From583 }584 if !to.After(from) || to.Sub(from) > alerts.MaxReplaySpan {585 return nil, httpx.Validation("The window must be positive and at most 31 days.")586 }587 rctx, cancel := context.WithTimeout(ctx, 20*time.Second)588 defer cancel()589 res, err := alerts.Replay(rctx, h.Pool, r, from, to)590 if errors.Is(err, alerts.ErrNotReplayable) {591 return nil, httpx.Err(422, "unprocessable", "This rule cannot be replayed: "+err.Error()+".")592 }593 if err != nil {594 return nil, err595 }596 out := AlertRuleValidation{EvaluatedFrom: res.From, EvaluatedTo: res.To, Resolution: Resolution(res.Resolution), NodesEvaluated: res.NodesEvaluated,597 NodesWithoutData: &res.NodesWithoutData, TotalFirings: len(res.Intervals), Intervals: []AlertFiringInterval{}, Warnings: res.Warnings}598 for _, iv := range res.Intervals {599 out.Intervals = append(out.Intervals, AlertFiringInterval{NodeId: iv.NodeID, NodeAlias: iv.NodeAlias, Label: iv.Label, StartedAt: iv.StartedAt,600 EndedAt: iv.EndedAt, PeakValue: ptr(float32(iv.Peak))})601 }602 return ValidateAlertRule200JSONResponse{Body: out}, nil603}604605// ── Silences ───────────────────────────────────────────────────────────────606607const silenceSelect = `select id, reason, scope, rule_id, starts_at, ends_at, maintenance_window_id, created_by, created_at from silences`608609func scanSilence(r pgx.Row, now time.Time) (Silence, error) {610 var s Silence611 var scope alerts.Scope612 err := r.Scan(&s.Id, &s.Reason, &scope, &s.RuleId, &s.StartsAt, &s.EndsAt, &s.MaintenanceWindowId, &s.CreatedBy, &s.CreatedAt)613 s.Scope = scopeOut(scope)614 s.Active = !now.Before(s.StartsAt) && now.Before(s.EndsAt)615 return s, err616}617618func (h *opsHandlers) ListSilences(ctx context.Context, req ListSilencesRequestObject) (ListSilencesResponseObject, error) {619 now := nowUTC()620 var w where621 if req.Params.Active != nil && *req.Params.Active {622 w.add("starts_at <= ? and ends_at > ?", now, now)623 }624 cur, err := uuidCursor(req.Params.Cursor)625 if err != nil {626 return nil, err627 }628 if cur != nil {629 w.add("id < ?", *cur)630 }631 limit := pageLimit(req.Params.Limit)632 rows, err := h.Pool.Query(ctx, silenceSelect+w.sql()+" order by id desc limit "+w.arg(limit+1), w.args...)633 if err != nil {634 return nil, err635 }636 items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Silence, error) { return scanSilence(r, now) })637 if err != nil {638 return nil, err639 }640 out := SilenceList{Items: items}641 if len(items) > limit {642 out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())643 }644 if out.Items == nil {645 out.Items = []Silence{}646 }647 return ListSilences200JSONResponse{Body: out}, nil648}649650func (h *opsHandlers) CreateSilence(ctx context.Context, req CreateSilenceRequestObject) (CreateSilenceResponseObject, error) {651 if req.Body == nil {652 return nil, httpx.Validation("Body required.")653 }654 b := req.Body655 now := nowUTC()656 reason := strings.TrimSpace(b.Reason)657 starts := now658 if b.StartsAt != nil {659 starts = *b.StartsAt660 }661 scope := scopeIn(&b.Scope)662 switch {663 case reason == "" || len(reason) > 500:664 return nil, httpx.Validation("reason is required (at most 500 characters).")665 case scope.Validate() != nil:666 return nil, httpx.Validation("Invalid scope: " + scope.Validate().Error() + ".")667 case !b.EndsAt.After(starts) || !b.EndsAt.After(now):668 return nil, httpx.Validation("ends_at must be in the future and after starts_at.")669 case b.EndsAt.Sub(starts) > 90*24*time.Hour:670 return nil, httpx.Validation("A silence lasts at most 90 days.")671 }672 if err := h.scopeNodesExist(ctx, scope.NodeIDs); err != nil {673 return nil, err674 }675 if b.RuleId != nil {676 if _, _, err := h.loadRule(ctx, h.Pool, *b.RuleId, ""); err != nil {677 return nil, httpx.Validation("rule_id does not exist.")678 }679 }680 id := ids.New()681 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {682 if _, err := tx.Exec(ctx, `insert into silences(id, reason, scope, rule_id, starts_at, ends_at, created_by, created_at) values ($1,$2,$3,$4,$5,$6,$7,$8)`,683 id, reason, scope, b.RuleId, starts, b.EndsAt, h.userID(ctx), now); err != nil {684 return err685 }686 return h.audit(ctx, tx, "silence.create", "silence", id.String(), audit.Success, "", map[string]any{"reason": reason, "ends_at": b.EndsAt, "all": scope.All})687 })688 if err != nil {689 return nil, err690 }691 s, err := scanSilence(h.Pool.QueryRow(ctx, silenceSelect+` where id=$1`, id), now)692 if err != nil {693 return nil, err694 }695 return CreateSilence201JSONResponse{Body: s}, nil696}697698func (h *opsHandlers) DeleteSilence(ctx context.Context, req DeleteSilenceRequestObject) (DeleteSilenceResponseObject, error) {699 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {700 var window *uuid.UUID701 if err := tx.QueryRow(ctx, `select maintenance_window_id from silences where id=$1 for update`, req.Id).Scan(&window); err != nil {702 return notFound(err)703 }704 if window != nil {705 return httpx.Conflict("conflict", "This silence belongs to a maintenance window; complete or cancel the window instead.")706 }707 if _, err := tx.Exec(ctx, `delete from silences where id=$1`, req.Id); err != nil {708 return err709 }710 return h.audit(ctx, tx, "silence.delete", "silence", req.Id.String(), audit.Success, "", nil)711 })712 if err != nil {713 return nil, err714 }715 return DeleteSilence204Response{}, nil716}717718// ── Incidents ──────────────────────────────────────────────────────────────719720const incidentSelect = `select i.id, i.title, i.severity::text, i.state, i.summary, i.owner_id, u.display_name, i.opened_at, i.resolved_at, i.created_by,721 (select count(*) from alerts a where a.incident_id=i.id) from incidents i left join users u on u.id=i.owner_id`722723func scanIncident(r pgx.Row) (Incident, error) {724 var i Incident725 var sev, state string726 err := r.Scan(&i.Id, &i.Title, &sev, &state, &i.Summary, &i.OwnerId, &i.OwnerDisplayName, &i.OpenedAt, &i.ResolvedAt, &i.CreatedBy, &i.AlertCount)727 i.Severity, i.State = Severity(sev), IncidentState(state)728 return i, err729}730731func publishIncident(ctx context.Context, tx pgx.Tx, i Incident) error {732 return events.Publish(ctx, tx, "incident.changed", map[string]any{"incident_id": i.Id, "state": i.State, "severity": i.Severity, "title": i.Title})733}734735func (h *opsHandlers) ListIncidents(ctx context.Context, req ListIncidentsRequestObject) (ListIncidentsResponseObject, error) {736 var w where737 if req.Params.State != nil {738 w.add("i.state = ?", string(*req.Params.State))739 }740 if req.Params.Severity != nil {741 w.add("i.severity = ?::severity", string(*req.Params.Severity))742 }743 cur, err := uuidCursor(req.Params.Cursor)744 if err != nil {745 return nil, err746 }747 if cur != nil {748 w.add("i.id < ?", *cur)749 }750 limit := pageLimit(req.Params.Limit)751 rows, err := h.Pool.Query(ctx, incidentSelect+w.sql()+" order by i.id desc limit "+w.arg(limit+1), w.args...)752 if err != nil {753 return nil, err754 }755 items, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Incident, error) { return scanIncident(r) })756 if err != nil {757 return nil, err758 }759 out := IncidentList{Items: items}760 if len(items) > limit {761 out.Items, out.NextCursor = items[:limit], ptr(items[limit-1].Id.String())762 }763 if out.Items == nil {764 out.Items = []Incident{}765 }766 return ListIncidents200JSONResponse{Body: out}, nil767}768769func (h *opsHandlers) GetIncident(ctx context.Context, req GetIncidentRequestObject) (GetIncidentResponseObject, error) {770 i, err := scanIncident(h.Pool.QueryRow(ctx, incidentSelect+` where i.id=$1`, req.Id))771 if err != nil {772 return nil, notFound(err)773 }774 tl, err := h.timeline(ctx, "incident_id", req.Id)775 if err != nil {776 return nil, err777 }778 rows, err := h.Pool.Query(ctx, alertSelect+` where a.incident_id=$1 order by a.id desc limit 500`, req.Id)779 if err != nil {780 return nil, err781 }782 as, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Alert, error) { return scanAlert(r) })783 if err != nil {784 return nil, err785 }786 if as == nil {787 as = []Alert{}788 }789 return GetIncident200JSONResponse{Body: IncidentDetail{Incident: i, Timeline: tl, Alerts: as}}, nil790}791792// linkAlerts attaches (or detaches) alerts and records it on the incident timeline.793func (h *opsHandlers) linkAlerts(ctx context.Context, tx pgx.Tx, incident uuid.UUID, alertIDs []uuid.UUID, attach bool) error {794 for _, id := range alertIDs {795 a, err := scanAlert(tx.QueryRow(ctx, alertSelect+` where a.id=$1 for update of a`, id))796 if err != nil {797 return notFound(err)798 }799 kind := "alert_linked"800 if attach {801 _, err = tx.Exec(ctx, `update alerts set incident_id=$2 where id=$1`, id, incident)802 a.IncidentId = &incident803 } else {804 if a.IncidentId == nil || *a.IncidentId != incident {805 continue806 }807 kind = "alert_unlinked"808 _, err = tx.Exec(ctx, `update alerts set incident_id=null where id=$1`, id)809 a.IncidentId = nil810 }811 if err != nil {812 return err813 }814 if _, err := h.addTimeline(ctx, tx, nil, &incident, kind, a.Summary); err != nil {815 return err816 }817 if err := publishAlert(ctx, tx, a); err != nil {818 return err819 }820 }821 return nil822}823824func (h *opsHandlers) userExists(ctx context.Context, id uuid.UUID) error {825 var one int826 if err := h.Pool.QueryRow(ctx, `select 1 from users where id=$1 and disabled_at is null`, id).Scan(&one); err != nil {827 if errors.Is(err, pgx.ErrNoRows) {828 return httpx.Validation("owner_id is not an active user.")829 }830 return err831 }832 return nil833}834835func (h *opsHandlers) CreateIncident(ctx context.Context, req CreateIncidentRequestObject) (CreateIncidentResponseObject, error) {836 if req.Body == nil {837 return nil, httpx.Validation("Body required.")838 }839 b := req.Body840 title := strings.TrimSpace(b.Title)841 sev := string(b.Severity)842 if title == "" || len(title) > 160 || len(deref(b.Summary)) > 8000 || len(deref(b.AlertIds)) > 100 || (sev != "info" && sev != "warning" && sev != "critical") {843 return nil, httpx.Validation("title (1-160 characters) and severity are required; summary at most 8000 characters; at most 100 alerts.")844 }845 if b.OwnerId != nil {846 if err := h.userExists(ctx, *b.OwnerId); err != nil {847 return nil, err848 }849 }850 id := ids.New()851 var out Incident852 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {853 if _, err := tx.Exec(ctx, `insert into incidents(id, title, severity, summary, owner_id, opened_at, created_by) values ($1,$2,$3::severity,$4,$5,$6,$7)`,854 id, title, sev, deref(b.Summary), b.OwnerId, nowUTC(), h.userID(ctx)); err != nil {855 return err856 }857 if _, err := h.addTimeline(ctx, tx, nil, &id, "opened", title); err != nil {858 return err859 }860 if err := h.linkAlerts(ctx, tx, id, deref(b.AlertIds), true); err != nil {861 return err862 }863 var err error864 if out, err = scanIncident(tx.QueryRow(ctx, incidentSelect+` where i.id=$1`, id)); err != nil {865 return err866 }867 if err := publishIncident(ctx, tx, out); err != nil {868 return err869 }870 return h.audit(ctx, tx, "incident.create", "incident", id.String(), audit.Success, "", map[string]any{"title": title, "severity": sev, "alerts": len(deref(b.AlertIds))})871 })872 if err != nil {873 return nil, err874 }875 return CreateIncident201JSONResponse{Body: out}, nil876}877878func (h *opsHandlers) UpdateIncident(ctx context.Context, req UpdateIncidentRequestObject) (UpdateIncidentResponseObject, error) {879 if req.Body == nil {880 return nil, httpx.Validation("Body required.")881 }882 b := req.Body883 if b.Title != nil && (strings.TrimSpace(*b.Title) == "" || len(*b.Title) > 160) || len(deref(b.Summary)) > 8000 ||884 len(deref(b.AddAlertIds)) > 100 || len(deref(b.RemoveAlertIds)) > 100 {885 return nil, httpx.Validation("Invalid incident update.")886 }887 if b.OwnerId != nil {888 if err := h.userExists(ctx, *b.OwnerId); err != nil {889 return nil, err890 }891 }892 var out Incident893 err := pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {894 cur, err := scanIncident(tx.QueryRow(ctx, incidentSelect+` where i.id=$1 for update of i`, req.Id))895 if err != nil {896 return notFound(err)897 }898 now := nowUTC()899 changed := map[string]any{}900 if b.State != nil && *b.State != cur.State {901 st := string(*b.State)902 if st != "open" && st != "mitigated" && st != "resolved" {903 return httpx.Validation("state must be open, mitigated or resolved.")904 }905 var resolved *time.Time906 if st == "resolved" {907 resolved = &now908 }909 if _, err := tx.Exec(ctx, `update incidents set state=$2, resolved_at=$3 where id=$1`, req.Id, st, resolved); err != nil {910 return err911 }912 if _, err := h.addTimeline(ctx, tx, nil, &req.Id, "state_changed", string(cur.State)+" → "+st); err != nil {913 return err914 }915 changed["state"] = st916 }917 if b.OwnerId != nil && (cur.OwnerId == nil || *cur.OwnerId != *b.OwnerId) {918 if _, err := tx.Exec(ctx, `update incidents set owner_id=$2 where id=$1`, req.Id, *b.OwnerId); err != nil {919 return err920 }921 if _, err := h.addTimeline(ctx, tx, nil, &req.Id, "owner_changed", ""); err != nil {922 return err923 }924 changed["owner_id"] = b.OwnerId.String()925 }926 if b.Title != nil || b.Severity != nil || b.Summary != nil {927 var sev *string928 if b.Severity != nil {929 s := string(*b.Severity)930 if s != "info" && s != "warning" && s != "critical" {931 return httpx.Validation("Invalid severity.")932 }933 sev = &s934 changed["severity"] = s935 }936 var title *string937 if b.Title != nil {938 title = ptr(strings.TrimSpace(*b.Title))939 changed["title"] = *title940 }941 if _, err := tx.Exec(ctx, `update incidents set title=coalesce($2,title), severity=coalesce($3::severity,severity), summary=coalesce($4,summary) where id=$1`,942 req.Id, title, sev, b.Summary); err != nil {943 return err944 }945 }946 if err := h.linkAlerts(ctx, tx, req.Id, deref(b.AddAlertIds), true); err != nil {947 return err948 }949 if err := h.linkAlerts(ctx, tx, req.Id, deref(b.RemoveAlertIds), false); err != nil {950 return err951 }952 if out, err = scanIncident(tx.QueryRow(ctx, incidentSelect+` where i.id=$1`, req.Id)); err != nil {953 return err954 }955 if err := publishIncident(ctx, tx, out); err != nil {956 return err957 }958 return h.audit(ctx, tx, "incident.update", "incident", req.Id.String(), audit.Success, "", changed)959 })960 if err != nil {961 return nil, err962 }963 return UpdateIncident200JSONResponse{Body: out}, nil964}965966func (h *opsHandlers) CommentIncident(ctx context.Context, req CommentIncidentRequestObject) (CommentIncidentResponseObject, error) {967 body, err := commentBody(req.Body)968 if err != nil {969 return nil, err970 }971 var t TimelineEvent972 err = pgx.BeginFunc(ctx, h.Pool, func(tx pgx.Tx) error {973 var one int974 if err := tx.QueryRow(ctx, `select 1 from incidents where id=$1`, req.Id).Scan(&one); err != nil {975 return notFound(err)976 }977 if t, err = h.addTimeline(ctx, tx, nil, &req.Id, "comment", body); err != nil {978 return err979 }980 return h.audit(ctx, tx, "incident.comment", "incident", req.Id.String(), audit.Success, "", nil)981 })982 if err != nil {983 return nil, err984 }985 return CommentIncident201JSONResponse{Body: t}, nil986}987