Repository Public
1// Package notify delivers alert notifications. The SSE event is produced by2// the alert engine itself (events_outbox); this package pushes to Apple3// devices through APNs, from the durable queue alert_notifications.4package notify56import (7 "context"8 "errors"9 "fmt"10 "time"1112 "github.com/google/uuid"13 "github.com/jackc/pgx/v5"14 "github.com/jackc/pgx/v5/pgxpool"1516 "clustr.spboucher.ai/mcc/internal/authz"17 "clustr.spboucher.ai/mcc/internal/ids"18)1920// Dispatcher drains alert_notifications. A notification is attempted at most21// MaxAttempts times, then dropped as failed; one older than MaxAge is dropped22// as superseded (a late push about an old transition would only mislead).23type Dispatcher struct {24 Pool *pgxpool.Pool25 APNs *APNs26 Now func() time.Time27 MaxAttempts int // default 428 MaxAge time.Duration // default 30 min29}3031func (d *Dispatcher) now() time.Time {32 if d.Now != nil {33 return d.Now()34 }35 return time.Now().UTC()36}3738type recipient struct {39 device uuid.UUID40 token string41 env string42 kind string43}4445// recipients: non-revoked Apple devices with a token whose user is active and holds alerts.read (on that node, when there is one).46func (d *Dispatcher) recipients(ctx context.Context, tx pgx.Tx, node *uuid.UUID) ([]recipient, error) {47 grows, err := tx.Query(ctx, `select subject_kind, subject, permission, coalesce(resource_kind,''), coalesce(resource_id,''), effect::text from policy_grants where permission='alerts.read'`)48 if err != nil {49 return nil, err50 }51 grants, err := pgx.CollectRows(grows, func(r pgx.CollectableRow) (authz.Grant, error) {52 var g authz.Grant53 var perm, eff string54 err := r.Scan(&g.SubjectKind, &g.Subject, &perm, &g.ResourceKind, &g.ResourceID, &eff)55 g.Permission, g.Effect = authz.Permission(perm), authz.Effect(eff)56 return g, err57 })58 if err != nil {59 return nil, err60 }61 rows, err := tx.Query(ctx, `select d.id, d.push_token, coalesce(d.push_env,'production'), d.kind::text, u.id, u.role::text62 from devices d join users u on u.id=d.user_id63 where d.revoked_at is null and d.push_token is not null and d.push_token <> '' and d.kind in ('ios','macos') and u.disabled_at is null`)64 if err != nil {65 return nil, err66 }67 defer rows.Close()68 var out []recipient69 for rows.Next() {70 var r recipient71 var user uuid.UUID72 var role string73 if err := rows.Scan(&r.device, &r.token, &r.env, &r.kind, &user, &role); err != nil {74 return nil, err75 }76 sub := authz.Subject{UserID: user.String(), Role: authz.Role(role)}77 ok := authz.Decide(sub, authz.AlertsRead, authz.Resource{}, grants)78 if ok && node != nil {79 ok = authz.Decide(sub, authz.AlertsRead, authz.Resource{Kind: "node", ID: node.String()}, grants)80 }81 if ok {82 out = append(out, r)83 }84 }85 return out, rows.Err()86}8788// Drain processes what is due. It returns the number of notifications handled.89func (d *Dispatcher) Drain(ctx context.Context) (int, error) {90 maxAttempts, maxAge := d.MaxAttempts, d.MaxAge91 if maxAttempts <= 0 {92 maxAttempts = 493 }94 if maxAge <= 0 {95 maxAge = 30 * time.Minute96 }97 handled := 098 for handled < 200 {99 more, err := d.one(ctx, maxAttempts, maxAge)100 if err != nil {101 return handled, err102 }103 if !more {104 break105 }106 handled++107 }108 return handled, nil109}110111func (d *Dispatcher) one(ctx context.Context, maxAttempts int, maxAge time.Duration) (bool, error) {112 now := d.now()113 tx, err := d.Pool.Begin(ctx)114 if err != nil {115 return false, err116 }117 defer tx.Rollback(ctx)118 var (119 id int64120 alertID uuid.UUID121 transition string122 created time.Time123 attempts int124 severity string125 state string126 suppressed *string127 node *uuid.UUID128 alias *string129 )130 err = tx.QueryRow(ctx, `select q.id, q.alert_id, q.transition, q.created_at, q.attempts, a.severity::text, a.state::text, a.suppressed_by, a.node_id, n.alias::text131 from alert_notifications q join alerts a on a.id=q.alert_id left join nodes n on n.id=a.node_id132 where q.done_at is null and q.next_try_at <= $1 order by q.id limit 1 for update of q skip locked`, now).133 Scan(&id, &alertID, &transition, &created, &attempts, &severity, &state, &suppressed, &node, &alias)134 if errors.Is(err, pgx.ErrNoRows) {135 return false, nil136 }137 if err != nil {138 return false, err139 }140 done := func(outcome string) (bool, error) {141 if _, err := tx.Exec(ctx, `update alert_notifications set done_at=$2, outcome=$3, attempts=$4 where id=$1`, id, now, outcome, attempts); err != nil {142 return false, err143 }144 return true, tx.Commit(ctx)145 }146 switch {147 case now.Sub(created) > maxAge, transition == "firing" && state == "resolved":148 return done("superseded")149 case suppressed != nil: // silenced or in maintenance since the transition150 return done("suppressed")151 case d.APNs == nil || !d.APNs.Active():152 return done("inactive")153 }154 rcpts, err := d.recipients(ctx, tx, node)155 if err != nil {156 return false, err157 }158 if len(rcpts) == 0 {159 return done("no_recipient")160 }161 attempts++162 sent, failed := 0, 0163 for _, r := range rcpts {164 out, _ := d.APNs.Send(ctx, Push{DeviceToken: r.token, Env: r.env, DeviceKind: r.kind, AlertID: alertID.String(), Severity: severity,165 NodeAlias: deref(alias), State: transition})166 switch out {167 case Sent:168 sent++169 case InvalidToken:170 // Apple says this token is dead: forget it (only if it has not been replaced meanwhile).171 if _, err := tx.Exec(ctx, `update devices set push_token=null, push_env=null where id=$1 and push_token=$2`, r.device, r.token); err != nil {172 return false, err173 }174 case Failed:175 failed++176 }177 }178 if sent > 0 {179 if _, err := tx.Exec(ctx, `insert into timeline_events(id, alert_id, ts, kind, body) values ($1,$2,$3,'notified',$4)`,180 ids.New(), alertID, now, fmt.Sprintf("push notification (%s) sent to %d device(s)", transition, sent)); err != nil {181 return false, err182 }183 }184 if failed > 0 && sent == 0 && attempts < maxAttempts {185 // Nobody was reached: try again later. (When some devices were reached we do not retry the others:186 // a retry would notify the reached ones twice.)187 if _, err := tx.Exec(ctx, `update alert_notifications set attempts=$2, next_try_at=$3 where id=$1`, id, attempts, now.Add(time.Duration(attempts)*time.Minute)); err != nil {188 return false, err189 }190 return true, tx.Commit(ctx)191 }192 if sent == 0 && failed > 0 {193 return done("failed")194 }195 if sent == 0 {196 return done("no_recipient")197 }198 return done("sent")199}200201func deref[T any](p *T) (v T) {202 if p != nil {203 v = *p204 }205 return v206}207