SPB Git

Repository Public

dispatch.go 6.3 KB 207 lines go

1// Package notify delivers alert notifications. The SSE event is produced by2// the alert engine itself (events_outbox); this package pushes to Apple3// devices through APNs, from the durable queue alert_notifications.4package notify56import (7	"context"8	"errors"9	"fmt"10	"time"1112	"github.com/google/uuid"13	"github.com/jackc/pgx/v5"14	"github.com/jackc/pgx/v5/pgxpool"1516	"clustr.spboucher.ai/mcc/internal/authz"17	"clustr.spboucher.ai/mcc/internal/ids"18)1920// Dispatcher drains alert_notifications. A notification is attempted at most21// MaxAttempts times, then dropped as failed; one older than MaxAge is dropped22// as superseded (a late push about an old transition would only mislead).23type Dispatcher struct {24	Pool        *pgxpool.Pool25	APNs        *APNs26	Now         func() time.Time27	MaxAttempts int           // default 428	MaxAge      time.Duration // default 30 min29}3031func (d *Dispatcher) now() time.Time {32	if d.Now != nil {33		return d.Now()34	}35	return time.Now().UTC()36}3738type recipient struct {39	device uuid.UUID40	token  string41	env    string42	kind   string43}4445// recipients: non-revoked Apple devices with a token whose user is active and holds alerts.read (on that node, when there is one).46func (d *Dispatcher) recipients(ctx context.Context, tx pgx.Tx, node *uuid.UUID) ([]recipient, error) {47	grows, err := tx.Query(ctx, `select subject_kind, subject, permission, coalesce(resource_kind,''), coalesce(resource_id,''), effect::text from policy_grants where permission='alerts.read'`)48	if err != nil {49		return nil, err50	}51	grants, err := pgx.CollectRows(grows, func(r pgx.CollectableRow) (authz.Grant, error) {52		var g authz.Grant53		var perm, eff string54		err := r.Scan(&g.SubjectKind, &g.Subject, &perm, &g.ResourceKind, &g.ResourceID, &eff)55		g.Permission, g.Effect = authz.Permission(perm), authz.Effect(eff)56		return g, err57	})58	if err != nil {59		return nil, err60	}61	rows, err := tx.Query(ctx, `select d.id, d.push_token, coalesce(d.push_env,'production'), d.kind::text, u.id, u.role::text62		from devices d join users u on u.id=d.user_id63		where d.revoked_at is null and d.push_token is not null and d.push_token <> '' and d.kind in ('ios','macos') and u.disabled_at is null`)64	if err != nil {65		return nil, err66	}67	defer rows.Close()68	var out []recipient69	for rows.Next() {70		var r recipient71		var user uuid.UUID72		var role string73		if err := rows.Scan(&r.device, &r.token, &r.env, &r.kind, &user, &role); err != nil {74			return nil, err75		}76		sub := authz.Subject{UserID: user.String(), Role: authz.Role(role)}77		ok := authz.Decide(sub, authz.AlertsRead, authz.Resource{}, grants)78		if ok && node != nil {79			ok = authz.Decide(sub, authz.AlertsRead, authz.Resource{Kind: "node", ID: node.String()}, grants)80		}81		if ok {82			out = append(out, r)83		}84	}85	return out, rows.Err()86}8788// Drain processes what is due. It returns the number of notifications handled.89func (d *Dispatcher) Drain(ctx context.Context) (int, error) {90	maxAttempts, maxAge := d.MaxAttempts, d.MaxAge91	if maxAttempts <= 0 {92		maxAttempts = 493	}94	if maxAge <= 0 {95		maxAge = 30 * time.Minute96	}97	handled := 098	for handled < 200 {99		more, err := d.one(ctx, maxAttempts, maxAge)100		if err != nil {101			return handled, err102		}103		if !more {104			break105		}106		handled++107	}108	return handled, nil109}110111func (d *Dispatcher) one(ctx context.Context, maxAttempts int, maxAge time.Duration) (bool, error) {112	now := d.now()113	tx, err := d.Pool.Begin(ctx)114	if err != nil {115		return false, err116	}117	defer tx.Rollback(ctx)118	var (119		id         int64120		alertID    uuid.UUID121		transition string122		created    time.Time123		attempts   int124		severity   string125		state      string126		suppressed *string127		node       *uuid.UUID128		alias      *string129	)130	err = tx.QueryRow(ctx, `select q.id, q.alert_id, q.transition, q.created_at, q.attempts, a.severity::text, a.state::text, a.suppressed_by, a.node_id, n.alias::text131		from alert_notifications q join alerts a on a.id=q.alert_id left join nodes n on n.id=a.node_id132		where q.done_at is null and q.next_try_at <= $1 order by q.id limit 1 for update of q skip locked`, now).133		Scan(&id, &alertID, &transition, &created, &attempts, &severity, &state, &suppressed, &node, &alias)134	if errors.Is(err, pgx.ErrNoRows) {135		return false, nil136	}137	if err != nil {138		return false, err139	}140	done := func(outcome string) (bool, error) {141		if _, err := tx.Exec(ctx, `update alert_notifications set done_at=$2, outcome=$3, attempts=$4 where id=$1`, id, now, outcome, attempts); err != nil {142			return false, err143		}144		return true, tx.Commit(ctx)145	}146	switch {147	case now.Sub(created) > maxAge, transition == "firing" && state == "resolved":148		return done("superseded")149	case suppressed != nil: // silenced or in maintenance since the transition150		return done("suppressed")151	case d.APNs == nil || !d.APNs.Active():152		return done("inactive")153	}154	rcpts, err := d.recipients(ctx, tx, node)155	if err != nil {156		return false, err157	}158	if len(rcpts) == 0 {159		return done("no_recipient")160	}161	attempts++162	sent, failed := 0, 0163	for _, r := range rcpts {164		out, _ := d.APNs.Send(ctx, Push{DeviceToken: r.token, Env: r.env, DeviceKind: r.kind, AlertID: alertID.String(), Severity: severity,165			NodeAlias: deref(alias), State: transition})166		switch out {167		case Sent:168			sent++169		case InvalidToken:170			// Apple says this token is dead: forget it (only if it has not been replaced meanwhile).171			if _, err := tx.Exec(ctx, `update devices set push_token=null, push_env=null where id=$1 and push_token=$2`, r.device, r.token); err != nil {172				return false, err173			}174		case Failed:175			failed++176		}177	}178	if sent > 0 {179		if _, err := tx.Exec(ctx, `insert into timeline_events(id, alert_id, ts, kind, body) values ($1,$2,$3,'notified',$4)`,180			ids.New(), alertID, now, fmt.Sprintf("push notification (%s) sent to %d device(s)", transition, sent)); err != nil {181			return false, err182		}183	}184	if failed > 0 && sent == 0 && attempts < maxAttempts {185		// Nobody was reached: try again later. (When some devices were reached we do not retry the others:186		// a retry would notify the reached ones twice.)187		if _, err := tx.Exec(ctx, `update alert_notifications set attempts=$2, next_try_at=$3 where id=$1`, id, attempts, now.Add(time.Duration(attempts)*time.Minute)); err != nil {188			return false, err189		}190		return true, tx.Commit(ctx)191	}192	if sent == 0 && failed > 0 {193		return done("failed")194	}195	if sent == 0 {196		return done("no_recipient")197	}198	return done("sent")199}200201func deref[T any](p *T) (v T) {202	if p != nil {203		v = *p204	}205	return v206}207
Files main
Menu

Theme