Repository Public
1package notify23import (4 "context"5 "encoding/json"6 "strings"7 "testing"8 "time"910 "github.com/google/uuid"11 "github.com/jackc/pgx/v5/pgxpool"1213 "clustr.spboucher.ai/mcc/internal/ids"14 "clustr.spboucher.ai/mcc/internal/testdb"15)1617type dispEnv struct {18 t *testing.T19 pool *pgxpool.Pool20 fx fixture21 d *Dispatcher22 node uuid.UUID23 rule uuid.UUID24 clock time.Time25}2627const (28 tokA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"29 tokB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"30 tokC = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"31)3233func newDispEnv(t *testing.T) *dispEnv {34 t.Helper()35 pool, _ := testdb.New(t)36 e := &dispEnv{t: t, pool: pool, fx: newFixture(t), clock: time.Now().UTC()}37 e.d = &Dispatcher{Pool: pool, APNs: e.fx.apns, Now: func() time.Time { return e.clock }}38 e.node = testdb.Node(t, pool, "T-NODE")39 e.rule = ids.New()40 if _, err := pool.Exec(context.Background(), `insert into alert_rules(id, name, kind, severity) values ($1,'r','absence','critical')`, e.rule); err != nil {41 t.Fatal(err)42 }43 return e44}4546func (e *dispEnv) device(handle, role, kind, token string) uuid.UUID {47 e.t.Helper()48 var user uuid.UUID49 if err := e.pool.QueryRow(context.Background(), `select id from users where handle=$1`, handle).Scan(&user); err != nil {50 user = testdb.User(e.t, e.pool, handle, role)51 }52 id := ids.New()53 var tok *string54 if token != "" {55 tok = &token56 }57 if _, err := e.pool.Exec(context.Background(), `insert into devices(id, user_id, kind, push_token, push_env) values ($1,$2,$3::device_kind,$4,'production')`, id, user, kind, tok); err != nil {58 e.t.Fatal(err)59 }60 return id61}6263// alert queues a notification for a new alert and returns the alert id.64func (e *dispEnv) alert(transition, state string) uuid.UUID {65 e.t.Helper()66 id := ids.New()67 // The summary deliberately holds things that must never reach a push payload.68 if _, err := e.pool.Exec(context.Background(), `insert into alerts(id, rule_id, node_id, fingerprint, state, severity, summary, started_at, last_eval_at)69 values ($1,$2,$3,$4,$5::alert_state,'critical','disk at 99% on 192.168.2.44 token=SECRETVALUE /var/log/app.log',$6,$6)`, id, e.rule, e.node, id.String(), state, e.clock); err != nil {70 e.t.Fatal(err)71 }72 if _, err := e.pool.Exec(context.Background(), `insert into alert_notifications(alert_id, transition, created_at, next_try_at) values ($1,$2,$3,$3)`, id, transition, e.clock); err != nil {73 e.t.Fatal(err)74 }75 return id76}7778func (e *dispEnv) outcome(alert uuid.UUID) (string, int) {79 e.t.Helper()80 var o *string81 var attempts int82 if err := e.pool.QueryRow(context.Background(), `select outcome, attempts from alert_notifications where alert_id=$1`, alert).Scan(&o, &attempts); err != nil {83 e.t.Fatal(err)84 }85 if o == nil {86 return "", attempts87 }88 return *o, attempts89}9091func (e *dispEnv) drain() {92 e.t.Helper()93 if _, err := e.d.Drain(context.Background()); err != nil {94 e.t.Fatal(err)95 }96}9798func TestDispatcherPushesOnlyToEntitledDevices(t *testing.T) {99 e := newDispEnv(t)100 ctx := context.Background()101 e.device("owner", "owner", "ios", tokA)102 e.device("owner", "owner", "macos", tokB)103 e.device("owner", "owner", "web", tokC) // web devices never get APNs104 revoked := e.device("viewer", "viewer", "ios", strings.Repeat("d", 64)) // revoked device105 e.pool.Exec(ctx, `update devices set revoked_at=now() where id=$1`, revoked)106 e.device("gone", "admin", "ios", strings.Repeat("e", 64)) // disabled account107 e.pool.Exec(ctx, `update users set disabled_at=now() where handle='gone'`)108 e.device("blind", "viewer", "ios", strings.Repeat("f", 64)) // alerts.read denied by policy109 e.pool.Exec(ctx, `insert into policy_grants(id, subject_kind, subject, permission, effect) select $1,'user',id::text,'alerts.read','deny' from users where handle='blind'`, ids.New())110 e.device("scoped", "viewer", "ios", strings.Repeat("9", 64)) // alerts.read denied on this node only111 e.pool.Exec(ctx, `insert into policy_grants(id, subject_kind, subject, permission, resource_kind, resource_id, effect)112 select $1,'user',id::text,'alerts.read','node',$2,'deny' from users where handle='scoped'`, ids.New(), e.node.String())113 e.device("notoken", "owner", "ios", "")114115 id := e.alert("firing", "firing")116 e.drain()117 if o, _ := e.outcome(id); o != "sent" {118 t.Fatalf("outcome %q", o)119 }120 reqs := e.fx.fake.reqs()121 if len(reqs) != 2 {122 t.Fatalf("%d pushes, want 2 (owner's iOS and macOS devices)", len(reqs))123 }124 seen := map[string]string{}125 for _, r := range reqs {126 seen[r.Path[strings.LastIndex(r.Path, "/")+1:]] = r.Header.Get("apns-topic")127 // The payload carries a generic title, the severity, the node alias and the alert id — nothing else.128 var p map[string]any129 if err := json.Unmarshal(r.Body, &p); err != nil {130 t.Fatal(err)131 }132 for k := range p {133 if k != "aps" && k != "alert_id" && k != "severity" && k != "node" && k != "state" {134 t.Fatalf("unexpected payload field %q", k)135 }136 }137 if p["alert_id"] != id.String() || p["severity"] != "critical" || p["node"] != "T-NODE" {138 t.Fatalf("payload: %s", r.Body)139 }140 for _, leak := range []string{"192.168", "SECRETVALUE", "/var/log", "disk at", "token="} {141 if strings.Contains(string(r.Body), leak) {142 t.Fatalf("payload leaks %q: %s", leak, r.Body)143 }144 }145 }146 if seen[tokA] != "ai.spboucher.mcc.ios" || seen[tokB] != "ai.spboucher.mcc.macos" {147 t.Fatalf("recipients/topics: %v", seen)148 }149 var n int150 e.pool.QueryRow(ctx, `select count(*) from timeline_events where alert_id=$1 and kind='notified'`, id).Scan(&n)151 if n != 1 {152 t.Fatal("no 'notified' timeline event")153 }154 // Nothing is sent twice.155 e.drain()156 if len(e.fx.fake.reqs()) != 2 {157 t.Fatal("notification sent again")158 }159}160161func TestDispatcherInvalidatesDeadTokensAndRetriesBounded(t *testing.T) {162 e := newDispEnv(t)163 ctx := context.Background()164 dev := e.device("owner", "owner", "ios", tokA)165 // 410: Apple says the token is dead.166 e.fx.fake.setRespond(func(int) (int, string) { return 410, "Unregistered" })167 id := e.alert("firing", "firing")168 e.drain()169 var tok *string170 e.pool.QueryRow(ctx, `select push_token from devices where id=$1`, dev).Scan(&tok)171 if o, _ := e.outcome(id); tok != nil || o != "no_recipient" {172 t.Fatalf("token kept (%v) or outcome %q", tok, o)173 }174 // Provider down: bounded attempts with a growing delay, then dropped as failed.175 e.pool.Exec(ctx, `update devices set push_token=$2 where id=$1`, dev, tokB)176 e.fx.fake.setRespond(func(int) (int, string) { return 503, "ServiceUnavailable" })177 id = e.alert("firing", "firing")178 before := len(e.fx.fake.reqs())179 for i := 0; i < 10; i++ {180 e.drain()181 e.clock = e.clock.Add(5 * time.Minute)182 }183 o, attempts := e.outcome(id)184 if o != "failed" || attempts != 4 {185 t.Fatalf("outcome %q after %d attempts", o, attempts)186 }187 if got := len(e.fx.fake.reqs()) - before; got != 12 { // 4 attempts × 3 tries each188 t.Fatalf("%d requests", got)189 }190 e.pool.QueryRow(ctx, `select push_token from devices where id=$1`, dev).Scan(&tok)191 if tok == nil {192 t.Fatal("a provider outage must not invalidate tokens")193 }194 if st := e.fx.apns.Status(); st.State != "degraded" {195 t.Fatalf("status %+v", st)196 }197}198199func TestDispatcherSkipsStaleSupersededSuppressedAndInactive(t *testing.T) {200 e := newDispEnv(t)201 ctx := context.Background()202 e.device("owner", "owner", "ios", tokA)203 // The alert resolved before its "firing" push left.204 a := e.alert("firing", "resolved")205 // Silenced meanwhile.206 b := e.alert("firing", "firing")207 e.pool.Exec(ctx, `update alerts set suppressed_by='silence' where id=$1`, b)208 // Too old to be worth pushing.209 c := e.alert("resolved", "resolved")210 e.pool.Exec(ctx, `update alert_notifications set created_at=$2 where alert_id=$1`, c, e.clock.Add(-2*time.Hour))211 e.drain()212 for id, want := range map[uuid.UUID]string{a: "superseded", b: "suppressed", c: "superseded"} {213 if o, _ := e.outcome(id); o != want {214 t.Errorf("outcome %q, want %q", o, want)215 }216 }217 if len(e.fx.fake.reqs()) != 0 {218 t.Fatal("something was pushed")219 }220 // Unconfigured provider: inactive, visible, nothing fails and nothing piles up.221 e.d.APNs = NewAPNs(APNsConfig{})222 d := e.alert("firing", "firing")223 e.drain()224 if o, _ := e.outcome(d); o != "inactive" {225 t.Fatalf("outcome %q", o)226 }227 if st := e.d.APNs.Status(); st.State != "inactive" || st.Reason == "" {228 t.Fatalf("status %+v", st)229 }230 var pending int231 e.pool.QueryRow(ctx, `select count(*) from alert_notifications where done_at is null`).Scan(&pending)232 if pending != 0 {233 t.Fatalf("%d notifications left pending", pending)234 }235}236