SPB Git

Repository Public

dispatch_test.go 8.4 KB 236 lines go

1package notify23import (4	"context"5	"encoding/json"6	"strings"7	"testing"8	"time"910	"github.com/google/uuid"11	"github.com/jackc/pgx/v5/pgxpool"1213	"clustr.spboucher.ai/mcc/internal/ids"14	"clustr.spboucher.ai/mcc/internal/testdb"15)1617type dispEnv struct {18	t     *testing.T19	pool  *pgxpool.Pool20	fx    fixture21	d     *Dispatcher22	node  uuid.UUID23	rule  uuid.UUID24	clock time.Time25}2627const (28	tokA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"29	tokB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"30	tokC = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"31)3233func newDispEnv(t *testing.T) *dispEnv {34	t.Helper()35	pool, _ := testdb.New(t)36	e := &dispEnv{t: t, pool: pool, fx: newFixture(t), clock: time.Now().UTC()}37	e.d = &Dispatcher{Pool: pool, APNs: e.fx.apns, Now: func() time.Time { return e.clock }}38	e.node = testdb.Node(t, pool, "T-NODE")39	e.rule = ids.New()40	if _, err := pool.Exec(context.Background(), `insert into alert_rules(id, name, kind, severity) values ($1,'r','absence','critical')`, e.rule); err != nil {41		t.Fatal(err)42	}43	return e44}4546func (e *dispEnv) device(handle, role, kind, token string) uuid.UUID {47	e.t.Helper()48	var user uuid.UUID49	if err := e.pool.QueryRow(context.Background(), `select id from users where handle=$1`, handle).Scan(&user); err != nil {50		user = testdb.User(e.t, e.pool, handle, role)51	}52	id := ids.New()53	var tok *string54	if token != "" {55		tok = &token56	}57	if _, err := e.pool.Exec(context.Background(), `insert into devices(id, user_id, kind, push_token, push_env) values ($1,$2,$3::device_kind,$4,'production')`, id, user, kind, tok); err != nil {58		e.t.Fatal(err)59	}60	return id61}6263// alert queues a notification for a new alert and returns the alert id.64func (e *dispEnv) alert(transition, state string) uuid.UUID {65	e.t.Helper()66	id := ids.New()67	// The summary deliberately holds things that must never reach a push payload.68	if _, err := e.pool.Exec(context.Background(), `insert into alerts(id, rule_id, node_id, fingerprint, state, severity, summary, started_at, last_eval_at)69		values ($1,$2,$3,$4,$5::alert_state,'critical','disk at 99% on 192.168.2.44 token=SECRETVALUE /var/log/app.log',$6,$6)`, id, e.rule, e.node, id.String(), state, e.clock); err != nil {70		e.t.Fatal(err)71	}72	if _, err := e.pool.Exec(context.Background(), `insert into alert_notifications(alert_id, transition, created_at, next_try_at) values ($1,$2,$3,$3)`, id, transition, e.clock); err != nil {73		e.t.Fatal(err)74	}75	return id76}7778func (e *dispEnv) outcome(alert uuid.UUID) (string, int) {79	e.t.Helper()80	var o *string81	var attempts int82	if err := e.pool.QueryRow(context.Background(), `select outcome, attempts from alert_notifications where alert_id=$1`, alert).Scan(&o, &attempts); err != nil {83		e.t.Fatal(err)84	}85	if o == nil {86		return "", attempts87	}88	return *o, attempts89}9091func (e *dispEnv) drain() {92	e.t.Helper()93	if _, err := e.d.Drain(context.Background()); err != nil {94		e.t.Fatal(err)95	}96}9798func TestDispatcherPushesOnlyToEntitledDevices(t *testing.T) {99	e := newDispEnv(t)100	ctx := context.Background()101	e.device("owner", "owner", "ios", tokA)102	e.device("owner", "owner", "macos", tokB)103	e.device("owner", "owner", "web", tokC)                                 // web devices never get APNs104	revoked := e.device("viewer", "viewer", "ios", strings.Repeat("d", 64)) // revoked device105	e.pool.Exec(ctx, `update devices set revoked_at=now() where id=$1`, revoked)106	e.device("gone", "admin", "ios", strings.Repeat("e", 64)) // disabled account107	e.pool.Exec(ctx, `update users set disabled_at=now() where handle='gone'`)108	e.device("blind", "viewer", "ios", strings.Repeat("f", 64)) // alerts.read denied by policy109	e.pool.Exec(ctx, `insert into policy_grants(id, subject_kind, subject, permission, effect) select $1,'user',id::text,'alerts.read','deny' from users where handle='blind'`, ids.New())110	e.device("scoped", "viewer", "ios", strings.Repeat("9", 64)) // alerts.read denied on this node only111	e.pool.Exec(ctx, `insert into policy_grants(id, subject_kind, subject, permission, resource_kind, resource_id, effect)112		select $1,'user',id::text,'alerts.read','node',$2,'deny' from users where handle='scoped'`, ids.New(), e.node.String())113	e.device("notoken", "owner", "ios", "")114115	id := e.alert("firing", "firing")116	e.drain()117	if o, _ := e.outcome(id); o != "sent" {118		t.Fatalf("outcome %q", o)119	}120	reqs := e.fx.fake.reqs()121	if len(reqs) != 2 {122		t.Fatalf("%d pushes, want 2 (owner's iOS and macOS devices)", len(reqs))123	}124	seen := map[string]string{}125	for _, r := range reqs {126		seen[r.Path[strings.LastIndex(r.Path, "/")+1:]] = r.Header.Get("apns-topic")127		// The payload carries a generic title, the severity, the node alias and the alert id — nothing else.128		var p map[string]any129		if err := json.Unmarshal(r.Body, &p); err != nil {130			t.Fatal(err)131		}132		for k := range p {133			if k != "aps" && k != "alert_id" && k != "severity" && k != "node" && k != "state" {134				t.Fatalf("unexpected payload field %q", k)135			}136		}137		if p["alert_id"] != id.String() || p["severity"] != "critical" || p["node"] != "T-NODE" {138			t.Fatalf("payload: %s", r.Body)139		}140		for _, leak := range []string{"192.168", "SECRETVALUE", "/var/log", "disk at", "token="} {141			if strings.Contains(string(r.Body), leak) {142				t.Fatalf("payload leaks %q: %s", leak, r.Body)143			}144		}145	}146	if seen[tokA] != "ai.spboucher.mcc.ios" || seen[tokB] != "ai.spboucher.mcc.macos" {147		t.Fatalf("recipients/topics: %v", seen)148	}149	var n int150	e.pool.QueryRow(ctx, `select count(*) from timeline_events where alert_id=$1 and kind='notified'`, id).Scan(&n)151	if n != 1 {152		t.Fatal("no 'notified' timeline event")153	}154	// Nothing is sent twice.155	e.drain()156	if len(e.fx.fake.reqs()) != 2 {157		t.Fatal("notification sent again")158	}159}160161func TestDispatcherInvalidatesDeadTokensAndRetriesBounded(t *testing.T) {162	e := newDispEnv(t)163	ctx := context.Background()164	dev := e.device("owner", "owner", "ios", tokA)165	// 410: Apple says the token is dead.166	e.fx.fake.setRespond(func(int) (int, string) { return 410, "Unregistered" })167	id := e.alert("firing", "firing")168	e.drain()169	var tok *string170	e.pool.QueryRow(ctx, `select push_token from devices where id=$1`, dev).Scan(&tok)171	if o, _ := e.outcome(id); tok != nil || o != "no_recipient" {172		t.Fatalf("token kept (%v) or outcome %q", tok, o)173	}174	// Provider down: bounded attempts with a growing delay, then dropped as failed.175	e.pool.Exec(ctx, `update devices set push_token=$2 where id=$1`, dev, tokB)176	e.fx.fake.setRespond(func(int) (int, string) { return 503, "ServiceUnavailable" })177	id = e.alert("firing", "firing")178	before := len(e.fx.fake.reqs())179	for i := 0; i < 10; i++ {180		e.drain()181		e.clock = e.clock.Add(5 * time.Minute)182	}183	o, attempts := e.outcome(id)184	if o != "failed" || attempts != 4 {185		t.Fatalf("outcome %q after %d attempts", o, attempts)186	}187	if got := len(e.fx.fake.reqs()) - before; got != 12 { // 4 attempts × 3 tries each188		t.Fatalf("%d requests", got)189	}190	e.pool.QueryRow(ctx, `select push_token from devices where id=$1`, dev).Scan(&tok)191	if tok == nil {192		t.Fatal("a provider outage must not invalidate tokens")193	}194	if st := e.fx.apns.Status(); st.State != "degraded" {195		t.Fatalf("status %+v", st)196	}197}198199func TestDispatcherSkipsStaleSupersededSuppressedAndInactive(t *testing.T) {200	e := newDispEnv(t)201	ctx := context.Background()202	e.device("owner", "owner", "ios", tokA)203	// The alert resolved before its "firing" push left.204	a := e.alert("firing", "resolved")205	// Silenced meanwhile.206	b := e.alert("firing", "firing")207	e.pool.Exec(ctx, `update alerts set suppressed_by='silence' where id=$1`, b)208	// Too old to be worth pushing.209	c := e.alert("resolved", "resolved")210	e.pool.Exec(ctx, `update alert_notifications set created_at=$2 where alert_id=$1`, c, e.clock.Add(-2*time.Hour))211	e.drain()212	for id, want := range map[uuid.UUID]string{a: "superseded", b: "suppressed", c: "superseded"} {213		if o, _ := e.outcome(id); o != want {214			t.Errorf("outcome %q, want %q", o, want)215		}216	}217	if len(e.fx.fake.reqs()) != 0 {218		t.Fatal("something was pushed")219	}220	// Unconfigured provider: inactive, visible, nothing fails and nothing piles up.221	e.d.APNs = NewAPNs(APNsConfig{})222	d := e.alert("firing", "firing")223	e.drain()224	if o, _ := e.outcome(d); o != "inactive" {225		t.Fatalf("outcome %q", o)226	}227	if st := e.d.APNs.Status(); st.State != "inactive" || st.Reason == "" {228		t.Fatalf("status %+v", st)229	}230	var pending int231	e.pool.QueryRow(ctx, `select count(*) from alert_notifications where done_at is null`).Scan(&pending)232	if pending != 0 {233		t.Fatalf("%d notifications left pending", pending)234	}235}236
Files main
Menu

Theme