Repository Public
1// The alert dispatcher on its own database: event rules (each fires once, digests, switches), exactly-once across a restart,2// a crash before the commit, two dispatchers at once and out-of-order commits, condition rules (debounce, episodes), delivery3// (quiet hours, hourly cap with ONE summary, zero subscriptions, lock-screen privacy) and the signed webhook.4import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';5import { randomUUID } from 'node:crypto';6import type { AlertDispatcher } from '../src/alerts/dispatcher.ts';7import { decryptPush, generateSubscriptionKeys } from '../src/alerts/webpush.ts';8import { verifyWebhook } from '../src/alerts/webhook.ts';9import { addAgent, addPaperAccount, addProposal, addVenueAccount, riskEvent, SIGNER_OK, world, type World } from './alerts-helpers.ts';1011let w: World, d: AlertDispatcher, sql: any;12const kinds = async (where?: string) => (await w.alerts(where)).map((a) => a.kind).sort();13const one = async (kind: string) => { const r = await w.alerts(`kind = '${kind}'`); expect(r, kind).toHaveLength(1); return r[0]; };14const settings = async (patch: Record<string, unknown> = {}) => {15 const base = w.store.DEFAULT_SETTINGS;16 await w.store.saveAlertSettings({ ...base, ...patch, rules: { ...base.rules, ...(patch.rules as object) }, thresholds: { ...base.thresholds, ...(patch.thresholds as object) } } as never);17};18const calls: { url: string; headers: Headers; body: Buffer }[] = [];19let pushStatus = 201;2021beforeAll(async () => {22 w = await world('alerts_engine'); sql = w.sql; d = w.dispatcher();23 w.env.fetch = (async (url: unknown, init: RequestInit) => { calls.push({ url: String(url), headers: new Headers(init.headers), body: Buffer.from(init.body as Uint8Array | string) }); return new Response(null, { status: pushStatus }); }) as typeof fetch;24 await d.tail(); // cursors are created at the present25});26afterAll(async () => { await sql?.end(); });27beforeEach(async () => {28 await d.tail(); // nothing left over from the previous test29 await sql`truncate alerts restart identity`;30 await sql`delete from alert_state`; await sql`delete from push_subscriptions`;31 await sql`truncate deriv_orders, deriv_fills, deriv_funding, deriv_equity, deriv_accounts, deriv_paper_accounts, proposals`;32 await sql`truncate agents cascade`;33 await settings();34 Object.assign(w.env, { signer: SIGNER_OK, rpc: [], loops: [], backup: null, disk: null, fault: undefined });35 calls.length = 0; pushStatus = 201;36 const health = await import('../src/monitor/health.ts'); health.resetHealthCaches();37});3839describe('event rules', () => {40 it('history is not replayed: only what happens after the first pass alerts', async () => {41 await sql`delete from alert_cursors where source = 'audit'`;42 await w.db.audit('deposit.seen', 'chain', { network: 'base', txid: '0xold', asset: 'native', amount: '1000000000000000000' });43 expect(await d.tail()).toBe(0);44 await w.db.audit('deposit.seen', 'chain', { network: 'base', txid: '0xnew', asset: 'native', amount: '1500000000000000000' });45 expect(await d.tail()).toBe(1);46 const a = await one('deposit.received');47 expect(a.params).toEqual({ network: 'Base', symbol: 'ETH', amount: '1.5' });48 expect(a.subject).toMatchObject({ type: 'deposit', id: '0xnew' });49 expect(a).toMatchObject({ severity: 'info', source: 'audit', delivery: 'pending' });50 });5152 it('every audit rule fires exactly once, with codes and parameters (not sentences)', async () => {53 await settings({ rules: { 'deriv.position_opened': true } });54 const agent = await addAgent(sql, { name: 'Momentum' }), paper = await addPaperAccount(sql), venue = await addVenueAccount(sql);55 const tr = await addProposal(sql), trFail = await addProposal(sql, { state: 'failed', error: 'reverted on-chain' });56 const sw = await addProposal(sql, { kind: 'swap', symbol: 'ETH', decimals: 18, amount: '500000000000000000', swap: { toSymbol: 'USDC' } }), swFail = await addProposal(sql, { kind: 'swap', state: 'failed' });57 const br = await addProposal(sql, { kind: 'bridge', bridge: { toNetwork: 'arbitrum', phase: 'done' } }), brFail = await addProposal(sql, { kind: 'bridge', state: 'failed', bridge: { toNetwork: 'arbitrum' } });58 const order = randomUUID();59 await sql`insert into agent_orders (id, agent_id, mode, from_asset, to_asset, from_symbol, to_symbol, amount_in, state) values (${order}, ${agent}, 'LIVE', 'a', 'b', 'USDC', 'ETH', 1000000, 'filled')`;60 await sql`insert into deriv_orders (account_id, client_id, instrument, side, type, size, reduce_only, tif, status, created_at, updated_at, state, final)61 values (${venue}, 'c-1', 'hyperliquid:BTC', 'buy', 'limit', 0.01, false, 'gtc', 'filled', now(), now(), 'filled', true)`;62 await sql`insert into sessions (id_hash, expires_at, ua, ip, created_at) values ('old', now() + interval '1 hour', 'Mozilla/5.0 (Macintosh) Chrome/130', '198.51.100.1', now() - interval '1 day')`;63 await sql`insert into sessions (id_hash, expires_at, ua, ip) values ('new', now() + interval '1 hour', 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_0) Safari/605', '203.0.113.9')`;64 const ev: [string, string, Record<string, unknown>][] = [65 ['transfer.broadcast', 'system', { id: tr }], ['transfer.finalized', 'system', { id: tr }], ['transfer.failed', 'system', { id: trFail, error: 'reverted on-chain' }],66 ['transfer.finalized', 'system', { id: sw }], ['transfer.failed', 'system', { id: swFail }],67 ['transfer.finalized', 'system', { id: br }], ['bridge.done', 'chain', { id: br, to: 'arbitrum', minted: '24990000' }], ['transfer.failed', 'system', { id: brFail }], ['bridge.mint_reverted', 'chain', { id: br }],68 ['agent.order_filled', 'chain', { agent, id: order }],69 ['agent.stopped', 'engine', { id: agent, reason: 'loss limit reached: equity 80.00 USD against 100.00 USD allocated (limit 15 %); positions were sold to USDC' }],70 ['agent.stopped', 'engine', { id: agent, reason: 'mandate expired' }], ['agent.stopped', 'engine', { id: agent, reason: 'the approved strategy no longer matches the mandate' }],71 ['mandate.revoked', 'owner', { agent }], ['agent.order_refused', 'signer', { agent, reason: 'daily limit exceeded' }],72 ['deriv.paper.fill', 'paper-engine', { account: paper, fill: 'paper-f-1', clientId: 'p1', instrument: 'hyperliquid:BTC', side: 'buy', size: '0.5', price: '100000' }],73 ['deriv.paper.fill', 'paper-engine', { account: paper, fill: 'paper-f-2', clientId: 'p2', instrument: 'hyperliquid:BTC', side: 'sell', size: '0.5', price: '101000' }],74 ['deriv.paper.liquidation', 'paper-engine', { account: paper, positions: 2 }], ['deriv.paper.halt', 'paper-engine', { account: paper, code: 'daily_loss_exceeded' }],75 ['deriv.live.halt', 'owner', { account: venue, reason: 'manual stop' }], ['deriv.live.order_rejected', 'owner', { account: venue, clientId: 'c-0', reason: 'venue said no' }],76 ['deriv.live.order_filled', 'reconciler', { account: venue, clientId: 'c-1', filled: '0.01', avgFillPrice: '100500', final: true }],77 ['deriv.live.policy_replaced', 'owner', { account: venue, maxLeverage: '3' }],78 ['device.added', 'owner', { credential: 'abc', label: 'YubiKey' }], ['login.failed', 'unknown', { why: 'bad assertion', ip: '192.0.2.4' }], ['login.failed', 'unknown', { why: 'bad assertion', ip: '192.0.2.5' }],79 ['login.ok', 'owner', { credential: 'abc', ip: '203.0.113.9' }], ['login.ok', 'owner', { credential: 'abc', ip: '203.0.113.9' }],80 ['addressbook.saved', 'owner', { network: 'base', address: '0x1', label: 'x' }], ['swap.settled', 'chain', { id: sw }],81 ];82 for (const [t, actor, data] of ev) await w.db.audit(t, actor, data);83 await d.tail();84 expect(await kinds()).toEqual(['agent.mandate_revoked', 'agent.order_filled', 'agent.stopped', 'agent.stopped_loss', 'agent.stopped_mandate', 'bridge.confirmed', 'bridge.failed', 'bridge.failed',85 'deriv.halt', 'deriv.live.filled', 'deriv.loss_limit', 'deriv.order_rejected', 'deriv.paper.filled', 'deriv.paper.filled', 'deriv.paper.liquidation', 'deriv.position_closed', 'deriv.position_opened', 'deriv.position_opened',86 'security.login_failed', 'security.new_device', 'security.passkey_added', 'security.policy_changed', 'security.signer_refused', 'swap.confirmed', 'swap.failed', 'transfer.confirmed', 'transfer.failed'].sort());87 expect(await d.tail()).toBe(0); // a second pass adds nothing88 expect((await one('transfer.confirmed')).params).toEqual({ network: 'Base', symbol: 'USDC', amount: '25' });89 expect((await one('swap.confirmed')).params).toEqual({ network: 'Base', symbol: 'ETH', amount: '0.5', toSymbol: 'USDC' });90 expect((await one('bridge.confirmed')).params).toMatchObject({ network: 'Base', to: 'Arbitrum One', amount: '24.99' });91 expect((await one('agent.stopped_loss'))).toMatchObject({ severity: 'critical', subject: { type: 'agent', id: agent } });92 expect((await one('agent.order_filled')).params).toEqual({ agent: 'Momentum', from: 'USDC', to: 'ETH', count: 1 });93 expect((await one('deriv.loss_limit'))).toMatchObject({ severity: 'warning', params: { mode: '@paper', scope: '@daily' } }); // a paper account is simulated: not a critical94 expect((await one('deriv.halt')).severity).toBe('critical');95 expect((await one('deriv.position_closed')).params).toMatchObject({ account: 'Paper HL', mode: '@paper', instrument: 'hyperliquid:BTC' });96 expect((await one('security.login_failed'))).toMatchObject({ count: 2, params: { count: 2, ip: '192.0.2.5' } });97 expect((await one('security.new_device')).params).toEqual({ device: 'Safari · iPhone', ip: '203.0.113.9' }); // once, although the device signed in twice98 expect((await one('security.passkey_added')).severity).toBe('critical');99 for (const a of await w.alerts()) expect(JSON.stringify(a.params)).not.toMatch(/0xto|0xfrom/); // no address in any alert100 });101102 it('risk events and the research log: digests, signal failures in a row, pinned code', async () => {103 const paper = await addPaperAccount(sql), venue = await addVenueAccount(sql), agent = await addAgent(sql, { name: 'Trend' });104 await riskEvent(sql, paper, 'margin_warning', 'margin_warning', 'maintenance margin 480 has reached 80 % of equity 600');105 await riskEvent(sql, paper, 'order_refused', 'leverage_exceeded', 'leverage 12 exceeds 5');106 await riskEvent(sql, paper, 'order_refused', 'order_notional_exceeded', 'order notional too large');107 await riskEvent(sql, venue, 'order_refused', 'signer', 'refused by the signer: daily order count reached');108 await riskEvent(sql, venue, 'divergence', 'state_divergence', 'position hyperliquid:BTC: venue 0.2, wallet records 0.1');109 await riskEvent(sql, venue, 'divergence_cleared', null, 'agree again');110 await riskEvent(sql, paper, 'liquidation', 'liquidated', 'simulated liquidation'); // alerts for this come from the audit chain: ignored here111 for (let i = 0; i < 2; i++) await sql`insert into research_log (agent_id, kind, title, body) values (${agent}, 'hold', 'No valid signal: holding', 'weights sum to more than 1')`;112 await d.tail();113 expect(await kinds(`kind = 'agent.signal_failures'`)).toEqual([]); // two failures: below the threshold of three114 await sql`insert into research_log (agent_id, kind, title, body) values (${agent}, 'hold', 'No valid signal: holding', 'weights sum to more than 1')`;115 await sql`insert into research_log (agent_id, kind, title, body) values (${agent}, 'hold', 'No valid signal: holding', 'again')`;116 await sql`insert into research_log (agent_id, kind, title, body) values (${agent}, 'hold', 'Order not placed', 'the swap router bytecode is not the reviewed one; swaps are refused')`;117 await sql`insert into research_log (agent_id, kind, title, body) values (${agent}, 'note', 'Backtest', 'ok')`;118 await d.tail(); await d.tail();119 expect(await kinds()).toEqual(['agent.signal_failures', 'deriv.divergence', 'deriv.divergence_cleared', 'deriv.margin_warning', 'deriv.order_rejected', 'security.pin_mismatch', 'security.signer_refused']);120 expect(await one('deriv.order_rejected')).toMatchObject({ count: 2, params: { account: 'Paper HL', mode: '@paper', count: 2, reason: 'order notional too large' } });121 expect((await one('deriv.divergence')).severity).toBe('critical');122 expect((await one('agent.signal_failures')).params).toEqual({ agent: 'Trend', count: 3 });123 expect((await one('security.pin_mismatch'))).toMatchObject({ severity: 'critical', params: { what: '@router' } });124 });125126 it('agent fills: one alert per agent per window, or one per order when the digest is off', async () => {127 const a = await addAgent(sql, { name: 'A' }), b = await addAgent(sql, { name: 'B' });128 for (const agent of [a, a, a, b]) await w.db.audit('agent.order_filled', 'chain', { agent, id: randomUUID() });129 await d.tail();130 const rows = await w.alerts();131 expect(rows.map((r) => [r.params.agent, r.count])).toEqual([['A', 3], ['B', 1]]);132 await sql`update alerts set read_at = now()`;133 await w.db.audit('agent.order_filled', 'chain', { agent: a, id: randomUUID() });134 await d.tail();135 expect((await w.alerts(`params->>'agent' = 'A'`))[0]).toMatchObject({ count: 4, read_at: null }); // the digest is unread again136 await settings({ thresholds: { agentDigestMinutes: 0 } });137 for (let i = 0; i < 2; i++) await w.db.audit('agent.order_filled', 'chain', { agent: b, id: randomUUID() });138 await d.tail();139 expect(await w.alerts(`params->>'agent' = 'B'`)).toHaveLength(3);140 });141142 it('a rule switched off produces nothing — and the event is consumed, not kept for later', async () => {143 await settings({ rules: { 'deposit.received': false } });144 await w.db.audit('deposit.seen', 'chain', { network: 'base', txid: '0xa', asset: 'native', amount: '1' });145 await w.db.audit('device.added', 'owner', { label: 'Key' });146 await d.tail();147 expect(await kinds()).toEqual(['security.passkey_added']);148 await settings();149 expect(await d.tail()).toBe(0);150 });151152 it('a rule that fails on one event does not stop the others', async () => {153 await w.db.audit('agent.order_filled', 'chain', { agent: 'not-a-uuid', id: 'not-a-uuid' }); // the lookup of the order fails in SQL154 await w.db.audit('device.added', 'owner', { label: 'Key' });155 await d.tail();156 expect(await kinds()).toEqual(['security.passkey_added']);157 expect(await d.tail()).toBe(0);158 });159});160161describe('exactly once', () => {162 it('a restart neither repeats nor loses an event', async () => {163 for (const l of ['a', 'b', 'c']) await w.db.audit('device.added', 'owner', { label: l });164 expect(await d.tail()).toBe(3);165 const d2 = w.dispatcher(); // a new process: no memory, same database166 expect(await d2.tail()).toBe(0);167 await w.db.audit('device.added', 'owner', { label: 'while-down' }); // written while no dispatcher runs168 const d3 = w.dispatcher();169 expect(await d3.tail()).toBe(1);170 expect(await d.tail()).toBe(0);171 expect((await w.alerts()).map((a) => a.params.label)).toEqual(['a', 'b', 'c', 'while-down']);172 });173174 it('a crash before the commit leaves no alert and no cursor move; the next pass does the work once', async () => {175 const [before] = await sql`select cursor from alert_cursors where source = 'audit'`;176 await w.db.audit('device.added', 'owner', { label: 'x' });177 await w.db.audit('deposit.seen', 'chain', { network: 'base', txid: '0xb', asset: 'native', amount: '1' });178 w.env.fault = (_s, src) => { if (src === 'audit') throw new Error('power cut'); };179 await expect(d.tail()).rejects.toThrow('power cut');180 expect(await w.alerts()).toHaveLength(0); // the alerts written in that transaction are gone with it181 expect((await sql`select cursor from alert_cursors where source = 'audit'`)[0].cursor).toBe(before.cursor);182 w.env.fault = undefined;183 expect(await w.dispatcher().tail()).toBe(2);184 expect(await w.dispatcher().tail()).toBe(0);185 expect(await kinds()).toEqual(['deposit.received', 'security.passkey_added']);186 });187188 it('two dispatchers at the same time produce each alert once', async () => {189 for (let i = 0; i < 20; i++) await w.db.audit('device.added', 'owner', { label: `k${i}` });190 const [a, b] = await Promise.all([w.dispatcher().tail(), w.dispatcher().tail()]);191 expect(a + b).toBe(20);192 expect(await w.alerts()).toHaveLength(20);193 });194195 it('a risk event that commits late (lower id, visible after a higher one) is still alerted once', async () => {196 const venue = await addVenueAccount(sql);197 await d.tail();198 const slow = await sql.reserve();199 await slow`begin`;200 await slow`insert into deriv_risk_events (account_id, ts, kind, code, message, data) values (${venue}, now(), 'divergence', 'state_divergence', 'late', '{}')`; // takes the lower id, not committed201 await riskEvent(sql, venue, 'divergence_cleared', null, 'early'); // higher id, committed first202 expect(await d.tail()).toBe(1);203 expect(await kinds()).toEqual(['deriv.divergence_cleared']);204 await slow`commit`; slow.release();205 expect(await d.tail()).toBe(1); // found in the overlap window behind the cursor206 expect(await d.tail()).toBe(0);207 expect(await w.dispatcher().tail()).toBe(0);208 expect(await kinds()).toEqual(['deriv.divergence', 'deriv.divergence_cleared']);209 });210});211212describe('condition rules', () => {213 const H = 3_600_000;214 it('stuck operations, expiring mandates, stale agents, unknown orders, liquidations, funding — once per episode', async () => {215 await d.conditions(); // sets the epoch: nothing before it is looked at216 w.clock.advance(1000);217 const now = w.clock.t;218 await addProposal(sql, { state: 'broadcast', createdAt: new Date(now - 2 * H) });219 await addProposal(sql, { state: 'broadcast', createdAt: new Date(now - 5 * 60_000) }); // recent: not stuck220 await addProposal(sql, { kind: 'bridge', state: 'finalized', bridge: { phase: 'attesting' }, createdAt: new Date(now - 3 * H) });221 await addProposal(sql, { kind: 'bridge', state: 'finalized', bridge: { phase: 'done' }, createdAt: new Date(now - 3 * H) });222 const mid = randomUUID();223 await addAgent(sql, { name: 'Live one', state: 'LIVE', mandateId: mid, mandate: { expiresAt: Math.floor((now + 60 * H) / 1000) }, lastBar: new Date(now - H) });224 await addAgent(sql, { name: 'Sleepy', state: 'PAPER_TRADING', lastBar: new Date(now - 10 * H) });225 await addAgent(sql, { name: 'Stopped', state: 'STOPPED', lastBar: new Date(now - 100 * H) });226 const venue = await addVenueAccount(sql), paper = await addPaperAccount(sql);227 await sql`insert into deriv_orders (account_id, client_id, instrument, side, type, size, reduce_only, tif, status, created_at, updated_at, state, final)228 values (${venue}, 'u-1', 'hyperliquid:ETH', 'buy', 'limit', 1, false, 'gtc', 'open', now(), ${new Date(now - 120_000)}, 'unknown', false),229 (${venue}, 'u-2', 'hyperliquid:ETH', 'buy', 'limit', 1, false, 'gtc', 'open', now(), ${new Date(now - 5_000)}, 'unknown', false)`;230 await sql`insert into deriv_fills (account_id, fill_id, instrument, side, price, size, fee, fee_asset, liquidation, ts) values (${venue}, 'f-liq', 'hyperliquid:ETH', 'sell', 3000, 1, 0, 'USDC', true, ${new Date(now - 500)})`;231 await sql`insert into deriv_funding (account_id, instrument, ts, amount, rate, size) values (${paper}, 'hyperliquid:BTC', ${new Date(now - 400)}, -7.5, 0.0001, 1), (${paper}, 'hyperliquid:BTC', ${new Date(now - 300)}, 0.4, 0.0001, 1),232 (${paper}, 'hyperliquid:BTC', ${new Date(now - 100 * H)}, -90, 0.0001, 1)`;233 await d.conditions();234 expect(await kinds()).toEqual(['agent.data_stale', 'deriv.funding', 'deriv.live.liquidation', 'deriv.order_unknown', 'mandate.expiring', 'transfer.stuck', 'transfer.stuck']);235 expect((await w.alerts(`kind = 'transfer.stuck'`)).map((a) => [a.params.what, a.params.state, a.params.minutes])).toEqual([['@transfer', '@broadcast', 120], ['@bridge', '@attesting', 180]]);236 expect((await one('mandate.expiring'))).toMatchObject({ severity: 'info', params: { agent: 'Live one', hours: 72 } });237 expect((await one('agent.data_stale')).params).toEqual({ agent: 'Sleepy', bars: 9, interval: '1h' });238 expect((await one('deriv.order_unknown'))).toMatchObject({ severity: 'critical', params: { clientId: 'u-1', seconds: 60 } });239 expect((await one('deriv.funding')).params).toMatchObject({ amount: '-7.50', mode: '@paper' });240 expect((await one('deriv.live.liquidation')).severity).toBe('critical');241 expect(await d.conditions()).toBe(0); // still true, said once242 expect(await w.dispatcher().conditions()).toBe(0); // and once across a restart243 w.clock.advance(40 * H); // 20 h left on the mandate244 await d.conditions();245 expect((await w.alerts(`kind = 'mandate.expiring'`)).map((a) => [a.params.hours, a.severity])).toEqual([[72, 'info'], [24, 'warning']]);246 });247248 it('signer: unreachable only after the debounce, back once; halt, resume and policy change', async () => {249 await d.conditions();250 w.env.signer = null;251 await d.conditions(); w.clock.advance(30_000); await d.conditions();252 w.env.signer = SIGNER_OK; // a blip shorter than the debounce: nothing253 await d.conditions();254 expect(await kinds()).toEqual([]);255 w.env.signer = null;256 await d.conditions(); w.clock.advance(125_000); await d.conditions(); w.clock.advance(60_000); await d.conditions();257 expect(await kinds()).toEqual(['security.signer_unreachable']);258 expect((await one('security.signer_unreachable'))).toMatchObject({ severity: 'critical', params: { minutes: 2 } });259 w.env.signer = SIGNER_OK;260 await d.conditions(); await d.conditions();261 expect(await kinds()).toEqual(['security.signer_back', 'security.signer_unreachable']);262 w.env.signer = { ...SIGNER_OK, halted: true };263 await d.conditions(); await d.conditions();264 w.env.signer = { ...SIGNER_OK, halted: false, policy: { ...SIGNER_OK.policy, version: 4 } as never };265 await d.conditions(); await w.dispatcher().conditions();266 expect(await kinds()).toEqual(['security.policy_changed', 'security.signer_back', 'security.signer_halted', 'security.signer_unhalted', 'security.signer_unreachable']);267 expect((await one('security.policy_changed')).params).toEqual({ what: '@signer', detail: 'version 4' });268 });269270 it('a flapping feed says nothing; a real outage says down once and back once', async () => {271 const h = (readable: boolean) => [{ network: 'base', ok: readable, readable, identityVerified: true, height: 1, endpoints: [{ url: 'u', ok: readable, ms: 10 }], checkedAt: new Date(w.clock.t).toISOString() }];272 for (let i = 0; i < 12; i++) { w.env.rpc = h(i % 2 === 0); await d.conditions(); w.clock.advance(20_000); } // down, up, down, up… every 20 s273 expect(await kinds()).toEqual([]);274 w.env.rpc = h(false); await d.conditions();275 for (let i = 0; i < 12; i++) { w.clock.advance(20_000); w.env.rpc = h(false); await d.conditions(); }276 expect(await kinds()).toEqual(['conn.down']);277 expect((await one('conn.down'))).toMatchObject({ params: { name: 'Base', minutes: 3 }, subject: { type: 'network', network: 'base' } });278 w.env.rpc = h(true); await d.conditions(); await d.conditions();279 expect(await kinds()).toEqual(['conn.down', 'conn.up']);280 });281282 it('backup older than 36 h, low disk, a loop failing repeatedly, a broken audit chain', async () => {283 const now = w.clock.t;284 w.env.backup = { dir: '/x', lastAt: now - 20 * 3_600_000, files: 3 }; w.env.disk = { path: '/', freePct: 42, freeGb: 100, totalGb: 240 };285 w.env.loops = [{ name: 'reconcile', everyMs: 8000, runs: 9, lastStartAt: now, lastOkAt: now, lastErrorAt: null, lastError: null, failures: 0 }];286 await d.conditions();287 expect(await kinds()).toEqual([]);288 w.env.backup = { dir: '/x', lastAt: now - 40 * 3_600_000, files: 3 }; w.env.disk = { path: '/', freePct: 4.2, freeGb: 10, totalGb: 240 };289 w.env.loops = [{ name: 'reconcile', everyMs: 8000, runs: 9, lastStartAt: now, lastOkAt: null, lastErrorAt: now, lastError: 'connection refused', failures: 6 }];290 await d.conditions(); await d.conditions();291 expect(await kinds()).toEqual(['system.backup_missing', 'system.disk_low', 'system.worker_failing']);292 expect((await one('system.backup_missing')).params).toEqual({ hours: 40 });293 expect((await one('system.worker_failing')).params).toEqual({ loop: 'reconcile', count: 6, reason: 'connection refused' });294 w.env.backup = null; // the API cannot see a backup directory: the rule is skipped, not guessed295 const health = await import('../src/monitor/health.ts');296 await sql`alter table audit_events disable trigger audit_no_update`;297 const [row] = await sql`select seq from audit_events order by seq desc offset 2 limit 1`;298 await sql`update audit_events set data = '{"tampered":true}' where seq = ${row.seq}`;299 await sql`alter table audit_events enable trigger audit_no_update`;300 health.resetHealthCaches();301 await d.conditions(); await d.conditions();302 const broken = await one('security.audit_broken');303 expect(broken).toMatchObject({ severity: 'critical', params: { seq: Number(row.seq) } });304 });305});306307describe('delivery', () => {308 const sub = generateSubscriptionKeys();309 const subscribe = () => sql`insert into push_subscriptions (id, endpoint, p256dh, auth, label, locale) values (${randomUUID()}, 'https://fcm.googleapis.com/fcm/send/abc', ${sub.p256dh}, ${sub.auth}, 'iPhone', 'fr')`;310 const pushes = () => calls.filter((c) => c.url.startsWith('https://fcm.')).map((c) => JSON.parse(decryptPush(c.body, sub).toString()));311312 it('with zero subscriptions and no webhook everything still works: alerts stay in the app', async () => {313 await d.raise({ kind: 'deposit.received', params: { network: 'Base', symbol: 'ETH', amount: '1' } });314 expect(await d.deliver()).toEqual({ sent: 0, quiet: 0, capped: 0 });315 expect((await one('deposit.received')).delivery).toBe('inapp');316 expect(calls).toHaveLength(0);317 });318319 it('a push carries a kind and parameters, hides amounts by default, and is sent once', async () => {320 await subscribe();321 await d.raise({ kind: 'transfer.confirmed', params: { network: 'Base', symbol: 'USDC', amount: '25' }, subject: { type: 'proposal', id: 'p1' } });322 expect(await d.deliver()).toMatchObject({ sent: 1 });323 expect(await d.deliver()).toMatchObject({ sent: 0 }); // claimed: never sent twice324 const [p] = pushes();325 expect(p).toMatchObject({ v: 1, kind: 'transfer.confirmed', sev: 'info', l: 'fr', d: false, p: { network: 'Base', count: 1 }, s: { type: 'proposal', id: 'p1' } });326 expect(JSON.stringify((p as { p: unknown }).p)).not.toMatch(/25|USDC/); // the parameters only: the timestamp may contain "25"327 expect(calls[0]!.headers.get('urgency')).toBe('normal');328 expect((await one('transfer.confirmed'))).toMatchObject({ delivery: 'sent', channels: { push: { sent: 1, failed: 0, removed: 0 } } });329 await settings({ showDetails: true });330 await d.raise({ kind: 'transfer.confirmed', params: { network: 'Base', symbol: 'USDC', amount: '25' } });331 await d.deliver();332 expect(pushes()[1]).toMatchObject({ d: true, p: { amount: '25', symbol: 'USDC' } });333 });334335 it('quiet hours hold back non-critical alerts only', async () => {336 await subscribe();337 const hhmm = (t: number) => new Date(t).toISOString().slice(11, 16);338 await settings({ quiet: { enabled: true, start: hhmm(w.clock.t - 3_600_000), end: hhmm(w.clock.t + 3_600_000), tz: 'UTC' } });339 await d.raise({ kind: 'deposit.received', params: { network: 'Base' } });340 await d.raise({ kind: 'security.passkey_added', params: { label: 'k' } });341 expect(await d.deliver()).toEqual({ sent: 1, quiet: 1, capped: 0 });342 expect(pushes().map((p) => p.kind)).toEqual(['security.passkey_added']);343 expect(calls[0]!.headers.get('urgency')).toBe('high');344 expect((await one('deposit.received')).delivery).toBe('quiet'); // in the app, not on the phone345 await d.deliver();346 expect(pushes()).toHaveLength(1); // not sent later either347 });348349 it('quiet hours follow the owner’s time zone and may cross midnight', () => {350 const q = { enabled: true, start: '22:00', end: '07:00', tz: 'America/Toronto' };351 expect(w.mod.inQuietHours(Date.parse('2026-11-11T04:30:00Z'), q)).toBe(true); // 23:30 in Toronto352 expect(w.mod.inQuietHours(Date.parse('2026-11-11T11:59:00Z'), q)).toBe(true); // 06:59353 expect(w.mod.inQuietHours(Date.parse('2026-11-11T12:00:00Z'), q)).toBe(false); // 07:00354 expect(w.mod.inQuietHours(Date.parse('2026-11-11T20:00:00Z'), q)).toBe(false); // 15:00355 expect(w.mod.inQuietHours(Date.parse('2026-07-11T02:30:00Z'), q)).toBe(true); // 22:30 in summer (UTC−4)356 expect(w.mod.inQuietHours(Date.parse('2026-11-11T04:30:00Z'), { ...q, enabled: false })).toBe(false);357 expect(w.mod.inQuietHours(Date.parse('2026-11-11T15:00:00Z'), { ...q, start: '09:00', end: '17:00' })).toBe(true);358 });359360 it('a storm is capped per hour: criticals always pass, the rest is summarized ONCE', async () => {361 await subscribe();362 await settings({ capPerHour: 3 });363 w.clock.t = Math.floor(w.clock.t / 3_600_000) * 3_600_000 + 60_000; // one minute into an hour364 for (let i = 0; i < 10; i++) await d.raise({ kind: 'conn.down', params: { name: `feed ${i}`, minutes: 3 } });365 await d.raise({ kind: 'security.audit_broken', params: { seq: 7 } });366 expect(await d.deliver()).toEqual({ sent: 4, quiet: 0, capped: 7 });367 expect(pushes().map((p) => p.kind)).toEqual(['conn.down', 'conn.down', 'conn.down', 'security.audit_broken']);368 w.clock.advance(20 * 60_000);369 for (let i = 0; i < 5; i++) await d.raise({ kind: 'conn.up', params: { name: `feed ${i}` } });370 expect(await d.deliver()).toEqual({ sent: 0, quiet: 0, capped: 5 });371 expect(await kinds(`kind = 'system.alerts_capped'`)).toEqual([]);372 w.clock.advance(45 * 60_000); // the next hour373 expect(await d.deliver()).toEqual({ sent: 1, quiet: 0, capped: 0 });374 expect((await one('system.alerts_capped'))).toMatchObject({ params: { count: 12 }, delivery: 'sent' });375 expect(pushes().at(-1)).toMatchObject({ kind: 'system.alerts_capped', p: { count: 12 } });376 await d.deliver(); w.clock.advance(3_600_000); await d.deliver();377 expect(await kinds(`kind = 'system.alerts_capped'`)).toHaveLength(1);378 expect(await w.alerts()).toHaveLength(17); // every alert is in the app379 });380381 it('subscriptions the push service no longer knows are removed; other failures are counted and kept', async () => {382 await subscribe();383 pushStatus = 500;384 await d.raise({ kind: 'security.passkey_added', params: { label: 'a' } }); await d.deliver();385 expect((await sql`select failures, last_error from push_subscriptions`)[0]).toMatchObject({ failures: 1, last_error: 'push service answered 500' });386 pushStatus = 201;387 await d.raise({ kind: 'security.passkey_added', params: { label: 'b' } }); await d.deliver();388 expect((await sql`select failures, last_error, last_ok_at from push_subscriptions`)[0]).toMatchObject({ failures: 0, last_error: null });389 pushStatus = 410;390 await d.raise({ kind: 'security.passkey_added', params: { label: 'c' } }); await d.deliver();391 expect(await sql`select 1 from push_subscriptions`).toHaveLength(0);392 expect((await w.alerts()).at(-1).channels).toEqual({ push: { sent: 0, failed: 0, removed: 1 } });393 });394395 it('webhook: off by default; when on, an HMAC-signed body without amounts unless the owner asked for details', async () => {396 const { rotateWebhookSecret } = await import('../src/alerts/secrets.ts');397 await d.raise({ kind: 'transfer.confirmed', params: { network: 'Base', symbol: 'USDC', amount: '25' } }); await d.deliver();398 expect(calls).toHaveLength(0);399 const secret = rotateWebhookSecret();400 await settings({ webhook: { enabled: true, url: 'https://hooks.example/spw', details: false } });401 await d.raise({ kind: 'transfer.confirmed', params: { network: 'Base', symbol: 'USDC', amount: '25' } }); await d.deliver();402 expect(calls).toHaveLength(1);403 const c = calls[0]!, body = c.body.toString();404 expect(c.url).toBe('https://hooks.example/spw');405 expect(verifyWebhook(secret, c.headers.get('x-spw-timestamp')!, c.headers.get('x-spw-signature')!, body, w.clock.t)).toBe(true);406 expect(verifyWebhook(secret, c.headers.get('x-spw-timestamp')!, c.headers.get('x-spw-signature')!, body.replace('Base', 'Bass'), w.clock.t)).toBe(false);407 expect(verifyWebhook('another-secret', c.headers.get('x-spw-timestamp')!, c.headers.get('x-spw-signature')!, body, w.clock.t)).toBe(false);408 expect(verifyWebhook(secret, c.headers.get('x-spw-timestamp')!, c.headers.get('x-spw-signature')!, body, w.clock.t + 10 * 60_000)).toBe(false); // replayed later409 expect(JSON.parse(body)).toMatchObject({ app: 'sp-wallet', kind: 'transfer.confirmed', severity: 'info', title: 'Transfer confirmed', body: 'On Base.', params: { network: 'Base' } });410 // v8: the amount and the symbol are looked for in the fields that could carry them — over the whole body, "25" also411 // matched digits of the timestamp and of the alert id (same family as the push test above)412 { const j = JSON.parse(body) as { title: string; body: string; params: unknown }; expect(JSON.stringify([j.title, j.body, j.params])).not.toMatch(/25|USDC|0x/); expect(Object.keys(j.params as object)).toEqual(['network']); }413 await settings({ webhook: { enabled: true, url: 'https://hooks.example/spw', details: true } });414 await d.raise({ kind: 'transfer.confirmed', params: { network: 'Base', symbol: 'USDC', amount: '25' } }); await d.deliver();415 expect(JSON.parse(calls[1]!.body.toString())).toMatchObject({ body: '25 USDC sent on Base.', params: { amount: '25' } });416 expect((await w.alerts()).at(-1).channels).toEqual({ webhook: { ok: true, status: 201 } });417 });418});419