Repository Public
1// Health of everything the wallet depends on, read PASSIVELY: this module never calls a venue, a swap provider or Circle. It reads2// what the workers already recorded (RPC health cache, market quality table, quotes log, equity snapshots, bridge rows, worker3// heartbeats) plus the signer status the API already polls. Shared by the dashboard and by the "source down" alerts.4import { existsSync, readdirSync, statSync } from 'node:fs';5import { statfs } from 'node:fs/promises';6import { homedir } from 'node:os';7import { join } from 'node:path';8import { NETWORKS, verifyAuditChain, type AuditRecord, type NetworkDef } from '@spw/core';9import type { Health } from '../chains/index.ts';10import { sql } from '../db.ts';11import type { SignerStatus } from '../signer-client.ts';12import type { LoopStat } from '../worker.ts';1314export interface Connector {15 id: string; group: 'signer' | 'rpc' | 'market' | 'venue' | 'swap' | 'bridge' | 'database';16 name: string;17 /** null = not measured (nothing in use that would tell) — never shown as healthy */18 ok: boolean | null;19 lastSuccessAt: number | null; detail: string;20}21export interface HealthDeps {22 now(): number;23 /** null when the signer does not answer */24 signerStatus(): Promise<SignerStatus | null>;25 rpcHealth(): Health[];26 loops(): LoopStat[];27}2829const lastOk = new Map<string, number>();30const seen = (id: string, ok: boolean | null, at: number | null): number | null => { if (ok && at) lastOk.set(id, Math.max(at, lastOk.get(id) ?? 0)); return lastOk.get(id) ?? null; };31const NETS = Object.values(NETWORKS) as NetworkDef[];3233export async function collectConnectors(d: HealthDeps): Promise<{ connectors: Connector[]; signer: SignerStatus | null }> {34 const now = d.now(), out: Connector[] = [];35 const signer = await d.signerStatus().catch(() => null);36 out.push({ id: 'signer', group: 'signer', name: 'Signer', ok: signer !== null, lastSuccessAt: seen('signer', signer !== null, now), detail: signer ? (signer.halted ? 'halted' : 'answering') : 'no answer' });37 const db = await sql`select 1`.then(() => true, () => false);38 out.push({ id: 'database', group: 'database', name: 'PostgreSQL', ok: db, lastSuccessAt: seen('database', db, now), detail: db ? 'answering' : 'no answer' });3940 const rpc = new Map(d.rpcHealth().map((h) => [h.network, h]));41 for (const n of NETS) {42 const h = rpc.get(n.id), id = `rpc:${n.id}`;43 if (!h) { out.push({ id, group: 'rpc', name: n.name, ok: null, lastSuccessAt: lastOk.get(id) ?? null, detail: 'not checked yet' }); continue; }44 const at = Date.parse(h.checkedAt), up = h.endpoints.filter((e) => e.ok).length;45 // "down" means unreadable: one slow public endpoint out of several is said, but is not an outage46 out.push({ id, group: 'rpc', name: n.name, ok: h.readable, lastSuccessAt: seen(id, h.readable, at), detail: `${up}/${h.endpoints.length} endpoints${h.readable && !h.ok ? ', degraded' : ''}${h.identityVerified ? '' : ', identity not verified'}` });47 }4849 // market data: one row per source, from the newest BTC bar it delivered (index scan on one symbol)50 const src = await sql<{ source: string; last: Date }[]>`select source, max(ts) as last from candles where symbol = 'BTC' group by source`.catch(() => []);51 const [mq] = await sql<{ n: number; fresh: number; checked: Date | null }[]>`select count(*)::int as n, (count(*) filter (where fresh))::int as fresh, max(checked_at) as checked from market_quality`.catch(() => []);52 for (const s of src) {53 const id = `market:${s.source}`, age = now - s.last.getTime(), ok = age < 3 * 3_600_000; // hourly bars: two missed closes54 out.push({ id, group: 'market', name: `Market data · ${s.source}`, ok, lastSuccessAt: seen(id, true, s.last.getTime()), detail: 'newest BTC bar' });55 }56 if (mq && mq.n > 0) {57 const checkedAt = mq.checked?.getTime() ?? 0, running = now - checkedAt < 10 * 60_000;58 out.push({ id: 'market:quality', group: 'market', name: 'Market data · quality', ok: running && mq.fresh * 2 >= mq.n, lastSuccessAt: seen('market:quality', running, checkedAt), detail: `${mq.fresh}/${mq.n} series fresh` });59 }6061 // derivatives venues: a venue account being reconciled (or a paper account with exposure) gets an equity snapshot every minute62 const venues = await sql<{ venue: string; network: string; kind: string; active: number; last: Date | null }[]>`63 select a.venue, a.network, 'venue' as kind, (count(*) filter (where a.mode <> 'READ_ONLY'))::int as active, max(e.ts) as last64 from deriv_accounts a left join lateral (select max(ts) as ts from deriv_equity where account_id = a.id) e on true group by a.venue, a.network65 union all66 select a.venue, a.network, 'paper' as kind, (count(*) filter (where jsonb_array_length(a.state->'positions') > 0))::int as active, max(e.ts) as last67 from deriv_paper_accounts a left join lateral (select max(ts) as ts from deriv_equity where account_id = a.id) e on true group by a.venue, a.network`.catch(() => []);68 for (const v of venues) {69 const id = `venue:${v.venue}:${v.network}:${v.kind}`, at = v.last?.getTime() ?? null;70 const ok = v.active > 0 ? at !== null && now - at < 5 * 60_000 : null;71 out.push({ id, group: 'venue', name: `${v.venue} ${v.network}${v.kind === 'paper' ? ' (paper feed)' : ''}`, ok, lastSuccessAt: seen(id, at !== null, at) ?? at,72 detail: v.active > 0 ? 'last equity snapshot' : 'no active account: not measured' });73 }7475 // swap providers: the quotes the owner (or an agent) actually asked for76 const quotes = await sql<{ provider: string; last_ok: Date | null; last: Date; ok: boolean }[]>`77 select provider, max(at) filter (where ok) as last_ok, max(at) as last, (array_agg(ok order by at desc))[1] as ok from swap_quotes where at > now() - interval '7 days' group by provider order by provider`.catch(() => []);78 for (const s of quotes) out.push({ id: `swap:${s.provider}`, group: 'swap', name: `Swap · ${s.provider}`, ok: now - s.last.getTime() < 6 * 3_600_000 ? s.ok : null,79 lastSuccessAt: s.last_ok?.getTime() ?? null, detail: now - s.last.getTime() < 6 * 3_600_000 ? 'last quote asked' : 'no recent quote: not measured' });8081 // Circle attestation API: only a bridge in flight tells82 const [br] = await sql<{ last: Date | null; waiting: number; overdue: number }[]>`83 select max(updated_at) filter (where bridge->>'message' is not null) as last,84 (count(*) filter (where bridge->>'phase' = 'attesting' and state not in ('failed','rejected','expired')))::int as waiting,85 (count(*) filter (where bridge->>'phase' = 'attesting' and state not in ('failed','rejected','expired') and updated_at < now() - interval '45 minutes'))::int as overdue86 from proposals where kind = 'bridge'`.catch(() => []);87 out.push({ id: 'bridge:circle', group: 'bridge', name: 'Circle attestation (CCTP)', ok: br && br.waiting > 0 ? br.overdue === 0 : null, lastSuccessAt: br?.last?.getTime() ?? null,88 detail: br && br.waiting > 0 ? `${br.waiting} transfer(s) awaiting attestation` : 'no transfer in flight: not measured' });89 return { connectors: out, signer };90}9192/** Where scripts/backup.sh writes (SPW_BACKUP_DIR, default ~/backups/sp-wallet). null when the API cannot see such a directory. */93export function lastBackup(dir = process.env.SPW_BACKUP_DIR || join(homedir(), 'backups', 'sp-wallet')): { dir: string; lastAt: number | null; files: number } | null {94 try {95 if (!existsSync(dir)) return null;96 const files = readdirSync(dir).filter((f) => /^sp-wallet-.*\.sql\.gz$/.test(f));97 let lastAt: number | null = null;98 for (const f of files) { const m = statSync(join(dir, f)).mtimeMs; if (lastAt === null || m > lastAt) lastAt = m; }99 return { dir, lastAt, files: files.length };100 } catch { return null; }101}102103export async function diskFree(path = process.env.SPW_DISK_PATH || homedir()): Promise<{ path: string; freePct: number; freeGb: number; totalGb: number } | null> {104 try {105 const s = await statfs(path), total = Number(s.blocks) * Number(s.bsize), free = Number(s.bavail) * Number(s.bsize);106 if (!(total > 0)) return null;107 return { path, freePct: Math.round((free / total) * 1000) / 10, freeGb: Math.round(free / 1e8) / 10, totalGb: Math.round(total / 1e8) / 10 };108 } catch { return null; }109}110111let auditCache: { at: number; total: number; intact: boolean; brokenAt: number | null } | null = null;112/** Full verification of the hash chain, at most once every `maxAgeMs` (it reads the whole table). */113export async function auditChainStatus(now: number, maxAgeMs = 10 * 60_000): Promise<{ checkedAt: number; total: number; intact: boolean; brokenAt: number | null }> {114 if (!auditCache || now - auditCache.at >= maxAgeMs) {115 const all = await sql<AuditRecord[]>`select seq::int as seq, ts, type, actor, data, prev, hash from audit_events order by seq asc`;116 const brokenAt = verifyAuditChain(all);117 auditCache = { at: now, total: all.length, intact: brokenAt === null, brokenAt };118 }119 return { checkedAt: auditCache.at, total: auditCache.total, intact: auditCache.intact, brokenAt: auditCache.brokenAt };120}121export const resetHealthCaches = (): void => { auditCache = null; lastOk.clear(); };122123declare const __SPW_COMMIT__: string | undefined;124const STARTED_AT = Date.now();125/** versions and identifiers only — never a secret */126export function versions(signer: SignerStatus | null): { apiCommit: string | null; apiVersion: string; node: string; startedAt: number; signerKeyId: string | null; signerPolicyVersion: number | null } {127 const commit = (typeof __SPW_COMMIT__ === 'string' && __SPW_COMMIT__) || process.env.SPW_COMMIT || null;128 return { apiCommit: commit, apiVersion: '1.0.0', node: process.version, startedAt: STARTED_AT, signerKeyId: signer?.keyId ?? null, signerPolicyVersion: (signer?.policy as { version?: number } | undefined)?.version ?? null };129}130