/** * Per-host fault isolation (SPEC §26): a concurrency gate, a politeness interval and a circuit * breaker. One broken or hostile domain must never stall the whole ingestion pipeline — after * `failures` consecutive failures the circuit opens and requests fail fast until the cooldown ends. * A single trial request is allowed when the cooldown elapses (half-open); success closes it. */ export interface HostGateOptions { concurrency: number; minIntervalMs: number; circuitFailures: number; circuitCooldownMs: number; } export type CircuitState = 'closed' | 'open' | 'half_open'; interface HostState { inFlight: number; waiters: Array<() => void>; lastStart: number; consecutiveFailures: number; openedAt: number | null; state: CircuitState; trialInFlight: boolean; } export class CircuitOpenError extends Error { constructor(public host: string, public retryAt: Date) { super(`circuit open for ${host} until ${retryAt.toISOString()}`); } } export class HostGates { private hosts = new Map(); constructor(private resolve: (host: string) => HostGateOptions) {} private get(host: string): HostState { let s = this.hosts.get(host); if (!s) { s = { inFlight: 0, waiters: [], lastStart: 0, consecutiveFailures: 0, openedAt: null, state: 'closed', trialInFlight: false }; this.hosts.set(host, s); } return s; } snapshot(host: string): { state: CircuitState; consecutiveFailures: number; inFlight: number; retryAt: Date | null } { const s = this.get(host); const o = this.resolve(host); return { state: s.state, consecutiveFailures: s.consecutiveFailures, inFlight: s.inFlight, retryAt: s.openedAt ? new Date(s.openedAt + o.circuitCooldownMs) : null }; } /** Acquire a slot; throws CircuitOpenError immediately when the circuit is open. */ async acquire(host: string): Promise<() => void> { const s = this.get(host); const o = this.resolve(host); if (s.state === 'open') { if (s.openedAt !== null && Date.now() - s.openedAt >= o.circuitCooldownMs) { s.state = 'half_open'; } else { throw new CircuitOpenError(host, new Date((s.openedAt ?? Date.now()) + o.circuitCooldownMs)); } } if (s.state === 'half_open') { if (s.trialInFlight) throw new CircuitOpenError(host, new Date(Date.now() + 5_000)); s.trialInFlight = true; } while (s.inFlight >= o.concurrency) await new Promise((r) => s.waiters.push(r)); s.inFlight++; const wait = s.lastStart + o.minIntervalMs - Date.now(); if (wait > 0) await new Promise((r) => setTimeout(r, wait)); s.lastStart = Date.now(); let released = false; return () => { if (released) return; released = true; s.inFlight--; s.waiters.shift()?.(); }; } /** Report the outcome of a request made under `acquire`. */ report(host: string, ok: boolean): void { const s = this.get(host); const o = this.resolve(host); if (ok) { s.consecutiveFailures = 0; s.openedAt = null; s.state = 'closed'; s.trialInFlight = false; return; } s.consecutiveFailures++; if (s.state === 'half_open') { s.trialInFlight = false; s.state = 'open'; s.openedAt = Date.now(); return; } if (s.consecutiveFailures >= o.circuitFailures) { s.state = 'open'; s.openedAt = Date.now(); } } /** Manually reset a host (admin "resume"). */ reset(host: string): void { this.hosts.delete(host); } hostsSnapshot(): Array<{ host: string; state: CircuitState; consecutiveFailures: number; inFlight: number }> { return [...this.hosts.entries()].map(([host, s]) => ({ host, state: s.state, consecutiveFailures: s.consecutiveFailures, inFlight: s.inFlight })); } }