import { db, providerConfigs, providerHealth, domainProfiles, featureFlags, eq } from "@fetcha/db"; import { ProviderRegistry, type ProviderId } from "@fetcha/providers"; import { CircuitBreaker, FetchExecutor, RoutingEngine, type DomainKnowledge } from "@fetcha/routing"; import { getBrowserPool, type BrowserPool } from "@fetcha/browser"; import { newId } from "@fetcha/core"; import { config } from "../config"; export interface Engine { registry: ProviderRegistry; circuit: CircuitBreaker; routing: RoutingEngine; executor: FetchExecutor; browser: BrowserPool; /** Global kill-switch from the `browser_enabled` feature flag (default on). */ browserEnabled: boolean; reload(): Promise; probeAll(): Promise; knowledge(domain: string): Promise; } let _engine: Engine | null = null; export async function getEngine(): Promise { if (_engine) return _engine; const circuit = new CircuitBreaker(); let registry = new ProviderRegistry(); let routing = new RoutingEngine(registry, circuit); let executor = new FetchExecutor(routing, circuit); const browser = getBrowserPool({ log: { info: (m) => console.log("[browser]", m), warn: (m) => console.warn("[browser]", m) } }); let browserEnabled = true; const engine: Engine = { get registry() { return registry; }, circuit, browser, get browserEnabled() { return browserEnabled && browser.enabled; }, get routing() { return routing; }, get executor() { return executor; }, async reload() { // Prices and enabled flags come from the DB; credentials always from the environment. const rows = await db.select().from(providerConfigs).catch(() => []); const prices: Partial>> = {}; const disabled: ProviderId[] = []; for (const r of rows) { prices[r.id as ProviderId] = r.pricePerGbUsd; if (!r.enabled) disabled.push(r.id as ProviderId); } registry = new ProviderRegistry({ prices, disabled }); routing = new RoutingEngine(registry, circuit); executor = new FetchExecutor(routing, circuit); const [flag] = await db.select().from(featureFlags).where(eq(featureFlags.key, "browser_enabled")).limit(1).catch(() => []); browserEnabled = flag ? flag.enabled : true; }, async probeAll() { const results = await Promise.all( registry.all().flatMap((p) => p.networks.map((n) => p.health(n).catch((e) => ({ provider: p.id, network: n, status: "down" as const, latencyMs: null, detail: (e as Error).message, checkedAt: new Date() })))), ); for (const h of results) { await db .insert(providerHealth) .values({ id: newId("evt"), provider: h.provider, network: h.network, status: h.status, latencyMs: h.latencyMs, detail: h.detail ?? null, checkedAt: h.checkedAt }) .catch(() => {}); } }, async knowledge(domain: string) { if (!domain) return null; const [row] = await db.select().from(domainProfiles).where(eq(domainProfiles.domain, domain)).limit(1).catch(() => []); if (!row) return null; return { domain, routeStats: row.routeStats ?? {}, policy: row.policy ?? null, browserRequiredRate: row.requests ? row.browserRequired / row.requests : 0, browserSamples: row.requests }; }, }; await engine.reload(); _engine = engine; return engine; } export function startHealthLoop(engine: Engine, log: { info: (m: string) => void; warn: (m: string) => void }): () => void { let stopped = false; let ticks = 0; const run = async () => { if (stopped) return; try { await engine.reload(); await engine.probeAll(); log.info("provider health probe complete"); } catch (e) { log.warn(`provider health probe failed: ${(e as Error).message}`); } // Retention prune roughly every 6 hours (every Nth probe), plus once shortly after boot. const every = Math.max(1, Math.round((6 * 60 * 60_000) / config.healthProbeIntervalMs)); if (ticks % every === 0) { try { const { pruneRetention } = await import("./retention"); await pruneRetention(log); } catch (e) { log.warn(`retention prune failed: ${(e as Error).message}`); } } ticks++; }; const t = setTimeout(run, 5_000); const interval = setInterval(run, config.healthProbeIntervalMs); return () => { stopped = true; clearTimeout(t); clearInterval(interval); }; }