1import type { Engine, ExtractionResult, Logger } from '@rareindex/shared';2import { createHttpEngine, type HttpEngine } from './engines/http.js';3import { createFirecrawlEngine, type FirecrawlEngine } from './engines/firecrawl.js';4import { createScrapflyEngine, type ScrapflyEngine } from './engines/scrapfly.js';5import { documentQuality, qualityScore, MIN_QUALITY } from './quality.js';6import type { EngineStats, FetchOptions } from './types.js';78export interface RouterOptions {9 firecrawlApiKey?: string;10 scrapflyApiKey?: string;11 log?: Logger;12 /** called for every engine attempt (cost ledger, metrics) */13 onAttempt?: (info: { engine: Engine; url: string; success: boolean; credits: number; ms: number; quality: number; error: string | null }) => void;14}1516export interface Router {17 fetch(url: string, opts?: FetchOptions, defaults?: { enginePriority: Engine[]; stats?: Record<string, EngineStats> }): Promise<ExtractionResult>;18 engines: { http: HttpEngine; firecrawl: FirecrawlEngine | null; scrapfly: ScrapflyEngine | null };19 available(): Engine[];20}2122/**23 * Firecrawl → Scrapfly router (§198). Tries engines in priority order, scores each result with the24 * connector's parser (or a document heuristic), and falls through until quality ≥ minQuality.25 * A result that fails everywhere is returned with requiresReview=true — never silently dropped.26 */27export function createRouter(opts: RouterOptions = {}): Router {28 const http = createHttpEngine();29 const firecrawl = opts.firecrawlApiKey ? createFirecrawlEngine({ apiKey: opts.firecrawlApiKey }) : null;30 const scrapfly = opts.scrapflyApiKey ? createScrapflyEngine({ apiKey: opts.scrapflyApiKey }) : null;3132 function available(): Engine[] {33 const out: Engine[] = ['api', 'feed'];34 if (firecrawl) out.push('firecrawl');35 if (scrapfly) out.push('scrapfly');36 return out;37 }3839 async function runEngine(engine: Engine, url: string, o: FetchOptions): Promise<ExtractionResult | null> {40 switch (engine) {41 case 'api':42 case 'feed':43 return http.fetch(url, o);44 case 'firecrawl':45 return firecrawl ? firecrawl.scrape(url, o) : null;46 case 'scrapfly':47 return scrapfly ? scrapfly.scrape(url, o) : null;48 default:49 return null;50 }51 }5253 async function routedFetch(url: string, o: FetchOptions = {}, defaults?: { enginePriority: Engine[]; stats?: Record<string, EngineStats> }): Promise<ExtractionResult> {54 const engines = (o.engines ?? defaults?.enginePriority ?? ['api', 'firecrawl', 'scrapfly']).filter((e) => available().includes(e));55 const minQuality = o.minQuality ?? MIN_QUALITY;56 let last: ExtractionResult | null = null;57 const errors: string[] = [];58 for (const engine of engines) {59 const res = await runEngine(engine, url, o);60 if (!res) continue;61 let quality = 0;62 if (res.success) {63 if (o.parse) {64 try {65 const found = o.parse(res);66 quality = qualityScore(found, o.expect);67 } catch (err) {68 quality = 0;69 res.error = `parse: ${err instanceof Error ? err.message : String(err)}`;70 }71 } else {72 quality = documentQuality(res);73 }74 }75 res.qualityScore = quality;76 const stats = defaults?.stats;77 if (stats) {78 const s = (stats[engine] ??= { attempts: 0, success: 0, credits: 0, ms: 0 });79 s.attempts++;80 s.credits += res.costCredits ?? 0;81 s.ms += res.durationMs ?? 0;82 if (res.success && quality >= minQuality) s.success++;83 }84 opts.onAttempt?.({ engine, url, success: res.success && quality >= minQuality, credits: res.costCredits ?? 0, ms: res.durationMs ?? 0, quality, error: res.error });85 if (res.success && quality >= minQuality) return res;86 errors.push(`${engine}: ${res.error ?? `quality ${quality}`}`);87 last = res;88 opts.log?.debug({ url, engine, quality, error: res.error }, 'engine fallback');89 // Do not escalate on hard "not found"/"gone" statuses: the page does not exist.90 if (res.httpStatus === 404 || res.httpStatus === 410) break;91 }92 return {93 ...(last ?? { engine: engines[0] ?? 'api', url, finalUrl: null, httpStatus: null, html: null, markdown: null, json: null, costCredits: 0, durationMs: 0, fetchedAt: new Date(), qualityScore: 0 }),94 success: false,95 requiresReview: true,96 error: errors.join(' | ') || 'no engine available',97 };98 }99100 return { fetch: routedFetch, engines: { http, firecrawl, scrapfly }, available };101}102