import { childLogger, sha256, type Logger } from '@rareindex/shared'; import type { Router } from './router.js'; import type { BackfillProgress, ConnectorMeta, CrawlContext, CrawlOptions, EngineStats, FetchOptions, HealthContext } from './types.js'; export interface BudgetStore { /** return previously stored content hash + next due time for url */ get(urlHash: string): Promise<{ contentHash: string | null; nextFetchAt: Date | null } | null>; /** record a fetch outcome; implementation updates change-interval estimate (§170) */ record(input: { urlHash: string; url: string; connectorId: string; contentHash: string | null; changed: boolean; status: number | null }): Promise; } export interface ContextDeps { router: Router; meta: ConnectorMeta; options: CrawlOptions; log?: Logger; budget?: BudgetStore; onCursor?: (cursor: Record) => Promise; onAnomaly?: (kind: string, detail?: string) => void; /** backfill progress sink (SPEC §9); only called in backfill mode */ onProgress?: (p: BackfillProgress) => Promise; signal?: AbortSignal; } /** Build the CrawlContext handed to connectors; wires the router, budget store and stats. */ export function createCrawlContext(deps: ContextDeps): CrawlContext & { anomalies: string[] } { const engineStats: Record = {}; const anomalies: string[] = []; const log = deps.log ?? childLogger({ connector: deps.meta.id }); const fc = deps.router.engines.firecrawl; const ctx: CrawlContext & { anomalies: string[] } = { meta: deps.meta, log, options: deps.options, engineStats, anomalies, signal: deps.signal, async fetch(url: string, opts: FetchOptions = {}) { const res = await deps.router.fetch(url, opts, { enginePriority: deps.meta.enginePriority, stats: engineStats }); if (deps.budget && !opts.force) { const urlHash = sha256(url); const content = res.json !== null && res.json !== undefined ? JSON.stringify(res.json) : (res.html ?? res.markdown ?? ''); const contentHash = content ? sha256(content) : null; const prev = await deps.budget.get(urlHash); await deps.budget.record({ urlHash, url, connectorId: deps.meta.id, contentHash, changed: prev?.contentHash !== contentHash, status: res.httpStatus }); } return res; }, search: fc ? async (query, o = {}) => { const r = await fc.search(query, o); const s = (engineStats.firecrawl ??= { attempts: 0, success: 0, credits: 0, ms: 0 }); s.attempts++; s.success++; s.credits += r.credits; return r.items; } : undefined, map: fc ? async (url, o = {}) => { const r = await fc.map(url, o); const s = (engineStats.firecrawl ??= { attempts: 0, success: 0, credits: 0, ms: 0 }); s.attempts++; s.success++; s.credits += r.credits; return r.links; } : undefined, async shouldFetch(url: string) { if (!deps.budget || deps.options.mode === 'backfill') return true; const prev = await deps.budget.get(sha256(url)); if (!prev || !prev.nextFetchAt) return true; return prev.nextFetchAt.getTime() <= Date.now(); }, async setCursor(cursor) { await deps.onCursor?.(cursor); }, anomaly(kind, detail) { const s = detail ? `${kind}: ${detail}` : kind; anomalies.push(s); deps.onAnomaly?.(kind, detail); log.warn({ kind, detail }, 'connector anomaly'); }, async progress(p) { if (deps.options.mode !== 'backfill') return; if (p.cursor) await deps.onCursor?.(p.cursor); await deps.onProgress?.(p); }, }; return ctx; } export function createHealthContext(deps: { router: Router; meta: ConnectorMeta; recentRuns: HealthContext['recentRuns']; log?: Logger }): HealthContext { return { meta: deps.meta, log: deps.log ?? childLogger({ connector: deps.meta.id }), recentRuns: deps.recentRuns, fetch: (url, opts) => deps.router.fetch(url, { ...opts, ignoreCircuit: true }, { enginePriority: deps.meta.enginePriority }), }; }