1import { childLogger, sha256, type Logger } from '@rareindex/shared';2import type { Router } from './router.js';3import type { ConnectorMeta, CrawlContext, CrawlOptions, EngineStats, FetchOptions, HealthContext } from './types.js';45export interface BudgetStore {6 /** return previously stored content hash + next due time for url */7 get(urlHash: string): Promise<{ contentHash: string | null; nextFetchAt: Date | null } | null>;8 /** record a fetch outcome; implementation updates change-interval estimate (§170) */9 record(input: { urlHash: string; url: string; connectorId: string; contentHash: string | null; changed: boolean; status: number | null }): Promise<void>;10}1112export interface ContextDeps {13 router: Router;14 meta: ConnectorMeta;15 options: CrawlOptions;16 log?: Logger;17 budget?: BudgetStore;18 onCursor?: (cursor: Record<string, unknown>) => Promise<void>;19 onAnomaly?: (kind: string, detail?: string) => void;20 signal?: AbortSignal;21}2223/** Build the CrawlContext handed to connectors; wires the router, budget store and stats. */24export function createCrawlContext(deps: ContextDeps): CrawlContext & { anomalies: string[] } {25 const engineStats: Record<string, EngineStats> = {};26 const anomalies: string[] = [];27 const log = deps.log ?? childLogger({ connector: deps.meta.id });28 const fc = deps.router.engines.firecrawl;2930 const ctx: CrawlContext & { anomalies: string[] } = {31 meta: deps.meta,32 log,33 options: deps.options,34 engineStats,35 anomalies,36 signal: deps.signal,37 async fetch(url: string, opts: FetchOptions = {}) {38 const res = await deps.router.fetch(url, opts, { enginePriority: deps.meta.enginePriority, stats: engineStats });39 if (deps.budget && !opts.force) {40 const urlHash = sha256(url);41 const content = res.json !== null && res.json !== undefined ? JSON.stringify(res.json) : (res.html ?? res.markdown ?? '');42 const contentHash = content ? sha256(content) : null;43 const prev = await deps.budget.get(urlHash);44 await deps.budget.record({ urlHash, url, connectorId: deps.meta.id, contentHash, changed: prev?.contentHash !== contentHash, status: res.httpStatus });45 }46 return res;47 },48 search: fc49 ? async (query, o = {}) => {50 const r = await fc.search(query, o);51 const s = (engineStats.firecrawl ??= { attempts: 0, success: 0, credits: 0, ms: 0 });52 s.attempts++;53 s.success++;54 s.credits += r.credits;55 return r.items;56 }57 : undefined,58 map: fc59 ? async (url, o = {}) => {60 const r = await fc.map(url, o);61 const s = (engineStats.firecrawl ??= { attempts: 0, success: 0, credits: 0, ms: 0 });62 s.attempts++;63 s.success++;64 s.credits += r.credits;65 return r.links;66 }67 : undefined,68 async shouldFetch(url: string) {69 if (!deps.budget || deps.options.mode === 'backfill') return true;70 const prev = await deps.budget.get(sha256(url));71 if (!prev || !prev.nextFetchAt) return true;72 return prev.nextFetchAt.getTime() <= Date.now();73 },74 async setCursor(cursor) {75 await deps.onCursor?.(cursor);76 },77 anomaly(kind, detail) {78 const s = detail ? `${kind}: ${detail}` : kind;79 anomalies.push(s);80 deps.onAnomaly?.(kind, detail);81 log.warn({ kind, detail }, 'connector anomaly');82 },83 };84 return ctx;85}8687export function createHealthContext(deps: { router: Router; meta: ConnectorMeta; recentRuns: HealthContext['recentRuns']; log?: Logger }): HealthContext {88 return {89 meta: deps.meta,90 log: deps.log ?? childLogger({ connector: deps.meta.id }),91 recentRuns: deps.recentRuns,92 fetch: (url, opts) => deps.router.fetch(url, opts, { enginePriority: deps.meta.enginePriority }),93 };94}95