TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { childLogger, sha256, type Logger } from '@rareindex/shared';2import type { Router } from './router.js';3import type { BackfillProgress, ConnectorMeta, CrawlContext, CrawlOptions, EngineStats, FetchOptions, HealthContext } from './types.js';45export interface BudgetStore {6 /** return previously stored content hash + next due time for url */7 get(urlHash: string): Promise<{ contentHash: string | null; nextFetchAt: Date | null } | null>;8 /** record a fetch outcome; implementation updates change-interval estimate (§170) */9 record(input: { urlHash: string; url: string; connectorId: string; contentHash: string | null; changed: boolean; status: number | null }): Promise<void>;10}1112export interface ContextDeps {13 router: Router;14 meta: ConnectorMeta;15 options: CrawlOptions;16 log?: Logger;17 budget?: BudgetStore;18 onCursor?: (cursor: Record<string, unknown>) => Promise<void>;19 onAnomaly?: (kind: string, detail?: string) => void;20 /** backfill progress sink (SPEC §9); only called in backfill mode */21 onProgress?: (p: BackfillProgress) => Promise<void>;22 signal?: AbortSignal;23}2425/** Build the CrawlContext handed to connectors; wires the router, budget store and stats. */26export function createCrawlContext(deps: ContextDeps): CrawlContext & { anomalies: string[] } {27 const engineStats: Record<string, EngineStats> = {};28 const anomalies: string[] = [];29 const log = deps.log ?? childLogger({ connector: deps.meta.id });30 const fc = deps.router.engines.firecrawl;3132 const ctx: CrawlContext & { anomalies: string[] } = {33 meta: deps.meta,34 log,35 options: deps.options,36 engineStats,37 anomalies,38 signal: deps.signal,39 async fetch(url: string, opts: FetchOptions = {}) {40 const res = await deps.router.fetch(url, opts, { enginePriority: deps.meta.enginePriority, stats: engineStats });41 if (deps.budget && !opts.force) {42 const urlHash = sha256(url);43 const content = res.json !== null && res.json !== undefined ? JSON.stringify(res.json) : (res.html ?? res.markdown ?? '');44 const contentHash = content ? sha256(content) : null;45 const prev = await deps.budget.get(urlHash);46 await deps.budget.record({ urlHash, url, connectorId: deps.meta.id, contentHash, changed: prev?.contentHash !== contentHash, status: res.httpStatus });47 }48 return res;49 },50 search: fc51 ? async (query, o = {}) => {52 const r = await fc.search(query, o);53 const s = (engineStats.firecrawl ??= { attempts: 0, success: 0, credits: 0, ms: 0 });54 s.attempts++;55 s.success++;56 s.credits += r.credits;57 return r.items;58 }59 : undefined,60 map: fc61 ? async (url, o = {}) => {62 const r = await fc.map(url, o);63 const s = (engineStats.firecrawl ??= { attempts: 0, success: 0, credits: 0, ms: 0 });64 s.attempts++;65 s.success++;66 s.credits += r.credits;67 return r.links;68 }69 : undefined,70 async shouldFetch(url: string) {71 if (!deps.budget || deps.options.mode === 'backfill') return true;72 const prev = await deps.budget.get(sha256(url));73 if (!prev || !prev.nextFetchAt) return true;74 return prev.nextFetchAt.getTime() <= Date.now();75 },76 async setCursor(cursor) {77 await deps.onCursor?.(cursor);78 },79 anomaly(kind, detail) {80 const s = detail ? `${kind}: ${detail}` : kind;81 anomalies.push(s);82 deps.onAnomaly?.(kind, detail);83 log.warn({ kind, detail }, 'connector anomaly');84 },85 async progress(p) {86 if (deps.options.mode !== 'backfill') return;87 if (p.cursor) await deps.onCursor?.(p.cursor);88 await deps.onProgress?.(p);89 },90 };91 return ctx;92}9394export function createHealthContext(deps: { router: Router; meta: ConnectorMeta; recentRuns: HealthContext['recentRuns']; log?: Logger }): HealthContext {95 return {96 meta: deps.meta,97 log: deps.log ?? childLogger({ connector: deps.meta.id }),98 recentRuns: deps.recentRuns,99 fetch: (url, opts) => deps.router.fetch(url, { ...opts, ignoreCircuit: true }, { enginePriority: deps.meta.enginePriority }),100 };101}102