spb/rareindex

1import type { ExtractionResult } from '@rareindex/shared';2import type { FetchOptions } from '../types.js';34export interface FirecrawlEngineOptions {5  apiKey: string;6  baseUrl?: string;7  defaultTimeoutMs?: number;8}910interface FirecrawlScrapeResponse {11  success: boolean;12  data?: {13    markdown?: string;14    html?: string;15    rawHtml?: string;16    json?: unknown;17    links?: string[];18    metadata?: { statusCode?: number; sourceURL?: string; url?: string; title?: string; error?: string; creditsUsed?: number };19  };20  error?: string;21  creditsUsed?: number;22}2324/**25 * Firecrawl v2 REST client (§103). Preferred engine for parseable public pages: returns markdown +26 * HTML (+ optional JSON extraction with a schema). Never used to bypass access controls (§179).27 */28export function createFirecrawlEngine(opts: FirecrawlEngineOptions) {29  const base = (opts.baseUrl ?? 'https://api.firecrawl.dev').replace(/\/$/, '');30  const headers = { authorization: `Bearer ${opts.apiKey}`, 'content-type': 'application/json' };3132  async function post<T>(path: string, body: unknown, timeoutMs: number): Promise<T> {33    const ctrl = new AbortController();34    const timer = setTimeout(() => ctrl.abort(), timeoutMs + 5_000);35    try {36      const res = await fetch(`${base}${path}`, { method: 'POST', headers, body: JSON.stringify(body), signal: ctrl.signal });37      const text = await res.text();38      let json: unknown = null;39      try {40        json = JSON.parse(text);41      } catch {42        throw new Error(`firecrawl ${path} HTTP ${res.status}: ${text.slice(0, 200)}`);43      }44      if (!res.ok) {45        const msg = (json as { error?: string })?.error ?? text.slice(0, 200);46        throw new Error(`firecrawl ${path} HTTP ${res.status}: ${msg}`);47      }48      return json as T;49    } finally {50      clearTimeout(timer);51    }52  }5354  async function scrape(url: string, o: FetchOptions = {}): Promise<ExtractionResult> {55    const started = Date.now();56    const timeoutMs = o.timeoutMs ?? opts.defaultTimeoutMs ?? 60_000;57    const formats: unknown[] = ['markdown', 'html', 'rawHtml'];58    if (o.jsonSchema || o.jsonPrompt) formats.push({ type: 'json', ...(o.jsonSchema ? { schema: o.jsonSchema } : {}), ...(o.jsonPrompt ? { prompt: o.jsonPrompt } : {}) });59    const body: Record<string, unknown> = {60      url,61      formats,62      onlyMainContent: false,63      timeout: timeoutMs,64      ...(o.waitForMs ? { waitFor: o.waitForMs } : {}),65      ...(o.country ? { location: { country: o.country.toUpperCase() } } : {}),66      ...(o.headers ? { headers: o.headers } : {}),67    };68    try {69      const r = await post<FirecrawlScrapeResponse>('/v2/scrape', body, timeoutMs);70      const d = r.data;71      const status = d?.metadata?.statusCode ?? null;72      const ok = Boolean(r.success && d && (status === null || status < 400));73      return {74        success: ok,75        engine: 'firecrawl',76        url,77        finalUrl: d?.metadata?.url ?? d?.metadata?.sourceURL ?? null,78        httpStatus: status,79        // rawHtml keeps <script> blocks (__NEXT_DATA__, JSON-LD); the cleaned html is the fallback.80        html: d?.rawHtml ?? d?.html ?? null,81        markdown: d?.markdown ?? null,82        json: d?.json ?? null,83        qualityScore: 0,84        requiresReview: false,85        error: ok ? null : (r.error ?? d?.metadata?.error ?? `status ${status}`),86        costCredits: r.creditsUsed ?? d?.metadata?.creditsUsed ?? 1,87        durationMs: Date.now() - started,88        fetchedAt: new Date(),89      };90    } catch (err) {91      return { success: false, engine: 'firecrawl', url, finalUrl: null, httpStatus: null, html: null, markdown: null, json: null, qualityScore: 0, requiresReview: false, error: err instanceof Error ? err.message : String(err), costCredits: 0, durationMs: Date.now() - started, fetchedAt: new Date() };92    }93  }9495  async function search(query: string, o: { limit?: number; scrape?: boolean; country?: string; tbs?: string } = {}) {96    const body: Record<string, unknown> = { query, limit: o.limit ?? 10, ...(o.country ? { country: o.country } : {}), ...(o.tbs ? { tbs: o.tbs } : {}) };97    if (o.scrape) body.scrapeOptions = { formats: ['markdown'] };98    const r = await post<{ success: boolean; data?: { web?: Array<{ url: string; title?: string; description?: string; markdown?: string }> } | Array<{ url: string; title?: string; description?: string; markdown?: string }>; creditsUsed?: number }>('/v2/search', body, 60_000);99    const data = r.data;100    const items = Array.isArray(data) ? data : (data?.web ?? []);101    return { items, credits: r.creditsUsed ?? items.length };102  }103104  async function map(url: string, o: { search?: string; limit?: number } = {}) {105    const r = await post<{ success: boolean; links?: Array<string | { url: string }>; creditsUsed?: number }>('/v2/map', { url, limit: o.limit ?? 5000, ...(o.search ? { search: o.search } : {}) }, 60_000);106    const links = (r.links ?? []).map((l) => (typeof l === 'string' ? l : l.url));107    return { links, credits: r.creditsUsed ?? 1 };108  }109110  return { scrape, search, map };111}112113export type FirecrawlEngine = ReturnType<typeof createFirecrawlEngine>;114