spb/rareindex

1import type { ExtractionResult } from '@rareindex/shared';2import type { FetchOptions } from '../types.js';34export interface ScrapflyEngineOptions {5  apiKey: string;6  baseUrl?: string;7  defaultCountry?: string;8  defaultTimeoutMs?: number;9}1011interface ScrapflyResponse {12  result?: {13    content?: string;14    status_code?: number;15    url?: string;16    success?: boolean;17    error?: { message?: string; code?: string } | null;18    format?: string;19    extracted_data?: { data?: unknown } | unknown;20  };21  context?: { cost?: { total?: number } };22  config?: Record<string, unknown>;23  cost?: number;24  message?: string;25}2627/**28 * Scrapfly REST client (§104): secondary engine for JS-heavy or difficult public pages, with29 * geo routing and rendering. `asp` (anti-scraping protection) is only used on publicly accessible30 * pages within the source's terms — never to defeat logins, paywalls or CAPTCHAs (§179).31 */32export function createScrapflyEngine(opts: ScrapflyEngineOptions) {33  const base = (opts.baseUrl ?? 'https://api.scrapfly.io').replace(/\/$/, '');3435  async function scrape(url: string, o: FetchOptions = {}): Promise<ExtractionResult> {36    const started = Date.now();37    const timeoutMs = o.timeoutMs ?? opts.defaultTimeoutMs ?? 90_000;38    const params = new URLSearchParams({39      key: opts.apiKey,40      url,41      asp: 'true',42      render_js: o.renderJs === false ? 'false' : 'true',43      country: (o.country ?? opts.defaultCountry ?? 'us').toLowerCase(),44      retry: 'true',45      format: 'raw',46    });47    if (o.waitForMs) params.set('rendering_wait', String(Math.min(o.waitForMs, 25_000)));48    if (o.extractionModel) params.set('extraction_model', o.extractionModel);49    if (o.headers) for (const [k, v] of Object.entries(o.headers)) params.append(`headers[${k}]`, v);50    if (o.method === 'POST') params.set('method', 'POST');51    const ctrl = new AbortController();52    const timer = setTimeout(() => ctrl.abort(), timeoutMs + 5_000);53    try {54      const res = await fetch(`${base}/scrape?${params.toString()}`, {55        method: o.method === 'POST' ? 'POST' : 'GET',56        body: o.method === 'POST' && o.body !== undefined ? JSON.stringify(o.body) : undefined,57        headers: o.method === 'POST' ? { 'content-type': 'application/json' } : undefined,58        signal: ctrl.signal,59      });60      const text = await res.text();61      let json: ScrapflyResponse | null = null;62      try {63        json = JSON.parse(text) as ScrapflyResponse;64      } catch {65        return fail(`scrapfly HTTP ${res.status}: ${text.slice(0, 200)}`, started, url);66      }67      const r = json.result;68      const status = r?.status_code ?? null;69      const cost = json.context?.cost?.total ?? json.cost ?? 1;70      if (!res.ok || !r) return fail(`scrapfly HTTP ${res.status}: ${json.message ?? r?.error?.message ?? 'no result'}`, started, url, cost);71      const ok = Boolean(r.success !== false && (status === null || status < 400));72      const content = r.content ?? null;73      const extracted = (r.extracted_data as { data?: unknown } | undefined)?.data ?? r.extracted_data ?? null;74      const looksJson = content && /^\s*[[{]/.test(content);75      let parsed: unknown = extracted;76      if (!parsed && looksJson) {77        try {78          parsed = JSON.parse(content!);79        } catch {80          parsed = null;81        }82      }83      return {84        success: ok,85        engine: 'scrapfly',86        url,87        finalUrl: r.url ?? null,88        httpStatus: status,89        html: content,90        markdown: null,91        json: parsed,92        qualityScore: 0,93        requiresReview: false,94        error: ok ? null : (r.error?.message ?? `status ${status}`),95        costCredits: cost,96        durationMs: Date.now() - started,97        fetchedAt: new Date(),98      };99    } catch (err) {100      return fail(err instanceof Error ? err.message : String(err), started, url);101    } finally {102      clearTimeout(timer);103    }104  }105106  function fail(error: string, started: number, url: string, cost = 0): ExtractionResult {107    return { success: false, engine: 'scrapfly', url, finalUrl: null, httpStatus: null, html: null, markdown: null, json: null, qualityScore: 0, requiresReview: false, error, costCredits: cost, durationMs: Date.now() - started, fetchedAt: new Date() };108  }109110  return { scrape };111}112113export type ScrapflyEngine = ReturnType<typeof createScrapflyEngine>;114