spb/rareindex

1/**2 * Minimal Scrapfly render helper that exposes `browser_data.xhr_call` (the XHR requests/responses the3 * rendered page performed). The shared engine in @rareindex/connectors only returns the page content;4 * Goldin's lot data lives in the page's own search XHR, so this connector needs the browser data.5 * TODO(framework): add `FetchOptions.captureXhr` + `ExtractionResult.browserData` and drop this file.6 */7export interface XhrCall {8  url: string;9  method: string;10  requestBody: string | null;11  responseBody: string | null;12  status: number | null;13}1415export interface ScrapflyRenderResult {16  ok: boolean;17  status: number | null;18  html: string | null;19  xhr: XhrCall[];20  cost: number;21  durationMs: number;22  error: string | null;23}2425export interface RenderOptions {26  apiKey: string;27  renderingWaitMs?: number;28  country?: string;29  timeoutMs?: number;30}3132export async function scrapflyRender(url: string, o: RenderOptions): Promise<ScrapflyRenderResult> {33  const started = Date.now();34  const params = new URLSearchParams({ key: o.apiKey, url, asp: 'true', render_js: 'true', country: (o.country ?? 'us').toLowerCase(), retry: 'true', rendering_wait: String(Math.min(o.renderingWaitMs ?? 6000, 25_000)) });35  const ctrl = new AbortController();36  const timer = setTimeout(() => ctrl.abort(), (o.timeoutMs ?? 170_000) + 5_000);37  try {38    const res = await fetch(`https://api.scrapfly.io/scrape?${params.toString()}`, { signal: ctrl.signal });39    const text = await res.text();40    let json: any;41    try {42      json = JSON.parse(text);43    } catch {44      return { ok: false, status: null, html: null, xhr: [], cost: 0, durationMs: Date.now() - started, error: `scrapfly HTTP ${res.status}: ${text.slice(0, 160)}` };45    }46    const r = json.result ?? {};47    const cost: number = json.context?.cost?.total ?? 0;48    if (!res.ok) return { ok: false, status: null, html: null, xhr: [], cost, durationMs: Date.now() - started, error: `scrapfly HTTP ${res.status}: ${json.message ?? r.error?.message ?? 'error'}` };49    const xhrRaw: any[] = r.browser_data?.xhr_call ?? [];50    const xhr: XhrCall[] = xhrRaw.map((c) => ({51      url: String(c.url ?? ''),52      method: String(c.method ?? 'GET'),53      requestBody: typeof c.body === 'string' ? c.body : c.body ? JSON.stringify(c.body) : null,54      responseBody: typeof c.response?.body === 'string' ? c.response.body : c.response?.body ? JSON.stringify(c.response.body) : null,55      status: c.response?.status ?? null,56    }));57    const status: number | null = r.status_code ?? null;58    const ok = r.success !== false && (status === null || status < 400);59    return { ok, status, html: r.content ?? null, xhr, cost, durationMs: Date.now() - started, error: ok ? null : (r.error?.message ?? `status ${status}`) };60  } catch (err) {61    return { ok: false, status: null, html: null, xhr: [], cost: 0, durationMs: Date.now() - started, error: err instanceof Error ? err.message : String(err) };62  } finally {63    clearTimeout(timer);64  }65}66