spb/websensor

1import type { Observation } from "@websensor/core";2import { assertUrlAllowed } from "@websensor/core";34/**5 * Scrapfly fallback (acquisition hierarchy step 14). Used only when ordinary HTTP failed with6 * an anti-bot response (403/429/503) AND the source registry allows it (`fallback.scrapfly`).7 * Budgeted per day; never used for the baseline crawl of ordinary pages.8 * Docs: https://scrapfly.io/docs/scrape-api/getting-started — GET /scrape?key&url&asp&render_js9 * Response: { result: { content, status_code, success, response_headers, url }, context: { cost } }10 */11let dayKey = "";12let used = 0;1314export function scrapflyBudget(): { used: number; limit: number; day: string } {15  roll();16  return { used, limit: Number(process.env.WS_SCRAPFLY_DAILY_BUDGET ?? 300), day: dayKey };17}1819function roll(): void {20  const today = new Date().toISOString().slice(0, 10);21  if (dayKey !== today) {22    dayKey = today;23    used = 0;24  }25}2627export function scrapflyAvailable(): boolean {28  roll();29  return Boolean(process.env.SCRAPFLY_API_KEY) && used < Number(process.env.WS_SCRAPFLY_DAILY_BUDGET ?? 300);30}3132export async function scrapflyFetch(sensorId: string, url: string, opts: { renderJs?: boolean; country?: string } = {}): Promise<Observation> {33  const started = Date.now();34  const key = process.env.SCRAPFLY_API_KEY;35  const fail = (code: string, message: string): Observation => ({ sensorId, url, fetchedAt: new Date(), notModified: false, error: { code, message }, meta: { status: 0, url, finalUrl: url, contentType: null, contentLength: 0, etag: null, lastModified: null, durationMs: Date.now() - started, redirects: 0, method: "API", headers: {} } });36  if (!key) return fail("scrapfly_unavailable", "SCRAPFLY_API_KEY not configured");37  if (!scrapflyAvailable()) return fail("scrapfly_budget", "daily Scrapfly budget exhausted");38  try {39    await assertUrlAllowed(url);40  } catch (e) {41    return fail("ssrf_blocked", (e as Error).message);42  }43  used++;44  const q = new URLSearchParams({ key, url, asp: "true", render_js: opts.renderJs ? "true" : "false", country: opts.country ?? "us", retry: "true" });45  const ac = new AbortController();46  const timer = setTimeout(() => ac.abort(), 90_000);47  try {48    const res = await fetch(`https://api.scrapfly.io/scrape?${q.toString()}`, { signal: ac.signal, headers: { accept: "application/json" } });49    const json = (await res.json()) as { result?: { content?: string; status_code?: number; success?: boolean; reason?: string; response_headers?: Record<string, string>; url?: string; error?: { message?: string } }; message?: string; code?: string };50    const r = json.result;51    if (!res.ok || !r) return fail("scrapfly_error", `${res.status} ${json.message ?? json.code ?? "no result"}`);52    if (!r.success) return fail("scrapfly_failed", `${r.status_code ?? 0} ${r.reason ?? r.error?.message ?? "upstream failed"}`);53    const body = Buffer.from(r.content ?? "", "utf8");54    const h = Object.fromEntries(Object.entries(r.response_headers ?? {}).map(([k, v]) => [k.toLowerCase(), String(v)]));55    return {56      sensorId,57      url,58      fetchedAt: new Date(),59      notModified: false,60      body,61      meta: { status: r.status_code ?? 200, url, finalUrl: r.url ?? url, contentType: h["content-type"] ?? null, contentLength: body.length, etag: null, lastModified: h["last-modified"] ?? null, durationMs: Date.now() - started, redirects: 0, method: "API", headers: { ...h, "x-websensor-via": "scrapfly" } },62    };63  } catch (e) {64    return fail(ac.signal.aborted ? "timeout" : "scrapfly_error", (e as Error).message);65  } finally {66    clearTimeout(timer);67  }68}69