spb/search-box Public

1/**2 * Search-box.ai3 * Author: Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * File: packages/firecrawl/src/index.ts6 * Description: Firecrawl v2 REST adapter (search + scrape) with retries and SSRF guard.7 */89import { sanitizeText } from "@search-box/shared";10import { assertSafeUrl } from "./url-guard.js";1112const BASE = "https://api.firecrawl.dev/v2";1314export interface SearchResultItem {15  url: string;16  title: string | null;17  description: string | null;18}1920export interface ScrapeResult {21  url: string;22  title: string | null;23  markdown: string;24  statusCode: number | null;25}2627export class FirecrawlError extends Error {28  constructor(29    message: string,30    public readonly status: number | null = null,31    public readonly retryable: boolean = false32  ) {33    super(message);34    this.name = "FirecrawlError";35  }36}3738function apiKey(): string {39  const key = process.env.FIRECRAWL_API_KEY;40  if (!key) throw new Error("FIRECRAWL_API_KEY is not set");41  return key;42}4344async function post<T>(path: string, body: unknown, timeoutMs: number): Promise<T> {45  const maxAttempts = 3;46  let lastErr: unknown = null;47  for (let attempt = 1; attempt <= maxAttempts; attempt++) {48    const controller = new AbortController();49    const timer = setTimeout(() => controller.abort(), timeoutMs);50    try {51      const res = await fetch(`${BASE}${path}`, {52        method: "POST",53        headers: {54          "Content-Type": "application/json",55          Authorization: `Bearer ${apiKey()}`56        },57        body: JSON.stringify(body),58        signal: controller.signal59      });60      if (res.status === 429 || res.status >= 500) {61        lastErr = new FirecrawlError(`firecrawl ${path} → ${res.status}`, res.status, true);62        await res.text().catch(() => "");63        await sleep(500 * attempt * attempt);64        continue;65      }66      if (!res.ok) {67        const text = await res.text().catch(() => "");68        throw new FirecrawlError(`firecrawl ${path} → ${res.status}: ${text.slice(0, 300)}`, res.status);69      }70      return (await res.json()) as T;71    } catch (err) {72      if (err instanceof FirecrawlError && !err.retryable) throw err;73      lastErr = err;74      if (attempt < maxAttempts) await sleep(500 * attempt * attempt);75    } finally {76      clearTimeout(timer);77    }78  }79  throw lastErr instanceof Error ? lastErr : new FirecrawlError(String(lastErr));80}8182function sleep(ms: number): Promise<void> {83  return new Promise((r) => setTimeout(r, ms));84}8586/** Web search. `tbs` filters by recency (e.g. "qdr:w" past week, "qdr:y" past year). */87export async function search(88  query: string,89  opts: { limit?: number; tbs?: string } = {}90): Promise<SearchResultItem[]> {91  interface Resp {92    success: boolean;93    data?: { web?: Array<{ url: string; title?: string; description?: string }> };94  }95  const resp = await post<Resp>(96    "/search",97    {98      query: query.slice(0, 500),99      limit: Math.min(opts.limit ?? 8, 20),100      sources: [{ type: "web" }],101      ...(opts.tbs ? { tbs: opts.tbs } : {})102    },103    45_000104  );105  const web = resp.data?.web ?? [];106  return web107    .filter((r) => typeof r.url === "string" && r.url.length > 0)108    .map((r) => ({109      url: r.url,110      title: r.title ? sanitizeText(r.title) : null,111      description: r.description ? sanitizeText(r.description) : null112    }));113}114115/** Scrape a single URL to markdown. */116export async function scrape(rawUrl: string): Promise<ScrapeResult> {117  const url = assertSafeUrl(rawUrl);118  interface Resp {119    success: boolean;120    data?: {121      markdown?: string;122      metadata?: { title?: string; sourceURL?: string; statusCode?: number };123    };124  }125  const resp = await post<Resp>(126    "/scrape",127    { url, formats: ["markdown"], onlyMainContent: true, timeout: 60_000, maxAge: 172_800_000 },128    90_000129  );130  const data = resp.data;131  if (!resp.success || !data?.markdown) {132    throw new FirecrawlError(`scrape returned no content for ${url}`);133  }134  return {135    url: data.metadata?.sourceURL ?? url,136    title: data.metadata?.title ? sanitizeText(data.metadata.title) : null,137    markdown: sanitizeText(data.markdown),138    statusCode: data.metadata?.statusCode ?? null139  };140}141142export { assertSafeUrl };143