spb/worthdoing Public

1/**2 * WorthDoing.ai3 * Author: Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * File: src/lib/firecrawl/client.ts6 * Description: Firecrawl REST adapter — search, scrape, crawl, extract with retries and structured errors.7 */8import { env } from "@/lib/env";910const BASE = "https://api.firecrawl.dev";11const MAX_RETRIES = 3;1213export class FirecrawlError extends Error {14  constructor(15    message: string,16    public readonly status: number | null,17    public readonly retryable: boolean,18    public readonly endpoint: string,19  ) {20    super(message);21    this.name = "FirecrawlError";22  }23}2425export type SearchResult = { url: string; title: string; description: string };26export type ScrapeResult = {27  markdown: string;28  title: string | null;29  description: string | null;30  statusCode: number | null;31  sourceUrl: string;32};33export type CrawlPage = ScrapeResult;3435async function fcFetch<T>(path: string, init: RequestInit, timeoutMs = 90_000): Promise<T> {36  let lastError: FirecrawlError | null = null;37  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {38    if (attempt > 0) {39      await new Promise((r) => setTimeout(r, 1000 * 2 ** (attempt - 1) + Math.random() * 500));40    }41    const controller = new AbortController();42    const timer = setTimeout(() => controller.abort(), timeoutMs);43    try {44      const res = await fetch(`${BASE}${path}`, {45        ...init,46        headers: {47          Authorization: `Bearer ${env().FIRECRAWL_API_KEY}`,48          "Content-Type": "application/json",49          ...init.headers,50        },51        signal: controller.signal,52      });53      if (res.ok) {54        return (await res.json()) as T;55      }56      const body = await res.text().catch(() => "");57      const retryable = res.status === 429 || res.status >= 500;58      lastError = new FirecrawlError(59        `Firecrawl ${path} failed (${res.status}): ${body.slice(0, 300)}`,60        res.status,61        retryable,62        path,63      );64      if (!retryable) throw lastError;65    } catch (err) {66      if (err instanceof FirecrawlError) {67        if (!err.retryable) throw err;68        lastError = err;69      } else {70        // Network error / timeout — retryable71        lastError = new FirecrawlError(72          `Firecrawl ${path} network error: ${err instanceof Error ? err.message : String(err)}`,73          null,74          true,75          path,76        );77      }78    } finally {79      clearTimeout(timer);80    }81  }82  throw lastError ?? new FirecrawlError(`Firecrawl ${path} failed`, null, false, path);83}8485/** Web search. Returns organic web results (no scraping). */86export async function searchWeb(query: string, limit = 8): Promise<SearchResult[]> {87  type Resp = {88    success: boolean;89    data?: { web?: Array<{ url: string; title?: string; description?: string }> } | Array<{90      url: string;91      title?: string;92      description?: string;93    }>;94  };95  const resp = await fcFetch<Resp>("/v2/search", {96    method: "POST",97    body: JSON.stringify({ query, limit, sources: [{ type: "web" }] }),98  });99  const raw = Array.isArray(resp.data) ? resp.data : (resp.data?.web ?? []);100  return raw101    .filter((r) => typeof r.url === "string" && r.url.startsWith("http"))102    .map((r) => ({ url: r.url, title: r.title ?? "", description: r.description ?? "" }));103}104105/** Scrape a single page to markdown (main content only). */106export async function scrapePage(url: string): Promise<ScrapeResult> {107  type Resp = {108    success: boolean;109    data?: {110      markdown?: string;111      metadata?: { title?: string; description?: string; statusCode?: number; sourceURL?: string };112    };113  };114  const resp = await fcFetch<Resp>("/v2/scrape", {115    method: "POST",116    body: JSON.stringify({ url, formats: ["markdown"], onlyMainContent: true, timeout: 60_000 }),117  });118  const d = resp.data;119  if (!d?.markdown) {120    throw new FirecrawlError(`Scrape returned no content for ${url}`, null, false, "/v2/scrape");121  }122  return {123    markdown: d.markdown,124    title: d.metadata?.title ?? null,125    description: d.metadata?.description ?? null,126    statusCode: d.metadata?.statusCode ?? null,127    sourceUrl: d.metadata?.sourceURL ?? url,128  };129}130131/** Selective crawl: start a bounded crawl and poll until done (or timeout). Never whole domains. */132export async function crawlSite(url: string, limit = 8, maxWaitMs = 180_000): Promise<CrawlPage[]> {133  type StartResp = { success: boolean; id?: string };134  const start = await fcFetch<StartResp>("/v2/crawl", {135    method: "POST",136    body: JSON.stringify({137      url,138      limit: Math.min(limit, 10),139      scrapeOptions: { formats: ["markdown"], onlyMainContent: true },140    }),141  });142  if (!start.id) throw new FirecrawlError("Crawl did not return a job id", null, false, "/v2/crawl");143144  type StatusResp = {145    status: "scraping" | "completed" | "failed" | "cancelled";146    data?: Array<{147      markdown?: string;148      metadata?: { title?: string; description?: string; statusCode?: number; sourceURL?: string };149    }>;150  };151  const deadline = Date.now() + maxWaitMs;152  for (;;) {153    await new Promise((r) => setTimeout(r, 5000));154    const status = await fcFetch<StatusResp>(`/v2/crawl/${start.id}`, { method: "GET" });155    if (status.status === "completed") {156      return (status.data ?? [])157        .filter((p) => p.markdown)158        .map((p) => ({159          markdown: p.markdown as string,160          title: p.metadata?.title ?? null,161          description: p.metadata?.description ?? null,162          statusCode: p.metadata?.statusCode ?? null,163          sourceUrl: p.metadata?.sourceURL ?? url,164        }));165    }166    if (status.status === "failed" || status.status === "cancelled") {167      throw new FirecrawlError(`Crawl ${status.status} for ${url}`, null, false, "/v2/crawl");168    }169    if (Date.now() > deadline) {170      throw new FirecrawlError(`Crawl timed out after ${maxWaitMs}ms for ${url}`, null, false, "/v2/crawl");171    }172  }173}174175/** Structured extraction from one or more URLs against a JSON schema. */176export async function extractStructured(177  urls: string[],178  prompt: string,179  schema: Record<string, unknown>,180  maxWaitMs = 120_000,181): Promise<unknown> {182  type StartResp = { success: boolean; id?: string; data?: unknown };183  const start = await fcFetch<StartResp>("/v2/extract", {184    method: "POST",185    body: JSON.stringify({ urls: urls.slice(0, 5), prompt, schema }),186  });187  if (start.data && !start.id) return start.data;188  if (!start.id) throw new FirecrawlError("Extract did not return a job id", null, false, "/v2/extract");189190  type StatusResp = { status: "processing" | "completed" | "failed" | "cancelled"; data?: unknown };191  const deadline = Date.now() + maxWaitMs;192  for (;;) {193    await new Promise((r) => setTimeout(r, 4000));194    const status = await fcFetch<StatusResp>(`/v2/extract/${start.id}`, { method: "GET" });195    if (status.status === "completed") return status.data;196    if (status.status === "failed" || status.status === "cancelled") {197      throw new FirecrawlError(`Extract ${status.status}`, null, false, "/v2/extract");198    }199    if (Date.now() > deadline) {200      throw new FirecrawlError(`Extract timed out after ${maxWaitMs}ms`, null, false, "/v2/extract");201    }202  }203}204