spb/vrai-prix

1// Auteur : Simon-Pierre Boucher — contact@spboucher.ai2/**3 * Client Firecrawl v2 (scrape / search / map) : timeouts, réessais, rythme par4 * domaine, hachage du contenu. Clé côté serveur uniquement (FIRECRAWL_API_KEY).5 * Aucun appel n'est fait depuis le chemin utilisateur.6 */7import { createHash } from "crypto";89const API = "https://api.firecrawl.dev/v2";1011export interface ScrapeResult {12  markdown: string;13  metadata: Record<string, unknown>;14  statusCode: number | null;15  links: string[];16}1718export interface SearchHit {19  url: string;20  title: string;21  description: string;22}2324export class FirecrawlError extends Error {25  constructor(message: string, public readonly status: number | null, public readonly url: string) {26    super(message);27    this.name = "FirecrawlError";28  }29}3031/* --------------------------------------------------------- rythme par domaine */3233const lastCall = new Map<string, number>();34const inflight = new Map<string, number>();3536function domainOf(url: string): string {37  try { return new URL(url).hostname.replace(/^www\./, ""); } catch { return "unknown"; }38}3940const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));4142/** Attend le créneau libre pour un domaine (rythme minimal + concurrence). */43export async function throttle(domain: string, minGapMs: number, concurrency: number): Promise<() => void> {44  for (;;) {45    const now = Date.now();46    const last = lastCall.get(domain) ?? 0;47    const busy = inflight.get(domain) ?? 0;48    if (busy < concurrency && now - last >= minGapMs) {49      lastCall.set(domain, now);50      inflight.set(domain, busy + 1);51      return () => inflight.set(domain, Math.max(0, (inflight.get(domain) ?? 1) - 1));52    }53    await sleep(Math.max(50, minGapMs - (now - last)));54  }55}5657/* ------------------------------------------------------------------ client */5859export function firecrawlKey(): string {60  const k = process.env.FIRECRAWL_API_KEY;61  if (!k) throw new FirecrawlError("FIRECRAWL_API_KEY manquante (serveur)", null, API);62  return k;63}6465async function post<T>(path: string, body: unknown, timeoutMs: number, retries: number): Promise<T> {66  let lastErr: unknown = null;67  for (let attempt = 0; attempt <= retries; attempt++) {68    const ctrl = new AbortController();69    const t = setTimeout(() => ctrl.abort(), timeoutMs);70    try {71      const res = await fetch(`${API}${path}`, { method: "POST", headers: { Authorization: `Bearer ${firecrawlKey()}`, "Content-Type": "application/json" }, body: JSON.stringify(body), signal: ctrl.signal });72      const json = (await res.json().catch(() => ({}))) as { success?: boolean; error?: string; data?: T };73      if (res.status === 429 || res.status >= 500) throw new FirecrawlError(`HTTP ${res.status} ${json.error ?? ""}`.trim(), res.status, path);74      if (!res.ok || json.success === false) throw new FirecrawlError(json.error ?? `HTTP ${res.status}`, res.status, path);75      return json.data as T;76    } catch (e) {77      lastErr = e;78      const status = e instanceof FirecrawlError ? e.status : null;79      const retriable = status == null || status === 429 || status >= 500;80      if (!retriable || attempt === retries) break;81      await sleep(1500 * 2 ** attempt);82    } finally {83      clearTimeout(t);84    }85  }86  if (lastErr instanceof Error && lastErr.name === "AbortError") throw new FirecrawlError("délai dépassé", null, path);87  throw lastErr instanceof Error ? lastErr : new FirecrawlError(String(lastErr), null, path);88}8990export interface ScrapeOptions { timeoutMs?: number; retries?: number; rateLimitMs?: number; concurrency?: number; links?: boolean }9192export async function scrape(url: string, o: ScrapeOptions = {}): Promise<ScrapeResult> {93  const release = await throttle(domainOf(url), o.rateLimitMs ?? 1000, o.concurrency ?? 2);94  try {95    const formats = o.links ? ["markdown", "links"] : ["markdown"];96    const data = await post<{ markdown?: string; metadata?: Record<string, unknown>; links?: string[] }>("/scrape", { url, formats, onlyMainContent: true }, o.timeoutMs ?? 90_000, o.retries ?? 2);97    const md = data?.metadata ?? {};98    const sc = typeof md.statusCode === "number" ? md.statusCode : null;99    return { markdown: data?.markdown ?? "", metadata: md, statusCode: sc, links: data?.links ?? [] };100  } finally {101    release();102  }103}104105export async function search(query: string, limit = 5, o: ScrapeOptions = {}): Promise<SearchHit[]> {106  const release = await throttle("api.firecrawl.dev/search", o.rateLimitMs ?? 1000, o.concurrency ?? 2);107  try {108    const data = await post<{ web?: { url: string; title?: string; description?: string }[] }>("/search", { query, limit }, o.timeoutMs ?? 60_000, o.retries ?? 2);109    return (data?.web ?? []).map((h) => ({ url: h.url, title: h.title ?? "", description: h.description ?? "" }));110  } finally {111    release();112  }113}114115export async function mapSite(url: string, searchTerm?: string, limit = 50, o: ScrapeOptions = {}): Promise<string[]> {116  const release = await throttle(domainOf(url), o.rateLimitMs ?? 1000, o.concurrency ?? 1);117  try {118    const data = await post<{ links?: { url: string }[] | string[] }>("/map", { url, search: searchTerm, limit }, o.timeoutMs ?? 60_000, o.retries ?? 1);119    return (data?.links ?? []).map((l) => (typeof l === "string" ? l : l.url));120  } finally {121    release();122  }123}124125/** Hachage stable du contenu utile (markdown normalisé) pour la détection de changement. */126export function contentHash(markdown: string, extra: Record<string, unknown> = {}): string {127  const norm = markdown.replace(/\s+/g, " ").trim();128  return createHash("sha256").update(norm).update(JSON.stringify(extra)).digest("hex");129}130131/** Parse « 5,98 $ », « $6.98 », « 1 234,50 » → nombre ; null si impossible. */132export function parseMoney(s: unknown): number | null {133  if (typeof s === "number") return Number.isFinite(s) ? s : null;134  if (typeof s !== "string") return null;135  const m = s.replace(/ /g, " ").match(/-?\d[\d\s]*(?:[.,]\d+)?/);136  if (!m) return null;137  const v = Number(m[0].replace(/\s/g, "").replace(",", "."));138  return Number.isFinite(v) ? v : null;139}140