TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import "server-only";23/**4 * Server-side client for the Fetcha API service (Fastify). Used by server actions and route5 * handlers only; authenticated with the shared internal service token.6 */7const API_URL = (process.env.API_URL ?? "http://127.0.0.1:8221").replace(/\/$/, "");8const TOKEN = process.env.INTERNAL_SERVICE_TOKEN ?? "";910export interface ApiErrorShape {11 error: { code: string; message: string; request_id: string | null; details?: Record<string, unknown> };12}1314export class InternalApiError extends Error {15 constructor(16 readonly status: number,17 readonly code: string,18 message: string,19 readonly requestId: string | null,20 readonly details?: Record<string, unknown>,21 ) {22 super(message);23 this.name = "InternalApiError";24 }25}2627async function call<T>(path: string, init: RequestInit & { timeoutMs?: number } = {}): Promise<T> {28 const ac = new AbortController();29 const t = setTimeout(() => ac.abort(), init.timeoutMs ?? 150_000);30 try {31 const res = await fetch(`${API_URL}${path}`, {32 ...init,33 headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json", ...(init.headers ?? {}) },34 signal: ac.signal,35 cache: "no-store",36 });37 const text = await res.text();38 let data: unknown = null;39 try {40 data = text ? JSON.parse(text) : null;41 } catch {42 data = null;43 }44 if (!res.ok) {45 const err = (data as ApiErrorShape | null)?.error;46 throw new InternalApiError(res.status, err?.code ?? "INTERNAL_ERROR", err?.message ?? `API error ${res.status}`, err?.request_id ?? res.headers.get("x-fetcha-request-id"), err?.details);47 }48 return data as T;49 } catch (e) {50 if (e instanceof InternalApiError) throw e;51 if ((e as Error).name === "AbortError") throw new InternalApiError(504, "TARGET_TIMEOUT", "The request timed out.", null);52 throw new InternalApiError(503, "PROVIDER_UNAVAILABLE", "The Fetcha API service is unreachable.", null);53 } finally {54 clearTimeout(t);55 }56}5758// ---------------------------------------------------------------------------59// Crawl & map (section 3 of docs/API-v0.2.md). Provider details are never included.60// ---------------------------------------------------------------------------6162export type CrawlJobStatus = "queued" | "running" | "completed" | "failed" | "cancelled";6364export interface CrawlJobStats {65 discovered: number;66 fetched: number;67 ok: number;68 blocked: number;69 failed: number;70 bytes: number;71}7273export interface CrawlJob {74 id: string;75 status: CrawlJobStatus;76 label: string | null;77 seed_url: string;78 domain: string;79 options: Record<string, unknown>;80 stats: CrawlJobStats;81 error: { code: string; message: string } | null;82 created_at: string;83 started_at: string | null;84 completed_at: string | null;85}8687export interface CrawlPage {88 id: string;89 url: string;90 final_url: string | null;91 depth: number;92 status: string;93 http_status: number | null;94 error_code: string | null;95 title: string | null;96 description: string | null;97 content_type: string | null;98 content: string | null;99 links_count: number | null;100 bytes: number | null;101 duration_ms: number | null;102 mode: "http" | "browser" | null;103 fetched_at: string | null;104}105106export interface CrawlPagesPage {107 data: CrawlPage[];108 next_cursor: string | null;109}110111export interface MapResult {112 url: string;113 count: number;114 urls: string[];115 sources: { sitemap: number; links: number };116 truncated: boolean;117}118119export interface BrowserStatus {120 enabled: boolean;121 running: number;122 capacity: number;123 queue: number;124}125126function qs(params: Record<string, string | number | undefined | null>): string {127 const sp = new URLSearchParams();128 for (const [k, v] of Object.entries(params)) if (v !== undefined && v !== null && v !== "") sp.set(k, String(v));129 const s = sp.toString();130 return s ? `?${s}` : "";131}132133export const internalApi = {134 health: () => call<{ status: string; version: string }>("/health", { timeoutMs: 5000 }),135 ready: () => call<{ status: string; checks: Record<string, boolean>; available_networks: string[] }>("/ready", { timeoutMs: 8000 }).catch((e) => (e instanceof InternalApiError && e.status === 503 ? { status: "degraded", checks: {}, available_networks: [] } : Promise.reject(e))),136 playgroundFetch: (projectId: string, userId: string, request: unknown) =>137 call<unknown>("/internal/fetch", { method: "POST", body: JSON.stringify({ project_id: projectId, user_id: userId, request }) }),138 createSession: (projectId: string, userId: string, options: unknown) =>139 call<unknown>("/internal/sessions", { method: "POST", body: JSON.stringify({ project_id: projectId, user_id: userId, options }) }),140 listSessions: (projectId: string, userId: string) => call<{ data: unknown[] }>(`/internal/sessions?project_id=${projectId}&user_id=${userId}`),141 closeSession: (projectId: string, userId: string, id: string) => call<unknown>(`/internal/sessions/${id}?project_id=${projectId}&user_id=${userId}`, { method: "DELETE" }),142 usage: (projectId: string, userId: string) => call<Record<string, unknown>>(`/internal/usage?project_id=${projectId}&user_id=${userId}`),143 providers: () =>144 call<{145 providers: Array<{146 id: string;147 label: string;148 configured: boolean;149 networks: string[];150 prices: Record<string, number>;151 health: Array<{ network: string; status: string; latency_ms: number | null; detail: string | null; checked_at: string }>;152 circuits: Array<{ key: string; state: string; failures: number; successes: number; openedAt: number | null }>;153 }>;154 available_networks: string[];155 }>("/internal/providers", { timeoutMs: 10_000 }),156 probeProviders: () => call<{ ok: boolean }>("/internal/providers/probe", { method: "POST", timeoutMs: 60_000 }),157 reloadProviders: () => call<{ ok: boolean }>("/internal/providers/reload", { method: "POST" }),158 resetCircuit: (key?: string) => call<{ ok: boolean }>("/internal/circuits/reset", { method: "POST", body: JSON.stringify({ key }) }),159 invalidateKey: (keyHash: string) => call<{ ok: boolean }>("/internal/keys/invalidate", { method: "POST", body: JSON.stringify({ key_hash: keyHash }) }).catch(() => ({ ok: false })),160161 // Crawl & map162 createCrawl: (projectId: string, userId: string, options: unknown) =>163 call<CrawlJob>("/internal/crawls", { method: "POST", body: JSON.stringify({ project_id: projectId, user_id: userId, options }), timeoutMs: 30_000 }),164 listCrawls: (projectId: string, userId: string, limit = 50) => call<{ data: CrawlJob[] }>(`/internal/crawls${qs({ project_id: projectId, user_id: userId, limit })}`, { timeoutMs: 15_000 }),165 getCrawl: (projectId: string, userId: string, id: string) => call<CrawlJob>(`/internal/crawls/${encodeURIComponent(id)}${qs({ project_id: projectId, user_id: userId })}`, { timeoutMs: 15_000 }),166 crawlPages: (projectId: string, userId: string, id: string, opts: { cursor?: string | null; limit?: number; status?: string | null } = {}) =>167 call<CrawlPagesPage>(`/internal/crawls/${encodeURIComponent(id)}/pages${qs({ project_id: projectId, user_id: userId, cursor: opts.cursor, limit: opts.limit, status: opts.status })}`, { timeoutMs: 20_000 }),168 cancelCrawl: (projectId: string, userId: string, id: string) =>169 call<{ id: string; status: "cancelled" }>(`/internal/crawls/${encodeURIComponent(id)}${qs({ project_id: projectId, user_id: userId })}`, { method: "DELETE", timeoutMs: 15_000 }),170 mapSite: (projectId: string, userId: string, options: unknown) =>171 call<MapResult>("/internal/map", { method: "POST", body: JSON.stringify({ project_id: projectId, user_id: userId, options }), timeoutMs: 90_000 }),172 browserStatus: () => call<BrowserStatus>("/internal/browser", { timeoutMs: 5000 }).catch(() => ({ enabled: false, running: 0, capacity: 0, queue: 0 }) as BrowserStatus),173};174