/** * @fetcha/sdk — Official JavaScript / TypeScript client for Fetcha. * * ```ts * import { Fetcha } from "@fetcha/sdk"; * const fetcha = new Fetcha({ apiKey: process.env.FETCHA_API_KEY! }); * const result = await fetcha.fetch({ url: "https://example.com", country: "CA", format: "markdown" }); * console.log(result.status, result.metadata.mode, result.markdown?.slice(0, 200)); * * const job = await fetcha.crawl.create({ url: "https://docs.example.com/", max_pages: 100 }); * const done = await fetcha.crawl.wait(job.id); * const pages = await fetcha.crawl.pages(job.id, { limit: 100 }); * ``` * * Zero dependencies; works on Node 18+, Deno, Bun and modern browsers (global `fetch`). */ export const SDK_VERSION = "0.2.0"; export type FetchaNetwork = "auto" | "datacenter" | "residential" | "isp" | "mobile"; export type FetchaFormat = "html" | "text" | "markdown" | "json" | "raw"; export type FetchaMode = "http" | "browser"; export type FetchaWaitUntil = "load" | "domcontentloaded" | "networkidle"; export interface FetchOptions { url: string; method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS"; headers?: Record; cookies?: Record; body?: string | Record; /** Overall deadline in ms for all attempts, including browser renders (1,000–120,000). */ timeout?: number; country?: string; region?: string; city?: string; network?: FetchaNetwork; session?: string; /** Render in the managed headless browser routed through the same network / geo / session. */ browser?: boolean; /** Escalate to the browser automatically when an HTTP attempt is blocked by a JS challenge (default true). */ browser_fallback?: boolean; /** Browser: CSS selector to wait for before capturing. */ wait_for?: string; /** Browser: extra settle time in ms (0–30,000). */ wait_ms?: number; /** Browser: navigation wait condition (default "domcontentloaded"). */ wait_until?: FetchaWaitUntil; /** Browser: disable scripting when false. */ javascript?: boolean; /** Browser: skip images, fonts and media (default true). */ block_resources?: boolean; /** Browser: return a PNG screenshot (base64) in `screenshot`. */ screenshot?: boolean; /** Browser: solve Cloudflare Turnstile challenges with the platform's captcha solver (default true). */ solve_captcha?: boolean; /** Return `links[]` (all hyperlinks, absolute). */ links?: boolean; /** Referer strategy: "auto" (none first, search-engine referer on retries), "none", or a literal URL. */ referer?: "auto" | "none" | string; device?: "desktop" | "mobile" | "tablet"; locale?: string; format?: FetchaFormat; follow_redirects?: boolean; max_redirects?: number; max_response_bytes?: number; retries?: number; debug?: boolean; } export interface PageMetadata { title: string | null; description: string | null; canonical: string | null; lang: string | null; og: Record; links_count: number; } export interface PageLink { url: string; text: string; internal: boolean; nofollow: boolean; } export interface FetchAttempt { provider: string; network: string; mode: FetchaMode; country: string | null; outcome: string; block_reason?: string | null; status: number | null; duration_ms: number; error?: string; } export interface FetchResult { request_id: string; success: boolean; status: number; url: string; final_url: string; content: string | null; content_type: string | null; headers: Record; cookies: Array<{ name: string; value: string; domain?: string; path?: string }>; /** Present for format "text". */ text?: string | null; /** Present for format "json" when the body parsed. */ json?: unknown; /** Present for format "markdown". */ markdown?: string | null; /** Parsed page metadata (HTML responses). */ page?: PageMetadata | null; /** Present with `links: true`. */ links?: PageLink[]; /** Present with browser rendering and `screenshot: true` (PNG, base64). */ screenshot?: string | null; metadata: { network: string; country: string | null; /** "http" for a plain fetch, "browser" when the final attempt was rendered. */ mode: FetchaMode; attempts: number; duration_ms: number; bytes: number; session: string | null; cached: boolean; timing?: Record; debug?: { attempts: FetchAttempt[] }; }; } export interface SessionOptions { country?: string; region?: string; city?: string; network?: FetchaNetwork; ttl?: number; label?: string; } export interface Session { id: string; status: string; network: string; country: string | null; expires_at: string; created_at: string; } // --------------------------------------------------------------------------- // Crawl & map // --------------------------------------------------------------------------- export type CrawlFormat = "markdown" | "text" | "html"; export type CrawlStatus = "queued" | "running" | "completed" | "failed" | "cancelled"; export interface CrawlOptions { url: string; /** Maximum pages to fetch (1–5,000; the seed counts as one). Default 25. */ max_pages?: number; /** Maximum link depth from the seed (0–10). Default 2. */ max_depth?: number; same_domain?: boolean; allow_subdomains?: boolean; /** Glob with `*` or `/regex/`. */ include_patterns?: string[]; exclude_patterns?: string[]; respect_robots?: boolean; use_sitemap?: boolean; /** Parallel page fetches (1–10). Default 3. */ concurrency?: number; delay_ms?: number; /** Per-page timeout in ms. */ timeout?: number; format?: CrawlFormat; main_content?: boolean; country?: string; network?: FetchaNetwork; browser?: boolean; browser_fallback?: boolean; headers?: Record; webhook_url?: string; label?: string; } export interface CrawlStats { discovered: number; fetched: number; ok: number; blocked: number; failed: number; bytes: number; } export interface CrawlJob { id: string; status: CrawlStatus; label: string | null; seed_url: string; domain: string; options: Record; stats: CrawlStats; error: { code: string; message: string } | null; created_at: string; started_at: string | null; completed_at: string | null; } /** Body of the 202 returned by `crawl.create`. */ export interface CrawlCreated { id: string; status: "queued"; seed_url: string; created_at: string; options: Record; } export interface CrawlPage { id: string; url: string; final_url: string | null; depth: number; status: string; http_status: number | null; error_code: string | null; title: string | null; description: string | null; content_type: string | null; content: string | null; links_count: number | null; bytes: number | null; duration_ms: number | null; mode: FetchaMode | null; fetched_at: string | null; } export interface CrawlPagesResult { data: CrawlPage[]; next_cursor: string | null; } export interface CrawlPagesOptions { cursor?: string | null; limit?: number; status?: "success" | "blocked" | "failed"; } export interface CrawlWaitOptions { /** Interval between polls in ms. Default 2,000. */ pollMs?: number; /** Give up after this many ms. Default 600,000 (10 min). */ timeoutMs?: number; /** Called after each poll with the current job. */ onPoll?: (job: CrawlJob) => void; } export interface MapOptions { url: string; /** Maximum URLs returned (1–10,000). Default 1,000. */ limit?: number; use_sitemap?: boolean; use_links?: boolean; same_domain?: boolean; allow_subdomains?: boolean; /** Substring, glob (`*`) or `/regex/` filter. */ search?: string; country?: string; network?: FetchaNetwork; timeout?: number; } export interface MapResult { url: string; count: number; urls: string[]; sources: { sitemap: number; links: number }; truncated: boolean; } export interface FetchaClientOptions { apiKey: string; baseUrl?: string; /** Default request timeout (ms) applied client-side on top of the API timeout. */ clientTimeout?: number; fetch?: typeof globalThis.fetch; } export class FetchaError extends Error { readonly code: string; readonly status: number; readonly requestId: string | null; readonly details?: Record; constructor(code: string, message: string, status: number, requestId: string | null, details?: Record) { super(message); this.name = "FetchaError"; this.code = code; this.status = status; this.requestId = requestId; this.details = details; } } const TERMINAL: ReadonlySet = new Set(["completed", "failed", "cancelled"]); function query(params: Record): string { const sp = new URLSearchParams(); for (const [k, v] of Object.entries(params)) if (v !== undefined && v !== null && v !== "") sp.set(k, String(v)); const s = sp.toString(); return s ? `?${s}` : ""; } export class Fetcha { private readonly apiKey: string; private readonly baseUrl: string; private readonly clientTimeout: number; private readonly _fetch: typeof globalThis.fetch; constructor(opts: FetchaClientOptions) { if (!opts?.apiKey) throw new Error("Fetcha: apiKey is required"); this.apiKey = opts.apiKey; this.baseUrl = (opts.baseUrl ?? "https://www.fetcha.co").replace(/\/$/, ""); this.clientTimeout = opts.clientTimeout ?? 150_000; this._fetch = opts.fetch ?? globalThis.fetch.bind(globalThis); } private async call(path: string, init: { method: string; body?: unknown; idempotencyKey?: string }): Promise { const ac = new AbortController(); const timer = setTimeout(() => ac.abort(), this.clientTimeout); try { const res = await this._fetch(`${this.baseUrl}${path}`, { method: init.method, headers: { authorization: `Bearer ${this.apiKey}`, "content-type": "application/json", "user-agent": `fetcha-sdk-js/${SDK_VERSION}`, ...(init.idempotencyKey ? { "idempotency-key": init.idempotencyKey } : {}), }, body: init.body === undefined ? undefined : JSON.stringify(init.body), signal: ac.signal, }); const requestId = res.headers.get("x-fetcha-request-id"); const text = await res.text(); let data: unknown = null; try { data = text ? JSON.parse(text) : null; } catch { data = null; } if (!res.ok) { const err = (data as { error?: { code?: string; message?: string; request_id?: string; details?: Record } } | null)?.error; throw new FetchaError(err?.code ?? "HTTP_ERROR", err?.message ?? `HTTP ${res.status}`, res.status, err?.request_id ?? requestId, err?.details); } return data as T; } finally { clearTimeout(timer); } } /** Fetch a URL through Fetcha's routing engine (optionally rendered in the managed browser). */ fetch(options: FetchOptions): Promise { return this.call("/v1/fetch", { method: "POST", body: options }); } /** Convenience: GET a page and return readable text. */ async text(url: string, options: Omit = {}): Promise { const r = await this.fetch({ ...options, url, format: "text" }); return r.text ?? ""; } /** Convenience: GET a page and return it as Markdown. */ async markdown(url: string, options: Omit = {}): Promise { const r = await this.fetch({ ...options, url, format: "markdown" }); return r.markdown ?? ""; } /** Convenience: GET a JSON endpoint. */ async json(url: string, options: Omit = {}): Promise { const r = await this.fetch({ ...options, url, format: "json" }); return r.json as T; } /** Convenience: render a page in the managed browser. */ render(url: string, options: Omit = {}): Promise { return this.fetch({ ...options, url, browser: true }); } /** Discover the URLs of a site (sitemap + links), synchronously. */ map(options: MapOptions): Promise { return this.call("/v1/map", { method: "POST", body: options }); } readonly crawl = { /** Start a crawl job. Returns immediately with status "queued". */ create: (options: CrawlOptions) => this.call("/v1/crawl", { method: "POST", body: options }), /** Get a job with its live status and stats. */ get: (id: string) => this.call(`/v1/crawl/${encodeURIComponent(id)}`, { method: "GET" }), /** List the most recent jobs of the project. */ list: (limit?: number) => this.call<{ data: CrawlJob[] }>(`/v1/crawl${query({ limit })}`, { method: "GET" }), /** Page through the crawled pages with `cursor` / `next_cursor`. */ pages: (id: string, options: CrawlPagesOptions = {}) => this.call(`/v1/crawl/${encodeURIComponent(id)}/pages${query({ cursor: options.cursor, limit: options.limit, status: options.status })}`, { method: "GET" }), /** Cancel a queued or running job. */ cancel: (id: string) => this.call<{ id: string; status: "cancelled" }>(`/v1/crawl/${encodeURIComponent(id)}`, { method: "DELETE" }), /** Poll `get` until the job reaches a terminal status (completed, failed or cancelled). */ wait: async (id: string, options: CrawlWaitOptions = {}): Promise => { const pollMs = Math.max(250, options.pollMs ?? 2000); const timeoutMs = options.timeoutMs ?? 600_000; const deadline = Date.now() + timeoutMs; for (;;) { const job = await this.crawl.get(id); options.onPoll?.(job); if (TERMINAL.has(job.status)) return job; if (Date.now() + pollMs > deadline) throw new FetchaError("CRAWL_WAIT_TIMEOUT", `Crawl ${id} did not finish within ${timeoutMs} ms (status: ${job.status}).`, 0, null); await new Promise((r) => setTimeout(r, pollMs)); } }, /** Iterate over every page of a job, following cursors. */ iteratePages: (id: string, options: Omit = {}): AsyncGenerator => { const pages = this.crawl.pages; return (async function* () { let cursor: string | null = null; do { const page: CrawlPagesResult = await pages(id, { ...options, cursor }); for (const p of page.data) yield p; cursor = page.next_cursor; } while (cursor); })(); }, }; readonly sessions = { create: (options: SessionOptions = {}, idempotencyKey?: string) => this.call("/v1/sessions", { method: "POST", body: options, idempotencyKey }), get: (id: string) => this.call(`/v1/sessions/${encodeURIComponent(id)}`, { method: "GET" }), close: (id: string) => this.call<{ id: string; status: string }>(`/v1/sessions/${encodeURIComponent(id)}`, { method: "DELETE" }), list: () => this.call<{ data: Session[] }>("/v1/sessions", { method: "GET" }), }; readonly usage = { current: () => this.call>("/v1/usage", { method: "GET" }), }; /** Validate the API key and return the project it belongs to. */ me(): Promise<{ project: { id: string; name: string }; organization: { id: string; name: string; plan: string }; key: { id: string; name: string; scopes: string[] } }> { return this.call("/v1/me", { method: "GET" }); } } export default Fetcha;