TypeScript 97.5%
SQL 1.4%
Python 0.8%
1/**2 * @fetcha/sdk — Official JavaScript / TypeScript client for Fetcha.3 *4 * ```ts5 * import { Fetcha } from "@fetcha/sdk";6 * const fetcha = new Fetcha({ apiKey: process.env.FETCHA_API_KEY! });7 * const result = await fetcha.fetch({ url: "https://example.com", country: "CA", format: "markdown" });8 * console.log(result.status, result.metadata.mode, result.markdown?.slice(0, 200));9 *10 * const job = await fetcha.crawl.create({ url: "https://docs.example.com/", max_pages: 100 });11 * const done = await fetcha.crawl.wait(job.id);12 * const pages = await fetcha.crawl.pages(job.id, { limit: 100 });13 * ```14 *15 * Zero dependencies; works on Node 18+, Deno, Bun and modern browsers (global `fetch`).16 */1718export const SDK_VERSION = "0.2.0";1920export type FetchaNetwork = "auto" | "datacenter" | "residential" | "isp" | "mobile";21export type FetchaFormat = "html" | "text" | "markdown" | "json" | "raw";22export type FetchaMode = "http" | "browser";23export type FetchaWaitUntil = "load" | "domcontentloaded" | "networkidle";2425export interface FetchOptions {26 url: string;27 method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS";28 headers?: Record<string, string>;29 cookies?: Record<string, string>;30 body?: string | Record<string, unknown>;31 /** Overall deadline in ms for all attempts, including browser renders (1,000–120,000). */32 timeout?: number;33 country?: string;34 region?: string;35 city?: string;36 network?: FetchaNetwork;37 session?: string;38 /** Render in the managed headless browser routed through the same network / geo / session. */39 browser?: boolean;40 /** Escalate to the browser automatically when an HTTP attempt is blocked by a JS challenge (default true). */41 browser_fallback?: boolean;42 /** Browser: CSS selector to wait for before capturing. */43 wait_for?: string;44 /** Browser: extra settle time in ms (0–30,000). */45 wait_ms?: number;46 /** Browser: navigation wait condition (default "domcontentloaded"). */47 wait_until?: FetchaWaitUntil;48 /** Browser: disable scripting when false. */49 javascript?: boolean;50 /** Browser: skip images, fonts and media (default true). */51 block_resources?: boolean;52 /** Browser: return a PNG screenshot (base64) in `screenshot`. */53 screenshot?: boolean;54 /** Browser: solve Cloudflare Turnstile challenges with the platform's captcha solver (default true). */55 solve_captcha?: boolean;56 /** Return `links[]` (all hyperlinks, absolute). */57 links?: boolean;58 /** Referer strategy: "auto" (none first, search-engine referer on retries), "none", or a literal URL. */59 referer?: "auto" | "none" | string;60 device?: "desktop" | "mobile" | "tablet";61 locale?: string;62 format?: FetchaFormat;63 follow_redirects?: boolean;64 max_redirects?: number;65 max_response_bytes?: number;66 retries?: number;67 debug?: boolean;68}6970export interface PageMetadata {71 title: string | null;72 description: string | null;73 canonical: string | null;74 lang: string | null;75 og: Record<string, string>;76 links_count: number;77}7879export interface PageLink {80 url: string;81 text: string;82 internal: boolean;83 nofollow: boolean;84}8586export interface FetchAttempt {87 provider: string;88 network: string;89 mode: FetchaMode;90 country: string | null;91 outcome: string;92 block_reason?: string | null;93 status: number | null;94 duration_ms: number;95 error?: string;96}9798export interface FetchResult {99 request_id: string;100 success: boolean;101 status: number;102 url: string;103 final_url: string;104 content: string | null;105 content_type: string | null;106 headers: Record<string, string>;107 cookies: Array<{ name: string; value: string; domain?: string; path?: string }>;108 /** Present for format "text". */109 text?: string | null;110 /** Present for format "json" when the body parsed. */111 json?: unknown;112 /** Present for format "markdown". */113 markdown?: string | null;114 /** Parsed page metadata (HTML responses). */115 page?: PageMetadata | null;116 /** Present with `links: true`. */117 links?: PageLink[];118 /** Present with browser rendering and `screenshot: true` (PNG, base64). */119 screenshot?: string | null;120 metadata: {121 network: string;122 country: string | null;123 /** "http" for a plain fetch, "browser" when the final attempt was rendered. */124 mode: FetchaMode;125 attempts: number;126 duration_ms: number;127 bytes: number;128 session: string | null;129 cached: boolean;130 timing?: Record<string, number>;131 debug?: { attempts: FetchAttempt[] };132 };133}134135export interface SessionOptions {136 country?: string;137 region?: string;138 city?: string;139 network?: FetchaNetwork;140 ttl?: number;141 label?: string;142}143144export interface Session {145 id: string;146 status: string;147 network: string;148 country: string | null;149 expires_at: string;150 created_at: string;151}152153// ---------------------------------------------------------------------------154// Crawl & map155// ---------------------------------------------------------------------------156157export type CrawlFormat = "markdown" | "text" | "html";158export type CrawlStatus = "queued" | "running" | "completed" | "failed" | "cancelled";159160export interface CrawlOptions {161 url: string;162 /** Maximum pages to fetch (1–5,000; the seed counts as one). Default 25. */163 max_pages?: number;164 /** Maximum link depth from the seed (0–10). Default 2. */165 max_depth?: number;166 same_domain?: boolean;167 allow_subdomains?: boolean;168 /** Glob with `*` or `/regex/`. */169 include_patterns?: string[];170 exclude_patterns?: string[];171 respect_robots?: boolean;172 use_sitemap?: boolean;173 /** Parallel page fetches (1–10). Default 3. */174 concurrency?: number;175 delay_ms?: number;176 /** Per-page timeout in ms. */177 timeout?: number;178 format?: CrawlFormat;179 main_content?: boolean;180 country?: string;181 network?: FetchaNetwork;182 browser?: boolean;183 browser_fallback?: boolean;184 headers?: Record<string, string>;185 webhook_url?: string;186 label?: string;187}188189export interface CrawlStats {190 discovered: number;191 fetched: number;192 ok: number;193 blocked: number;194 failed: number;195 bytes: number;196}197198export interface CrawlJob {199 id: string;200 status: CrawlStatus;201 label: string | null;202 seed_url: string;203 domain: string;204 options: Record<string, unknown>;205 stats: CrawlStats;206 error: { code: string; message: string } | null;207 created_at: string;208 started_at: string | null;209 completed_at: string | null;210}211212/** Body of the 202 returned by `crawl.create`. */213export interface CrawlCreated {214 id: string;215 status: "queued";216 seed_url: string;217 created_at: string;218 options: Record<string, unknown>;219}220221export interface CrawlPage {222 id: string;223 url: string;224 final_url: string | null;225 depth: number;226 status: string;227 http_status: number | null;228 error_code: string | null;229 title: string | null;230 description: string | null;231 content_type: string | null;232 content: string | null;233 links_count: number | null;234 bytes: number | null;235 duration_ms: number | null;236 mode: FetchaMode | null;237 fetched_at: string | null;238}239240export interface CrawlPagesResult {241 data: CrawlPage[];242 next_cursor: string | null;243}244245export interface CrawlPagesOptions {246 cursor?: string | null;247 limit?: number;248 status?: "success" | "blocked" | "failed";249}250251export interface CrawlWaitOptions {252 /** Interval between polls in ms. Default 2,000. */253 pollMs?: number;254 /** Give up after this many ms. Default 600,000 (10 min). */255 timeoutMs?: number;256 /** Called after each poll with the current job. */257 onPoll?: (job: CrawlJob) => void;258}259260export interface MapOptions {261 url: string;262 /** Maximum URLs returned (1–10,000). Default 1,000. */263 limit?: number;264 use_sitemap?: boolean;265 use_links?: boolean;266 same_domain?: boolean;267 allow_subdomains?: boolean;268 /** Substring, glob (`*`) or `/regex/` filter. */269 search?: string;270 country?: string;271 network?: FetchaNetwork;272 timeout?: number;273}274275export interface MapResult {276 url: string;277 count: number;278 urls: string[];279 sources: { sitemap: number; links: number };280 truncated: boolean;281}282283export interface FetchaClientOptions {284 apiKey: string;285 baseUrl?: string;286 /** Default request timeout (ms) applied client-side on top of the API timeout. */287 clientTimeout?: number;288 fetch?: typeof globalThis.fetch;289}290291export class FetchaError extends Error {292 readonly code: string;293 readonly status: number;294 readonly requestId: string | null;295 readonly details?: Record<string, unknown>;296 constructor(code: string, message: string, status: number, requestId: string | null, details?: Record<string, unknown>) {297 super(message);298 this.name = "FetchaError";299 this.code = code;300 this.status = status;301 this.requestId = requestId;302 this.details = details;303 }304}305306const TERMINAL: ReadonlySet<CrawlStatus> = new Set(["completed", "failed", "cancelled"]);307308function query(params: Record<string, string | number | null | undefined>): string {309 const sp = new URLSearchParams();310 for (const [k, v] of Object.entries(params)) if (v !== undefined && v !== null && v !== "") sp.set(k, String(v));311 const s = sp.toString();312 return s ? `?${s}` : "";313}314315export class Fetcha {316 private readonly apiKey: string;317 private readonly baseUrl: string;318 private readonly clientTimeout: number;319 private readonly _fetch: typeof globalThis.fetch;320321 constructor(opts: FetchaClientOptions) {322 if (!opts?.apiKey) throw new Error("Fetcha: apiKey is required");323 this.apiKey = opts.apiKey;324 this.baseUrl = (opts.baseUrl ?? "https://www.fetcha.co").replace(/\/$/, "");325 this.clientTimeout = opts.clientTimeout ?? 150_000;326 this._fetch = opts.fetch ?? globalThis.fetch.bind(globalThis);327 }328329 private async call<T>(path: string, init: { method: string; body?: unknown; idempotencyKey?: string }): Promise<T> {330 const ac = new AbortController();331 const timer = setTimeout(() => ac.abort(), this.clientTimeout);332 try {333 const res = await this._fetch(`${this.baseUrl}${path}`, {334 method: init.method,335 headers: {336 authorization: `Bearer ${this.apiKey}`,337 "content-type": "application/json",338 "user-agent": `fetcha-sdk-js/${SDK_VERSION}`,339 ...(init.idempotencyKey ? { "idempotency-key": init.idempotencyKey } : {}),340 },341 body: init.body === undefined ? undefined : JSON.stringify(init.body),342 signal: ac.signal,343 });344 const requestId = res.headers.get("x-fetcha-request-id");345 const text = await res.text();346 let data: unknown = null;347 try {348 data = text ? JSON.parse(text) : null;349 } catch {350 data = null;351 }352 if (!res.ok) {353 const err = (data as { error?: { code?: string; message?: string; request_id?: string; details?: Record<string, unknown> } } | null)?.error;354 throw new FetchaError(err?.code ?? "HTTP_ERROR", err?.message ?? `HTTP ${res.status}`, res.status, err?.request_id ?? requestId, err?.details);355 }356 return data as T;357 } finally {358 clearTimeout(timer);359 }360 }361362 /** Fetch a URL through Fetcha's routing engine (optionally rendered in the managed browser). */363 fetch(options: FetchOptions): Promise<FetchResult> {364 return this.call<FetchResult>("/v1/fetch", { method: "POST", body: options });365 }366367 /** Convenience: GET a page and return readable text. */368 async text(url: string, options: Omit<FetchOptions, "url" | "format"> = {}): Promise<string> {369 const r = await this.fetch({ ...options, url, format: "text" });370 return r.text ?? "";371 }372373 /** Convenience: GET a page and return it as Markdown. */374 async markdown(url: string, options: Omit<FetchOptions, "url" | "format"> = {}): Promise<string> {375 const r = await this.fetch({ ...options, url, format: "markdown" });376 return r.markdown ?? "";377 }378379 /** Convenience: GET a JSON endpoint. */380 async json<T = unknown>(url: string, options: Omit<FetchOptions, "url" | "format"> = {}): Promise<T> {381 const r = await this.fetch({ ...options, url, format: "json" });382 return r.json as T;383 }384385 /** Convenience: render a page in the managed browser. */386 render(url: string, options: Omit<FetchOptions, "url" | "browser"> = {}): Promise<FetchResult> {387 return this.fetch({ ...options, url, browser: true });388 }389390 /** Discover the URLs of a site (sitemap + links), synchronously. */391 map(options: MapOptions): Promise<MapResult> {392 return this.call<MapResult>("/v1/map", { method: "POST", body: options });393 }394395 readonly crawl = {396 /** Start a crawl job. Returns immediately with status "queued". */397 create: (options: CrawlOptions) => this.call<CrawlCreated>("/v1/crawl", { method: "POST", body: options }),398 /** Get a job with its live status and stats. */399 get: (id: string) => this.call<CrawlJob>(`/v1/crawl/${encodeURIComponent(id)}`, { method: "GET" }),400 /** List the most recent jobs of the project. */401 list: (limit?: number) => this.call<{ data: CrawlJob[] }>(`/v1/crawl${query({ limit })}`, { method: "GET" }),402 /** Page through the crawled pages with `cursor` / `next_cursor`. */403 pages: (id: string, options: CrawlPagesOptions = {}) => this.call<CrawlPagesResult>(`/v1/crawl/${encodeURIComponent(id)}/pages${query({ cursor: options.cursor, limit: options.limit, status: options.status })}`, { method: "GET" }),404 /** Cancel a queued or running job. */405 cancel: (id: string) => this.call<{ id: string; status: "cancelled" }>(`/v1/crawl/${encodeURIComponent(id)}`, { method: "DELETE" }),406 /** Poll `get` until the job reaches a terminal status (completed, failed or cancelled). */407 wait: async (id: string, options: CrawlWaitOptions = {}): Promise<CrawlJob> => {408 const pollMs = Math.max(250, options.pollMs ?? 2000);409 const timeoutMs = options.timeoutMs ?? 600_000;410 const deadline = Date.now() + timeoutMs;411 for (;;) {412 const job = await this.crawl.get(id);413 options.onPoll?.(job);414 if (TERMINAL.has(job.status)) return job;415 if (Date.now() + pollMs > deadline) throw new FetchaError("CRAWL_WAIT_TIMEOUT", `Crawl ${id} did not finish within ${timeoutMs} ms (status: ${job.status}).`, 0, null);416 await new Promise((r) => setTimeout(r, pollMs));417 }418 },419 /** Iterate over every page of a job, following cursors. */420 iteratePages: (id: string, options: Omit<CrawlPagesOptions, "cursor"> = {}): AsyncGenerator<CrawlPage, void, undefined> => {421 const pages = this.crawl.pages;422 return (async function* () {423 let cursor: string | null = null;424 do {425 const page: CrawlPagesResult = await pages(id, { ...options, cursor });426 for (const p of page.data) yield p;427 cursor = page.next_cursor;428 } while (cursor);429 })();430 },431 };432433 readonly sessions = {434 create: (options: SessionOptions = {}, idempotencyKey?: string) => this.call<Session>("/v1/sessions", { method: "POST", body: options, idempotencyKey }),435 get: (id: string) => this.call<Session>(`/v1/sessions/${encodeURIComponent(id)}`, { method: "GET" }),436 close: (id: string) => this.call<{ id: string; status: string }>(`/v1/sessions/${encodeURIComponent(id)}`, { method: "DELETE" }),437 list: () => this.call<{ data: Session[] }>("/v1/sessions", { method: "GET" }),438 };439440 readonly usage = {441 current: () => this.call<Record<string, unknown>>("/v1/usage", { method: "GET" }),442 };443444 /** Validate the API key and return the project it belongs to. */445 me(): Promise<{ project: { id: string; name: string }; organization: { id: string; name: string; plan: string }; key: { id: string; name: string; scopes: string[] } }> {446 return this.call("/v1/me", { method: "GET" });447 }448}449450export default Fetcha;451