import type { Metadata } from "next"; import Link from "next/link"; import { Download } from "lucide-react"; import { Button } from "@/components/ui/button"; import { CodeBlock } from "@/components/ui/code-block"; import { DocPage } from "@/components/docs/doc-page"; import { A, Code, H2, H3, Li, P, Strong, Table, TBody, Td, Th, THead, Tr, Ul } from "@/components/docs/prose"; import { Callout } from "@/components/docs/callout"; import { Endpoint } from "@/components/docs/endpoint"; import { ParamTable } from "@/components/docs/param-table"; import { CodeTabs } from "@/components/docs/code-tabs"; import { ResponseExample } from "@/components/docs/response-example"; import { fetchTabs } from "@/components/docs/snippets"; export const metadata: Metadata = { title: "Fetch API", description: "Complete reference for POST /v1/fetch: every request field including browser rendering, the response shape, formats (html, text, markdown, json, raw), page metadata, links, redirects, size limits and error behaviour.", }; const EXAMPLE_BODY = { url: "https://example.com/products/42", country: "CA", region: "QC", format: "text", timeout: 20000, headers: { Accept: "text/html" }, retries: 2, debug: true, }; const SUCCESS = { request_id: "req_k3j9d0f2a8b1c7e4", success: true, status: 200, url: "https://example.com/products/42", final_url: "https://example.com/products/42", content: null, content_type: "text/html; charset=utf-8", headers: { "content-type": "text/html; charset=utf-8", "cache-control": "private, max-age=0", "set-cookie": "sid=3f9a…; Path=/; Domain=example.com; HttpOnly", }, cookies: [{ name: "sid", value: "3f9a…", domain: "example.com", path: "/" }], text: "Example Product 42\nCA$ 129.00\nIn stock — ships from Montréal…", page: { title: "Example Product 42 — Example Shop", description: "Buy Example Product 42 for CA$ 129.00.", canonical: "https://example.com/products/42", lang: "en", og: { "og:title": "Example Product 42", "og:type": "product" }, links_count: 38 }, metadata: { network: "residential", country: "CA", mode: "http", attempts: 1, duration_ms: 1184, bytes: 48211, session: null, cached: false, timing: { dns_ms: 21, proxy_connect_ms: 0, tls_ms: 0, origin_ms: 934, processing_ms: 12, total_ms: 1184 }, debug: { attempts: [{ provider: "network-a", network: "residential", mode: "http", country: "CA", outcome: "success", block_reason: null, status: 200, duration_ms: 1102 }] }, }, }; const ESCALATED = { request_id: "req_7h2k9m4p1q8r5s3t", success: true, status: 200, url: "https://app.example.io/listings", final_url: "https://app.example.io/listings", content: null, content_type: "text/html; charset=utf-8", headers: { "content-type": "text/html; charset=utf-8" }, cookies: [{ name: "cf_clearance", value: "…", domain: ".example.io", path: "/" }], markdown: "# Listings\n\n| Ref | Price |\n| --- | --- |\n| A-1041 | CA$ 1,250 |\n…", page: { title: "Listings", description: null, canonical: null, lang: "en", og: {}, links_count: 112 }, metadata: { network: "residential", country: "CA", mode: "browser", attempts: 2, duration_ms: 6820, bytes: 391204, session: null, cached: false, timing: { dns_ms: 19, proxy_connect_ms: 0, tls_ms: 0, origin_ms: 5210, processing_ms: 88, total_ms: 6820 }, debug: { attempts: [ { provider: "network-a", network: "residential", mode: "http", country: "CA", outcome: "blocked", block_reason: "cloudflare_challenge", status: 403, duration_ms: 1204 }, { provider: "network-a", network: "residential", mode: "browser", country: "CA", outcome: "success", block_reason: null, status: 200, duration_ms: 5480 }, ], }, }, }; const BLOCKED = { request_id: "req_2b7e4c9a1d0f8g6h", success: false, status: 403, url: "https://shop.example.net/", final_url: "https://shop.example.net/", content: "Access Denied…", content_type: "text/html", headers: { "content-type": "text/html", server: "AkamaiGHost" }, cookies: [], metadata: { network: "residential", country: "US", mode: "browser", attempts: 4, duration_ms: 11410, bytes: 9120, session: null, cached: false, timing: { dns_ms: 18, proxy_connect_ms: 0, tls_ms: 0, origin_ms: 1710, processing_ms: 4, total_ms: 11410 }, }, }; const INVALID = { error: { code: "INVALID_REQUEST", message: "The request body is invalid.", request_id: "req_9c1d2e3f4a5b6c7d", details: { issues: [ { path: "timeout", message: "Number must be greater than or equal to 1000" }, { path: "format", message: "Invalid enum value. Expected 'html' | 'text' | 'markdown' | 'json' | 'raw', received 'pdf'" }, ], }, }, }; export default function FetchApiPage() { return (
Machine-readable description of the public endpoints (fetch, crawl, map, sessions, me, usage).

Request

Send a JSON object with Content-Type: application/json. The schema is strict: unknown fields cause 400 INVALID_REQUEST rather than being ignored. The request body itself is limited to 4 MB.

Absolute URL to fetch. Credentials in the URL, private/internal hosts and non-http schemes are refused with URL_NOT_ALLOWED. }, { name: "method", type: '"GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS"', default: '"GET"', description: <>HTTP method sent to the target. }, { name: "headers", type: "object", constraints: "≤ 64 entries; name ≤ 256, value ≤ 8,192 chars", description: <>Headers forwarded to the target. They override Fetcha's defaults: a complete, ordered browser header profile (Chrome, Firefox or Safari, rotated per attempt) with matching User-Agent, Accept, Accept-Language, Accept-Encoding and client hints. Hop-by-hop headers and Host are dropped. }, { name: "cookies", type: "object", constraints: "name ≤ 256, value ≤ 4,096 chars", description: <>Cookies sent with the request. They are serialised into a Cookie header and appended to any Cookie header you also set. }, { name: "body", type: "string | object", constraints: "string ≤ 2,000,000 chars", description: <>Request body. Strings are sent verbatim; objects are JSON-serialised. If you do not set Content-Type, it defaults to application/json. Ignored for GET and HEAD. }, { name: "timeout", type: "integer (ms)", default: "30000", constraints: "1,000–120,000", description: <>Overall deadline for the whole request, including every retry and any browser render (maximum 120 s). When fewer than 500 ms remain, Fetcha stops retrying and returns TARGET_TIMEOUT. }, { name: "country", type: "string", constraints: "exactly 2 chars, upper-cased", description: <>ISO 3166-1 alpha-2 country of the exit IP, e.g. {`"CA"`}. See Geolocation. }, { name: "region", type: "string", constraints: "≤ 64 chars", description: <>State or province. US states and Canadian provinces accept two-letter codes ({`"QC"`}, {`"NY"`}) or full names; other values are slugified. }, { name: "city", type: "string", constraints: "≤ 128 chars", description: <>City name, slugified ({`"Quebec"`} → quebec, {`"New York"`} → new_york). }, { name: "network", type: '"auto" | "datacenter" | "residential" | "isp" | "mobile"', default: '"auto"', description: <>Network class. auto lets the engine choose and escalate. Only residential is live today; explicitly requesting another class returns NETWORK_UNAVAILABLE. See Network selection. }, { name: "session", type: "string", constraints: "≤ 64 chars", description: <>Id of a session created with POST /v1/sessions (sess_…). Pins the exit identity, network and default country for this request. Unknown ids return SESSION_NOT_FOUND; expired or closed ones return SESSION_EXPIRED. }, { name: "format", type: '"html" | "text" | "markdown" | "json" | "raw"', default: '"html"', description: <>Controls the response. html and raw return the body in content. text returns readable text in text (scripts, styles and tags removed) and sets content to null. markdown converts the page to Markdown in markdown (main content first, navigation and boilerplate removed; headings, lists, links, tables and code preserved) and sets content to null. json returns the body in content and, when it parses, the parsed value in json. }, { name: "follow_redirects", type: "boolean", default: "true", description: <>Follow 301/302/303/307/308 responses. When false, the redirect response is returned as-is (success is true for 3xx). }, { name: "max_redirects", type: "integer", default: "10", constraints: "0–20", description: <>Maximum hops before TOO_MANY_REDIRECTS. }, { name: "max_response_bytes", type: "integer", constraints: "1,024–50,000,000", description: <>Lower the response size cap for this request. The platform limit is 20 MB; values above it are reduced to 20 MB. Exceeding the cap returns RESPONSE_TOO_LARGE. }, { name: "retries", type: "integer", default: "5", constraints: "0–5", description: <>Number of additional attempts after the first. Retries use a fresh exit IP and a different header profile, with jittered backoff; a Retry-After header from the target is honoured within the timeout. See Retries. }, { name: "device", type: '"desktop" | "mobile" | "tablet"', description: <>mobile sets an iPhone Safari User-Agent unless you provide your own. desktop is the default behaviour. tablet is accepted but currently changes nothing. }, { name: "locale", type: "string", constraints: "≤ 16 chars", description: <>Sets the Accept-Language header, e.g. {`"fr-CA"`}. }, { name: "debug", type: "boolean", default: "false", description: <>Adds metadata.debug.attempts with one entry per attempt (route alias, network, mode, country, outcome, block reason, status, duration). }, { name: "links", type: "boolean", default: "false", description: <>Return every hyperlink of the page in links[] as absolute URLs with anchor text, internal (same registrable domain) and nofollow flags. HTML responses only. page.links_count is always present. }, { name: "referer", type: '"auto" | "none" | url', default: '"auto"', constraints: "url ≤ 2,048 chars", description: <>Referer strategy. auto sends none on the first attempt and a search-engine referer on retries; none never sends one; a literal URL is sent as-is. }, { name: "browser", type: "boolean", default: "false", description: <>Render the page in the managed headless Chromium, routed through the same network class, geography and session as a plain fetch. Returns the DOM after the page settles. See Browser. }, { name: "browser_fallback", type: "boolean", default: "true", description: <>When an HTTP attempt is blocked by a JavaScript challenge or anti-bot page, automatically retry in the browser. Set to false to stay on plain HTTP. }, { name: "wait_for", type: "string (CSS selector)", constraints: "≤ 512 chars", description: <>Browser: selector that must be present before the DOM is captured. }, { name: "wait_ms", type: "integer (ms)", constraints: "0–30,000", description: <>Browser: extra settle time after the wait condition (and after wait_for when set). }, { name: "wait_until", type: '"load" | "domcontentloaded" | "networkidle"', default: '"domcontentloaded"', description: <>Browser: navigation event to wait for before applying wait_for / wait_ms. }, { name: "javascript", type: "boolean", default: "true", description: <>Browser: set to false to render with scripting disabled. }, { name: "block_resources", type: "boolean", default: "true", description: <>Browser: skip images, fonts and media to save bandwidth and time. Page scripts and XHR still run. }, { name: "screenshot", type: "boolean", default: "false", description: <>Browser: return a PNG of the viewport, base64-encoded, in screenshot. }, { name: "solve_captcha", type: "boolean", default: "true", description: <>Browser: when a Cloudflare Turnstile challenge blocks the render, obtain a token from the managed captcha solver and pass it to the page. Set false to never spend solver credits on a request. Attempts that needed a token report captcha_solved: true in metadata.debug. }, { name: "cache", type: "{ enabled?: boolean, ttl?: integer }", reserved: true, constraints: "ttl 1–86,400 s", description: <>Reserved for response caching. Accepted, ignored; metadata.cached is always false today. }, ]} /> wait_for, wait_ms, wait_until, javascript, block_resources and screenshot only take effect when a render happens: either because browser: true, or because an HTTP attempt was blocked and browser_fallback escalated to the browser. On a plain HTTP response they are accepted and have no effect.

Example

Fetch a product page as readable text through a Canadian residential exit in Québec, with a 20-second budget and debug metadata.

Response

Whenever Fetcha obtains a response from the target, the API answers 200 OK with the document below, even if the target replied with an error status. The HTTP status of the API call therefore means “Fetcha did its job”; the success and status fields tell you what the target did.

Identifier of this request (req_…). Also sent as the X-Fetcha-Request-ID header. }, { name: "success", type: "boolean", description: <>true when the final response has a 2xx or 3xx status and was not classified as a block page. false for 4xx/5xx and for targets that blocked every attempt. }, { name: "status", type: "integer", description: <>HTTP status returned by the target on the final attempt. }, { name: "url", type: "string", description: <>The URL you requested. }, { name: "final_url", type: "string", description: <>URL after redirects. }, { name: "content", type: "string | null", description: <>Response body for html, raw and json formats; null for text. Text-like content types (text/*, JSON, XML, JavaScript, form-encoded) are returned as UTF-8 text; everything else (images, PDFs, archives) is returned base64-encoded. }, { name: "content_type", type: "string | null", description: <>The target's Content-Type header. }, { name: "headers", type: "object", description: <>Response headers from the target with lower-cased names. Multiple values are joined with {`", "`}. content-encoding is removed because Fetcha decompresses gzip, deflate, brotli and zstd bodies for you. }, { name: "cookies", type: "array", description: <>Cookies parsed from Set-Cookie: {`{ name, value, domain?, path? }`}. Replay them via the cookies request field. }, { name: "text", type: "string | null", description: <>Present only for {`format: "text"`}. Readable text extracted from HTML; null when the body was binary. }, { name: "markdown", type: "string | null", description: <>Present only for {`format: "markdown"`}. The page converted to Markdown, main content first with navigation, footers, cookie banners and scripts removed; null when the body was not HTML. }, { name: "json", type: "any", description: <>Present only for {`format: "json"`} and only when the body parsed as JSON. Absent otherwise; check content in that case. }, { name: "page", type: "object | null", description: <>Parsed page metadata for HTML responses: {`{ title, description, canonical, lang, og, links_count }`}. og maps Open Graph property names to values. null for non-HTML bodies. }, { name: "links", type: "array", description: <>Present only with links: true on an HTML response. Items are {`{ url, text, internal, nofollow }`} with absolute URLs, de-duplicated, in document order. }, { name: "screenshot", type: "string", description: <>Present only when the page was rendered in the browser with screenshot: true. PNG, base64-encoded. }, { name: "metadata", type: "object", description: <>Routing information, see below. }, ]} />

Escalated to the browser

The same document is returned when Fetcha had to render the page. metadata.mode tells you how the final attempt was made, and with debug: true each attempt shows its mode and, when blocked, the block_reason that triggered the escalation.

metadata

Concrete network class that served the final attempt. Never auto. }, { name: "country", type: "string | null", description: <>Country targeted by the final attempt (the request's country, or the session's). null when no geography was requested. }, { name: "mode", type: '"http" | "browser"', description: <>http for a plain fetch, browser when the final attempt was rendered in the managed browser (requested or escalated). }, { name: "attempts", type: "integer", description: <>Number of attempts made, including the successful one. Greater than 1 means Fetcha retried or escalated. }, { name: "duration_ms", type: "integer", description: <>Wall-clock time of the whole request inside Fetcha, all attempts included. }, { name: "bytes", type: "integer", description: <>Bytes transferred across all attempts (request and response). This is the quantity used for bandwidth pricing on premium networks. }, { name: "session", type: "string | null", description: <>Session id used, if any. }, { name: "cached", type: "boolean", description: <>Always false today (caching is reserved). }, { name: "timing", type: "object", description: <>Breakdown of the final attempt, see below. }, { name: "debug", type: "object", description: <>Only with debug: true. attempts[] of {`{ provider, network, mode, country, outcome, block_reason, status, duration_ms }`}. provider is a neutral route alias (network-a, network-b, …); mode is http or browser; outcome is one of success, blocked, timeout, error, provider_error, too_large; block_reason names the detector that classified a blocked attempt (for example cloudflare_challenge, datadome, captcha, soft_block) and is null otherwise. }, ]} />

metadata.timing

All values are integers in milliseconds and describe the final attempt (except total_ms).

Field Meaning
dns_ms Time Fetcha spent resolving the target hostname to validate it against the URL policy (SSRF protection). This happens once per request, before routing.
proxy_connect_ms Time to establish the connection to the network. Reported as 0 in the current build; connection time is included in origin_ms.
tls_ms TLS handshake time. Reported as 0 in the current build; included in origin_ms.
origin_ms Time from sending the request until the first byte of the final response (connection, handshake, upstream routing and origin processing), summed across redirect hops of that attempt.
processing_ms Time Fetcha spent decompressing the body and converting it to the requested format.
total_ms End-to-end duration inside Fetcha for the whole request, identical to metadata.duration_ms.

Blocked targets

A block is any response Fetcha classifies as anti-bot interference. Detection covers HTTP 403, 407, 429 and 999, 503 challenge pages, captcha markup and WAF signatures on 4xx/5xx, plus vendor-specific fingerprints in headers, cookies and the first part of the body: Cloudflare challenges and Turnstile, DataDome, PerimeterX / HUMAN, Akamai Bot Manager, Kasada, Imperva / Incapsula, AWS WAF challenges and Vercel attack-mode pages. Soft blocks are detected too: a 200 whose body is a challenge, an interstitial, an empty shell or a “verify you are human” page rather than the real content.

Blocks trigger escalation: a new exit IP and header profile, then a premium route, and, when the block is a JavaScript challenge and browser_fallback is on (the default), a render in the managed browser, which solves most challenge pages. If every attempt is blocked, Fetcha still returns 200 with the last blocked page so you can inspect it, but with success: false. The request log records it as TARGET_BLOCKED, and debug.attempts[].block_reason tells you what was detected.

A plain 404 or 500 from the origin is not a block. It is returned after a single attempt with success: false and the origin status, without retrying.

Redirects

  • 301, 302, 303, 307 and 308 with a Location header are followed while follow_redirects is true, up to max_redirects hops.
  • Every hop is validated against the same URL policy as the original URL. A redirect to a private or internal address fails the request with URL_NOT_ALLOWED{" "} and is not retried.
  • A 303, or a 301/302 following a POST, switches the next hop to GET and drops the body, matching browser behaviour.
  • Exceeding max_redirects fails with 502 TOO_MANY_REDIRECTS, without retrying.

Size limits

  • Response bodies are capped at 20 MB. You can lower the cap per request with max_response_bytes; you cannot raise it above 20 MB. Larger bodies fail with{" "} 502 RESPONSE_TOO_LARGE.
  • Request bodies to the API are capped at 4 MB (413, reported as INVALID_REQUEST).
  • body strings are limited to 2,000,000 characters by the schema.

Content types

Fetcha does not transform binary content. When the target's Content-Type is not text-like (text/*, JSON, XML, JavaScript or form-encoded), content{" "} contains the body encoded as base64 and text is null. Decode it with your language's base64 helper; content_type tells you what it is.

Errors

When Fetcha cannot produce a response document, the API returns a non-200 status with an error envelope. Validation failures list every offending field in details.issues. The full catalogue is on the Errors page.

Situation Result
Invalid or unknown field, malformed JSON, wrong content type, body over 4 MB 400 INVALID_REQUEST
Private, internal or non-http URL (initial or via redirect) 400 URL_NOT_ALLOWED
Network class not in your plan or not live 400 NETWORK_UNAVAILABLE
browser: true while the browser pool is disabled or down 400 BROWSER_UNAVAILABLE
Page did not settle in the browser before the timeout 504 BROWSER_TIMEOUT
Timeout exhausted across all attempts 504 TARGET_TIMEOUT
DNS failure or connection refused on every attempt 502 TARGET_UNAVAILABLE
No route available at all 503 PROVIDER_UNAVAILABLE
); }