import type { Metadata } from "next"; import { CodeBlock } from "@/components/ui/code-block"; import { DocPage } from "@/components/docs/doc-page"; import { A, Code, H2, H3, Li, P, Strong, Table, TBody, Td, Th, THead, Tr, Ul } from "@/components/docs/prose"; import { Callout } from "@/components/docs/callout"; import { Endpoint } from "@/components/docs/endpoint"; import { ParamTable } from "@/components/docs/param-table"; import { CodeTabs } from "@/components/docs/code-tabs"; import { ResponseExample } from "@/components/docs/response-example"; import { apiTabs } from "@/components/docs/snippets"; export const metadata: Metadata = { title: "Crawl & Map", description: "Crawl a site into Markdown, text or HTML with POST /v1/crawl (asynchronous jobs, pages, cancel) and discover its URLs with POST /v1/map. Options, statuses, pagination, limits and polling examples.", }; const CREATE_BODY = { url: "https://docs.example.com/", max_pages: 200, max_depth: 3, include_patterns: ["/docs/*"], exclude_patterns: ["*/changelog*", "/\\.(pdf|zip)$/"], format: "markdown", country: "CA", label: "docs-site", }; const CREATED = { id: "crawl_3k9d0f2a8b1c7e4m", status: "queued", seed_url: "https://docs.example.com/", created_at: "2026-09-08T14:02:11.000Z", options: { url: "https://docs.example.com/", max_pages: 200, max_depth: 3, same_domain: true, allow_subdomains: false, include_patterns: ["/docs/*"], exclude_patterns: ["*/changelog*", "/\\.(pdf|zip)$/"], respect_robots: true, use_sitemap: false, concurrency: 3, delay_ms: 0, timeout: 30000, format: "markdown", main_content: true, country: "CA", network: "auto", browser: false, browser_fallback: true, label: "docs-site", }, }; const JOB = { id: "crawl_3k9d0f2a8b1c7e4m", status: "completed", label: "docs-site", seed_url: "https://docs.example.com/", domain: "docs.example.com", options: { "…": "as submitted, with defaults filled in" }, stats: { discovered: 312, fetched: 200, ok: 196, blocked: 1, failed: 3, bytes: 18422911 }, error: null, created_at: "2026-09-08T14:02:11.000Z", started_at: "2026-09-08T14:02:12.000Z", completed_at: "2026-09-08T14:06:48.000Z", }; const PAGES = { data: [ { id: "cpg_9a1b2c3d4e5f6g7h", url: "https://docs.example.com/docs/getting-started", final_url: "https://docs.example.com/docs/getting-started", depth: 1, status: "success", http_status: 200, error_code: null, title: "Getting started — Example Docs", description: "Install the CLI and run your first command.", content_type: "text/html; charset=utf-8", content: "# Getting started\n\nInstall the CLI:\n\n```bash\nnpm i -g example\n```\n…", links_count: 41, bytes: 88412, duration_ms: 812, mode: "http", fetched_at: "2026-09-08T14:02:14.000Z", }, { id: "cpg_1h2g3f4e5d6c7b8a", url: "https://docs.example.com/docs/pricing", final_url: "https://docs.example.com/docs/pricing", depth: 1, status: "success", http_status: 200, error_code: null, title: "Pricing — Example Docs", description: null, content_type: "text/html; charset=utf-8", content: "# Pricing\n\n| Plan | Price |\n| --- | --- |\n…", links_count: 27, bytes: 210934, duration_ms: 5104, mode: "browser", fetched_at: "2026-09-08T14:02:21.000Z", }, ], next_cursor: "eyJmIjoiMjAyNi0wOS0wOFQxNDowMjoyMS4wMDBaIiwiaWQiOiJjcGdfMWgyZzNmNGU1ZDZjN2I4YSJ9", }; const MAP_RESULT = { url: "https://docs.example.com/", count: 4, urls: ["https://docs.example.com/", "https://docs.example.com/docs/getting-started", "https://docs.example.com/docs/pricing", "https://docs.example.com/docs/api/fetch"], sources: { sitemap: 312, links: 58 }, truncated: false, }; const POLL_JS = `import { Fetcha } from "@fetcha/sdk"; const fetcha = new Fetcha({ apiKey: process.env.FETCHA_API_KEY! }); // 1. Start the job (returns immediately with status "queued") const job = await fetcha.crawl.create({ url: "https://docs.example.com/", max_pages: 200, max_depth: 3, format: "markdown" }); // 2. Wait for a terminal status (polls GET /v1/crawl/:id every 2 s, up to 10 min) const done = await fetcha.crawl.wait(job.id, { pollMs: 2000, timeoutMs: 600_000 }); console.log(done.status, done.stats); // "completed" { discovered, fetched, ok, blocked, failed, bytes } // 3. Page through the results with the cursor let cursor: string | null = null; do { const page = await fetcha.crawl.pages(job.id, { cursor, limit: 100, status: "success" }); for (const p of page.data) console.log(p.url, p.title, p.content?.length); cursor = page.next_cursor; } while (cursor);`; const POLL_PY = `import os from fetcha import Fetcha client = Fetcha(api_key=os.environ["FETCHA_API_KEY"]) # 1. Start the job job = client.crawl.create(url="https://docs.example.com/", max_pages=200, max_depth=3, format="markdown") # 2. Wait for a terminal status (polls GET /v1/crawl/:id) job = client.crawl.wait(job.id, poll_s=2.0, timeout_s=600) print(job.status, job.stats) # 3. Page through the results cursor = None while True: page = client.crawl.pages(job.id, cursor=cursor, limit=100, status="success") for p in page.data: print(p.url, p.title, len(p.content or "")) cursor = page.next_cursor if not cursor: break`; const POLL_RAW = `# Without the SDK: poll until status is completed | failed | cancelled JOB=$(curl -s https://www.fetcha.co/v1/crawl -X POST \\ -H "Authorization: Bearer $FETCHA_API_KEY" -H "Content-Type: application/json" \\ -d '{"url": "https://docs.example.com/", "max_pages": 200}' | jq -r .id) until [ "$(curl -s https://www.fetcha.co/v1/crawl/$JOB -H "Authorization: Bearer $FETCHA_API_KEY" | jq -r .status)" != "running" ]; do sleep 2; done curl -s "https://www.fetcha.co/v1/crawl/$JOB/pages?limit=100" -H "Authorization: Bearer $FETCHA_API_KEY" | jq '.data[] | {url, status, title}'`; export default function CrawlPage() { return (

How a crawl works

Up to 2,000 pages per job and 5 concurrent jobs per organization (a sixth returns 429 CRAWL_LIMIT_REACHED). Jobs and their pages are kept for the log retention period (90 days).

Start a crawl

Seed URL. Same URL policy as fetch (URL_NOT_ALLOWED for private or internal hosts). }, { name: "max_pages", type: "integer", default: "25", constraints: "1–5,000, capped at 2,000", description: <>Maximum number of pages to fetch. The seed counts as one. }, { name: "max_depth", type: "integer", default: "2", constraints: "0–10", description: <>Maximum link depth from the seed. 0 fetches the seed only. }, { name: "same_domain", type: "boolean", default: "true", description: <>Only follow links whose registrable domain matches the seed. }, { name: "allow_subdomains", type: "boolean", default: "false", description: <>With same_domain, also follow links on subdomains of the seed host. }, { name: "include_patterns", type: "string[]", constraints: "≤ 50 patterns, each ≤ 512 chars", description: <>Only crawl URLs matching at least one pattern. Glob with * (matched against the full URL or the path) or a regular expression written /…/. }, { name: "exclude_patterns", type: "string[]", constraints: "≤ 50 patterns", description: <>Never crawl URLs matching one of these patterns. Evaluated after include_patterns. }, { name: "respect_robots", type: "boolean", default: "true", description: <>Honour Disallow rules of the seed host's robots.txt. }, { name: "use_sitemap", type: "boolean", default: "false", description: <>Also seed the frontier with URLs from sitemap.xml and sitemaps listed in robots.txt (subject to the same scope rules). }, { name: "concurrency", type: "integer", default: "3", constraints: "1–10", description: <>Parallel page fetches within the job. }, { name: "delay_ms", type: "integer", default: "0", constraints: "0–30,000", description: <>Pause between fetches per worker (politeness). }, { name: "timeout", type: "integer (ms)", default: "30000", constraints: "1,000–120,000", description: <>Per-page timeout, including retries and browser escalation. }, { name: "format", type: '"markdown" | "text" | "html"', default: '"markdown"', description: <>Format of content for each page. }, { name: "main_content", type: "boolean", default: "true", description: <>Keep only the main content (article, main, largest text block) when converting to Markdown or text. }, { name: "country", type: "string", constraints: "2 chars", description: <>Exit country for every page. }, { name: "network", type: '"auto" | "datacenter" | "residential" | "isp" | "mobile"', default: '"auto"', description: <>Network class for every page. }, { name: "browser", type: "boolean", default: "false", description: <>Render every page in the managed browser. Slower and heavier; prefer the default and let escalation handle JavaScript challenges. }, { name: "browser_fallback", type: "boolean", default: "true", description: <>Escalate blocked pages to the browser automatically. }, { name: "headers", type: "object", constraints: "≤ 64 entries", description: <>Headers sent with every page fetch. }, { name: "webhook_url", type: "string (url)", constraints: "≤ 2,048 chars", description: <>Called with a POST containing the job when it reaches a terminal status. Accepted and stored; delivery is part of the upcoming webhooks release. }, { name: "label", type: "string", constraints: "≤ 128 chars", description: <>Free-form label shown in the dashboard. }, ]} />

The response is 202 Accepted: the job is queued and the call returns immediately. Poll GET /v1/crawl/:id (or use the SDK's crawl.wait) to follow progress.

Get a job

Job identifier (crawl_…). }, { name: "status", type: '"queued" | "running" | "completed" | "failed" | "cancelled"', description: <>completed, failed and cancelled are terminal. }, { name: "label", type: "string | null", description: <>Your label. }, { name: "seed_url / domain", type: "string", description: <>The seed and its host. }, { name: "options", type: "object", description: <>The options as submitted, with defaults filled in. }, { name: "stats", type: "object", description: <>{`{ discovered, fetched, ok, blocked, failed, bytes }`}. discovered counts URLs added to the frontier; fetched = ok + blocked + failed. }, { name: "error", type: "object | null", description: <>{`{ code, message }`} when status is failed (for example the seed returned URL_NOT_ALLOWED or was blocked on every attempt). }, { name: "created_at / started_at / completed_at", type: "string | null", description: <>ISO 8601 timestamps. started_at is null while queued; completed_at is set on any terminal status. }, ]} />

List pages

Opaque cursor from the previous response's next_cursor. Omit for the first page. }, { name: "limit", type: "integer", default: "100", description: <>Pages per response. }, { name: "status", type: '"success" | "blocked" | "failed"', description: <>Only return pages with this status. }, ]} /> Page identifier. }, { name: "url / final_url", type: "string", description: <>Requested URL and URL after redirects. }, { name: "depth", type: "integer", description: <>Link distance from the seed (seed = 0). }, { name: "status", type: '"success" | "blocked" | "failed"', description: <>Outcome of the fetch. blocked means every attempt, including browser escalation, was classified as a block. }, { name: "http_status", type: "integer | null", description: <>Status of the final attempt. }, { name: "error_code", type: "string | null", description: <>Fetch error code for failed pages (TARGET_TIMEOUT, TARGET_UNAVAILABLE, …). }, { name: "title / description", type: "string | null", description: <>From the page's <title> and meta description. }, { name: "content_type", type: "string | null", description: <>The target's Content-Type. }, { name: "content", type: "string | null", description: <>Page content in the job's format. null for non-HTML bodies and blocked pages. }, { name: "links_count", type: "integer | null", description: <>Hyperlinks found in the page. }, { name: "bytes / duration_ms", type: "integer | null", description: <>Bytes transferred and wall-clock time for this page, all attempts included. }, { name: "mode", type: '"http" | "browser" | null', description: <>How the final attempt was made. }, { name: "fetched_at", type: "string | null", description: <>ISO 8601 timestamp. }, ]} />

Pages are ordered by fetched_at. next_cursor is null on the last page. While a job is running, new pages appear at the end; keep the last cursor to fetch only what is new.

Cancel a job

Cancellation is immediate for queued jobs; running jobs stop after the in-flight pages finish. Pages already fetched stay available. Cancelling a job that is already terminal returns it unchanged.

List jobs

Returns the most recent jobs of the project, newest first. ?limit= defaults to 50.

Condition Result
Invalid body (unknown field, pattern too long, out-of-range value) 400 INVALID_REQUEST
Seed URL private, internal or non-http 400 URL_NOT_ALLOWED
Id does not exist or belongs to another project 404 CRAWL_NOT_FOUND
Five jobs already queued or running 429 CRAWL_LIMIT_REACHED

Polling example

The SDKs wrap the four calls and add crawl.wait, which polls until a terminal status. Both examples start a job, wait for it, then stream the pages with the cursor.

Map a site

map answers the question “which URLs does this site have?” without fetching every page. It reads the sitemap(s) and the links of the seed page, filters them, and returns a de-duplicated list, synchronously (the call takes up to 60 s). Use it to pick include_patterns before a crawl, or to feed URLs into your own fetch loop.

Site or page to map. }, { name: "limit", type: "integer", default: "1000", constraints: "1–10,000", description: <>Maximum number of URLs returned. truncated is true when more were found. }, { name: "use_sitemap", type: "boolean", default: "true", description: <>Read sitemap.xml, sitemap indexes and sitemaps listed in robots.txt. }, { name: "use_links", type: "boolean", default: "true", description: <>Include hyperlinks found on the seed page. }, { name: "same_domain", type: "boolean", default: "true", description: <>Drop URLs outside the seed's registrable domain. }, { name: "allow_subdomains", type: "boolean", default: "false", description: <>Keep subdomains of the seed host. }, { name: "search", type: "string", constraints: "≤ 256 chars", description: <>Keep only URLs matching this substring, glob (*) or /regex/. }, { name: "country / network / timeout", type: "—", description: <>Same meaning as in fetch; apply to the sitemap and seed-page requests. }, ]} /> The seed you passed. }, { name: "count", type: "integer", description: <>Number of URLs in urls. }, { name: "urls", type: "string[]", description: <>Absolute, de-duplicated URLs; sitemap entries first, then links in document order. }, { name: "sources", type: "object", description: <>{`{ sitemap, links }`}: how many candidates each source contributed before filtering and de-duplication. }, { name: "truncated", type: "boolean", description: <>true when the result was cut at limit. }, ]} />

Map, then crawl

A map costs a handful of fetches (one per sitemap file plus the seed page) and does not create a job. See the SDKs page for the full client reference.

); }