TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import type { Metadata } from "next";2import { CodeBlock } from "@/components/ui/code-block";3import { DocPage } from "@/components/docs/doc-page";4import { A, Code, H2, H3, Li, P, Strong, Table, TBody, Td, Th, THead, Tr, Ul } from "@/components/docs/prose";5import { Callout } from "@/components/docs/callout";6import { Endpoint } from "@/components/docs/endpoint";7import { ParamTable } from "@/components/docs/param-table";8import { CodeTabs } from "@/components/docs/code-tabs";9import { ResponseExample } from "@/components/docs/response-example";10import { apiTabs } from "@/components/docs/snippets";1112export const metadata: Metadata = {13 title: "Crawl & Map",14 description: "Crawl a site into Markdown, text or HTML with POST /v1/crawl (asynchronous jobs, pages, cancel) and discover its URLs with POST /v1/map. Options, statuses, pagination, limits and polling examples.",15};1617const CREATE_BODY = {18 url: "https://docs.example.com/",19 max_pages: 200,20 max_depth: 3,21 include_patterns: ["/docs/*"],22 exclude_patterns: ["*/changelog*", "/\\.(pdf|zip)$/"],23 format: "markdown",24 country: "CA",25 label: "docs-site",26};2728const CREATED = {29 id: "crawl_3k9d0f2a8b1c7e4m",30 status: "queued",31 seed_url: "https://docs.example.com/",32 created_at: "2026-09-08T14:02:11.000Z",33 options: {34 url: "https://docs.example.com/",35 max_pages: 200,36 max_depth: 3,37 same_domain: true,38 allow_subdomains: false,39 include_patterns: ["/docs/*"],40 exclude_patterns: ["*/changelog*", "/\\.(pdf|zip)$/"],41 respect_robots: true,42 use_sitemap: false,43 concurrency: 3,44 delay_ms: 0,45 timeout: 30000,46 format: "markdown",47 main_content: true,48 country: "CA",49 network: "auto",50 browser: false,51 browser_fallback: true,52 label: "docs-site",53 },54};5556const JOB = {57 id: "crawl_3k9d0f2a8b1c7e4m",58 status: "completed",59 label: "docs-site",60 seed_url: "https://docs.example.com/",61 domain: "docs.example.com",62 options: { "…": "as submitted, with defaults filled in" },63 stats: { discovered: 312, fetched: 200, ok: 196, blocked: 1, failed: 3, bytes: 18422911 },64 error: null,65 created_at: "2026-09-08T14:02:11.000Z",66 started_at: "2026-09-08T14:02:12.000Z",67 completed_at: "2026-09-08T14:06:48.000Z",68};6970const PAGES = {71 data: [72 {73 id: "cpg_9a1b2c3d4e5f6g7h",74 url: "https://docs.example.com/docs/getting-started",75 final_url: "https://docs.example.com/docs/getting-started",76 depth: 1,77 status: "success",78 http_status: 200,79 error_code: null,80 title: "Getting started — Example Docs",81 description: "Install the CLI and run your first command.",82 content_type: "text/html; charset=utf-8",83 content: "# Getting started\n\nInstall the CLI:\n\n```bash\nnpm i -g example\n```\n…",84 links_count: 41,85 bytes: 88412,86 duration_ms: 812,87 mode: "http",88 fetched_at: "2026-09-08T14:02:14.000Z",89 },90 {91 id: "cpg_1h2g3f4e5d6c7b8a",92 url: "https://docs.example.com/docs/pricing",93 final_url: "https://docs.example.com/docs/pricing",94 depth: 1,95 status: "success",96 http_status: 200,97 error_code: null,98 title: "Pricing — Example Docs",99 description: null,100 content_type: "text/html; charset=utf-8",101 content: "# Pricing\n\n| Plan | Price |\n| --- | --- |\n…",102 links_count: 27,103 bytes: 210934,104 duration_ms: 5104,105 mode: "browser",106 fetched_at: "2026-09-08T14:02:21.000Z",107 },108 ],109 next_cursor: "eyJmIjoiMjAyNi0wOS0wOFQxNDowMjoyMS4wMDBaIiwiaWQiOiJjcGdfMWgyZzNmNGU1ZDZjN2I4YSJ9",110};111112const MAP_RESULT = {113 url: "https://docs.example.com/",114 count: 4,115 urls: ["https://docs.example.com/", "https://docs.example.com/docs/getting-started", "https://docs.example.com/docs/pricing", "https://docs.example.com/docs/api/fetch"],116 sources: { sitemap: 312, links: 58 },117 truncated: false,118};119120const POLL_JS = `import { Fetcha } from "@fetcha/sdk";121122const fetcha = new Fetcha({ apiKey: process.env.FETCHA_API_KEY! });123124// 1. Start the job (returns immediately with status "queued")125const job = await fetcha.crawl.create({ url: "https://docs.example.com/", max_pages: 200, max_depth: 3, format: "markdown" });126127// 2. Wait for a terminal status (polls GET /v1/crawl/:id every 2 s, up to 10 min)128const done = await fetcha.crawl.wait(job.id, { pollMs: 2000, timeoutMs: 600_000 });129console.log(done.status, done.stats); // "completed" { discovered, fetched, ok, blocked, failed, bytes }130131// 3. Page through the results with the cursor132let cursor: string | null = null;133do {134 const page = await fetcha.crawl.pages(job.id, { cursor, limit: 100, status: "success" });135 for (const p of page.data) console.log(p.url, p.title, p.content?.length);136 cursor = page.next_cursor;137} while (cursor);`;138139const POLL_PY = `import os140from fetcha import Fetcha141142client = Fetcha(api_key=os.environ["FETCHA_API_KEY"])143144# 1. Start the job145job = client.crawl.create(url="https://docs.example.com/", max_pages=200, max_depth=3, format="markdown")146147# 2. Wait for a terminal status (polls GET /v1/crawl/:id)148job = client.crawl.wait(job.id, poll_s=2.0, timeout_s=600)149print(job.status, job.stats)150151# 3. Page through the results152cursor = None153while True:154 page = client.crawl.pages(job.id, cursor=cursor, limit=100, status="success")155 for p in page.data:156 print(p.url, p.title, len(p.content or ""))157 cursor = page.next_cursor158 if not cursor:159 break`;160161const POLL_RAW = `# Without the SDK: poll until status is completed | failed | cancelled162JOB=$(curl -s https://www.fetcha.co/v1/crawl -X POST \\163 -H "Authorization: Bearer $FETCHA_API_KEY" -H "Content-Type: application/json" \\164 -d '{"url": "https://docs.example.com/", "max_pages": 200}' | jq -r .id)165166until [ "$(curl -s https://www.fetcha.co/v1/crawl/$JOB -H "Authorization: Bearer $FETCHA_API_KEY" | jq -r .status)" != "running" ]; do sleep 2; done167168curl -s "https://www.fetcha.co/v1/crawl/$JOB/pages?limit=100" -H "Authorization: Bearer $FETCHA_API_KEY" | jq '.data[] | {url, status, title}'`;169170export default function CrawlPage() {171 return (172 <DocPage path="/docs/crawl" eyebrow="Core API" title="Crawl & Map" description="Turn a site into Markdown, text or HTML in one call: POST /v1/crawl starts an asynchronous job that follows links from a seed URL and fetches every page through the routing engine. POST /v1/map lists a site's URLs synchronously without fetching them." status="Stable">173 <H2>How a crawl works</H2>174 <Ul>175 <Li>176 <Strong>Seed and frontier.</Strong> The seed URL is fetched first. Links are extracted, made absolute, normalised (fragments removed, tracking parameters kept) and added to the frontier if they pass the177 scope rules: <Code>same_domain</Code> / <Code>allow_subdomains</Code>, <Code>include_patterns</Code> / <Code>exclude_patterns</Code>, <Code>max_depth</Code> and, unless disabled,{" "}178 <Code>robots.txt</Code>.179 </Li>180 <Li>181 <Strong>Fetching.</Strong> Up to <Code>concurrency</Code> workers fetch pages until <Code>max_pages</Code> is reached or the frontier is empty. Each page is a normal fetch: routing intelligence,182 retries, block detection and automatic browser escalation apply, and the page appears in the request log with <Code>{`source: "crawl"`}</Code>.183 </Li>184 <Li>185 <Strong>Content.</Strong> Each page is stored in the requested <Code>format</Code> with its title, description, HTTP status, mode, bytes and duration. <Code>markdown</Code> (default) keeps the main186 content and drops navigation, footers and cookie banners when <Code>main_content</Code> is true.187 </Li>188 <Li>189 <Strong>Completion.</Strong> The job ends <Code>completed</Code> when the frontier is exhausted or the page budget is spent, <Code>failed</Code> if the seed could not be fetched, or{" "}190 <Code>cancelled</Code> if you deleted it. Pages fetched before a cancellation remain available.191 </Li>192 </Ul>193 <Callout variant="info" title="Limits">194 Up to <Strong>2,000 pages per job</Strong> and <Strong>5 concurrent jobs</Strong> per organization (a sixth returns <Code>429 CRAWL_LIMIT_REACHED</Code>). Jobs and their pages are kept for the195 log retention period (90 days).196 </Callout>197198 <H2>Start a crawl</H2>199 <Endpoint method="POST" path="/v1/crawl" scope="fetch:execute" status="Live" />200 <ParamTable201 rows={[202 { name: "url", type: "string", required: true, constraints: "1–8,192 chars, http or https", description: <>Seed URL. Same URL policy as fetch (<code>URL_NOT_ALLOWED</code> for private or internal hosts).</> },203 { name: "max_pages", type: "integer", default: "25", constraints: "1–5,000, capped at 2,000", description: <>Maximum number of pages to fetch. The seed counts as one.</> },204 { name: "max_depth", type: "integer", default: "2", constraints: "0–10", description: <>Maximum link depth from the seed. <code>0</code> fetches the seed only.</> },205 { name: "same_domain", type: "boolean", default: "true", description: <>Only follow links whose registrable domain matches the seed.</> },206 { name: "allow_subdomains", type: "boolean", default: "false", description: <>With <code>same_domain</code>, also follow links on subdomains of the seed host.</> },207 { name: "include_patterns", type: "string[]", constraints: "≤ 50 patterns, each ≤ 512 chars", description: <>Only crawl URLs matching at least one pattern. Glob with <code>*</code> (matched against the full URL or the path) or a regular expression written <code>/…/</code>.</> },208 { name: "exclude_patterns", type: "string[]", constraints: "≤ 50 patterns", description: <>Never crawl URLs matching one of these patterns. Evaluated after <code>include_patterns</code>.</> },209 { name: "respect_robots", type: "boolean", default: "true", description: <>Honour <code>Disallow</code> rules of the seed host's <code>robots.txt</code>.</> },210 { name: "use_sitemap", type: "boolean", default: "false", description: <>Also seed the frontier with URLs from <code>sitemap.xml</code> and sitemaps listed in <code>robots.txt</code> (subject to the same scope rules).</> },211 { name: "concurrency", type: "integer", default: "3", constraints: "1–10", description: <>Parallel page fetches within the job.</> },212 { name: "delay_ms", type: "integer", default: "0", constraints: "0–30,000", description: <>Pause between fetches per worker (politeness).</> },213 { name: "timeout", type: "integer (ms)", default: "30000", constraints: "1,000–120,000", description: <>Per-page timeout, including retries and browser escalation.</> },214 { name: "format", type: '"markdown" | "text" | "html"', default: '"markdown"', description: <>Format of <code>content</code> for each page.</> },215 { name: "main_content", type: "boolean", default: "true", description: <>Keep only the main content (article, main, largest text block) when converting to Markdown or text.</> },216 { name: "country", type: "string", constraints: "2 chars", description: <>Exit country for every page.</> },217 { name: "network", type: '"auto" | "datacenter" | "residential" | "isp" | "mobile"', default: '"auto"', description: <>Network class for every page.</> },218 { name: "browser", type: "boolean", default: "false", description: <>Render every page in the managed browser. Slower and heavier; prefer the default and let escalation handle JavaScript challenges.</> },219 { name: "browser_fallback", type: "boolean", default: "true", description: <>Escalate blocked pages to the browser automatically.</> },220 { name: "headers", type: "object<string, string>", constraints: "≤ 64 entries", description: <>Headers sent with every page fetch.</> },221 { name: "webhook_url", type: "string (url)", constraints: "≤ 2,048 chars", description: <>Called with a <code>POST</code> containing the job when it reaches a terminal status. Accepted and stored; delivery is part of the upcoming webhooks release.</> },222 { name: "label", type: "string", constraints: "≤ 128 chars", description: <>Free-form label shown in the dashboard.</> },223 ]}224 />225 <CodeTabs tabs={apiTabs({ method: "POST", path: "/v1/crawl", body: CREATE_BODY, after: { javascript: `console.log(data.id, data.status); // "crawl_…" "queued"`, python: `print(data["id"], data["status"]) # crawl_… queued` } })} />226 <ResponseExample status={202} statusText="Accepted" body={CREATED} />227 <P>228 The response is <Code>202 Accepted</Code>: the job is queued and the call returns immediately. Poll <Code>GET /v1/crawl/:id</Code> (or use the SDK's <Code>crawl.wait</Code>) to follow progress.229 </P>230231 <H2>Get a job</H2>232 <Endpoint method="GET" path="/v1/crawl/:id" scope={null} status="Live" />233 <CodeTabs tabs={apiTabs({ method: "GET", path: "/v1/crawl/crawl_3k9d0f2a8b1c7e4m" }, ["curl", "javascript", "python"])} />234 <ResponseExample status={200} body={JOB} />235 <ParamTable236 showDefault={false}237 rows={[238 { name: "id", type: "string", description: <>Job identifier (<code>crawl_…</code>).</> },239 { name: "status", type: '"queued" | "running" | "completed" | "failed" | "cancelled"', description: <><code>completed</code>, <code>failed</code> and <code>cancelled</code> are terminal.</> },240 { name: "label", type: "string | null", description: <>Your label.</> },241 { name: "seed_url / domain", type: "string", description: <>The seed and its host.</> },242 { name: "options", type: "object", description: <>The options as submitted, with defaults filled in.</> },243 { name: "stats", type: "object", description: <><code>{`{ discovered, fetched, ok, blocked, failed, bytes }`}</code>. <code>discovered</code> counts URLs added to the frontier; <code>fetched = ok + blocked + failed</code>.</> },244 { name: "error", type: "object | null", description: <><code>{`{ code, message }`}</code> when <code>status</code> is <code>failed</code> (for example the seed returned <code>URL_NOT_ALLOWED</code> or was blocked on every attempt).</> },245 { name: "created_at / started_at / completed_at", type: "string | null", description: <>ISO 8601 timestamps. <code>started_at</code> is <code>null</code> while queued; <code>completed_at</code> is set on any terminal status.</> },246 ]}247 />248249 <H2>List pages</H2>250 <Endpoint method="GET" path="/v1/crawl/:id/pages" scope={null} status="Live" />251 <ParamTable252 caption="Query parameters"253 rows={[254 { name: "cursor", type: "string", description: <>Opaque cursor from the previous response's <code>next_cursor</code>. Omit for the first page.</> },255 { name: "limit", type: "integer", default: "100", description: <>Pages per response.</> },256 { name: "status", type: '"success" | "blocked" | "failed"', description: <>Only return pages with this status.</> },257 ]}258 />259 <CodeTabs tabs={apiTabs({ method: "GET", path: "/v1/crawl/crawl_3k9d0f2a8b1c7e4m/pages?limit=100&status=success" }, ["curl", "javascript", "python"])} />260 <ResponseExample status={200} body={PAGES} />261 <ParamTable262 showDefault={false}263 rows={[264 { name: "id", type: "string", description: <>Page identifier.</> },265 { name: "url / final_url", type: "string", description: <>Requested URL and URL after redirects.</> },266 { name: "depth", type: "integer", description: <>Link distance from the seed (seed = 0).</> },267 { name: "status", type: '"success" | "blocked" | "failed"', description: <>Outcome of the fetch. <code>blocked</code> means every attempt, including browser escalation, was classified as a block.</> },268 { name: "http_status", type: "integer | null", description: <>Status of the final attempt.</> },269 { name: "error_code", type: "string | null", description: <>Fetch error code for failed pages (<code>TARGET_TIMEOUT</code>, <code>TARGET_UNAVAILABLE</code>, …).</> },270 { name: "title / description", type: "string | null", description: <>From the page's <code><title></code> and meta description.</> },271 { name: "content_type", type: "string | null", description: <>The target's Content-Type.</> },272 { name: "content", type: "string | null", description: <>Page content in the job's <code>format</code>. <code>null</code> for non-HTML bodies and blocked pages.</> },273 { name: "links_count", type: "integer | null", description: <>Hyperlinks found in the page.</> },274 { name: "bytes / duration_ms", type: "integer | null", description: <>Bytes transferred and wall-clock time for this page, all attempts included.</> },275 { name: "mode", type: '"http" | "browser" | null', description: <>How the final attempt was made.</> },276 { name: "fetched_at", type: "string | null", description: <>ISO 8601 timestamp.</> },277 ]}278 />279 <P>280 Pages are ordered by <Code>fetched_at</Code>. <Code>next_cursor</Code> is <Code>null</Code> on the last page. While a job is running, new pages appear at the end; keep the last cursor to fetch only281 what is new.282 </P>283284 <H2>Cancel a job</H2>285 <Endpoint method="DELETE" path="/v1/crawl/:id" scope="fetch:execute" status="Live" />286 <CodeTabs tabs={apiTabs({ method: "DELETE", path: "/v1/crawl/crawl_3k9d0f2a8b1c7e4m" }, ["curl", "javascript", "python"])} />287 <ResponseExample status={200} body={{ id: "crawl_3k9d0f2a8b1c7e4m", status: "cancelled" }} />288 <P>289 Cancellation is immediate for queued jobs; running jobs stop after the in-flight pages finish. Pages already fetched stay available. Cancelling a job that is already terminal returns it unchanged.290 </P>291292 <H2>List jobs</H2>293 <Endpoint method="GET" path="/v1/crawl" scope={null} status="Live" />294 <P>295 Returns the most recent jobs of the project, newest first. <Code>?limit=</Code> defaults to 50.296 </P>297 <CodeTabs tabs={apiTabs({ method: "GET", path: "/v1/crawl?limit=50" }, ["curl", "javascript", "python"])} />298 <ResponseExample status={200} body={{ data: [JOB] }} />299 <Table dense>300 <THead>301 <Tr>302 <Th>Condition</Th>303 <Th>Result</Th>304 </Tr>305 </THead>306 <TBody>307 <Tr>308 <Td>Invalid body (unknown field, pattern too long, out-of-range value)</Td>309 <Td mono>400 INVALID_REQUEST</Td>310 </Tr>311 <Tr>312 <Td>Seed URL private, internal or non-http</Td>313 <Td mono>400 URL_NOT_ALLOWED</Td>314 </Tr>315 <Tr>316 <Td>Id does not exist or belongs to another project</Td>317 <Td mono>404 CRAWL_NOT_FOUND</Td>318 </Tr>319 <Tr>320 <Td>Five jobs already queued or running</Td>321 <Td mono>429 CRAWL_LIMIT_REACHED</Td>322 </Tr>323 </TBody>324 </Table>325326 <H2>Polling example</H2>327 <P>328 The SDKs wrap the four calls and add <Code>crawl.wait</Code>, which polls until a terminal status. Both examples start a job, wait for it, then stream the pages with the cursor.329 </P>330 <CodeBlock lang="typescript" title="crawl.ts" code={POLL_JS} />331 <CodeBlock lang="python" title="crawl.py" code={POLL_PY} />332 <CodeBlock lang="bash" title="Terminal" code={POLL_RAW} />333334 <H2>Map a site</H2>335 <Endpoint method="POST" path="/v1/map" scope="fetch:execute" status="Live" />336 <P>337 <Code>map</Code> answers the question “which URLs does this site have?” without fetching every page. It reads the sitemap(s) and the links of the seed page, filters them, and returns a338 de-duplicated list, <Strong>synchronously</Strong> (the call takes up to 60 s). Use it to pick <Code>include_patterns</Code> before a crawl, or to feed URLs into your own fetch loop.339 </P>340 <ParamTable341 rows={[342 { name: "url", type: "string", required: true, description: <>Site or page to map.</> },343 { name: "limit", type: "integer", default: "1000", constraints: "1–10,000", description: <>Maximum number of URLs returned. <code>truncated</code> is <code>true</code> when more were found.</> },344 { name: "use_sitemap", type: "boolean", default: "true", description: <>Read <code>sitemap.xml</code>, sitemap indexes and sitemaps listed in <code>robots.txt</code>.</> },345 { name: "use_links", type: "boolean", default: "true", description: <>Include hyperlinks found on the seed page.</> },346 { name: "same_domain", type: "boolean", default: "true", description: <>Drop URLs outside the seed's registrable domain.</> },347 { name: "allow_subdomains", type: "boolean", default: "false", description: <>Keep subdomains of the seed host.</> },348 { name: "search", type: "string", constraints: "≤ 256 chars", description: <>Keep only URLs matching this substring, glob (<code>*</code>) or <code>/regex/</code>.</> },349 { name: "country / network / timeout", type: "—", description: <>Same meaning as in fetch; apply to the sitemap and seed-page requests.</> },350 ]}351 />352 <CodeTabs tabs={apiTabs({ method: "POST", path: "/v1/map", body: { url: "https://docs.example.com/", search: "/docs/*", limit: 500 }, after: { javascript: `console.log(data.count, data.urls.slice(0, 3));`, python: `print(data["count"], data["urls"][:3])` } })} />353 <ResponseExample status={200} body={MAP_RESULT} />354 <ParamTable355 showDefault={false}356 rows={[357 { name: "url", type: "string", description: <>The seed you passed.</> },358 { name: "count", type: "integer", description: <>Number of URLs in <code>urls</code>.</> },359 { name: "urls", type: "string[]", description: <>Absolute, de-duplicated URLs; sitemap entries first, then links in document order.</> },360 { name: "sources", type: "object", description: <><code>{`{ sitemap, links }`}</code>: how many candidates each source contributed before filtering and de-duplication.</> },361 { name: "truncated", type: "boolean", description: <><code>true</code> when the result was cut at <code>limit</code>.</> },362 ]}363 />364 <H3>Map, then crawl</H3>365 <CodeBlock366 lang="typescript"367 code={`const map = await fetcha.map({ url: "https://docs.example.com/", search: "/docs/api/*" });368console.log(map.count, "API pages");369370// Crawl exactly that section371const job = await fetcha.crawl.create({ url: "https://docs.example.com/docs/api/", include_patterns: ["/docs/api/*"], max_pages: map.count });372const done = await fetcha.crawl.wait(job.id);`}373 />374 <P>375 A map costs a handful of fetches (one per sitemap file plus the seed page) and does not create a job. See the <A href="/docs/sdks">SDKs</A> page for the full client reference.376 </P>377 </DocPage>378 );379}380