TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import type { Metadata } from "next";2import { DocPage } from "@/components/docs/doc-page";3import { A, Code, H2, Li, P, Strong, Ul } from "@/components/docs/prose";4import { Callout } from "@/components/docs/callout";5import { CodeTabs } from "@/components/docs/code-tabs";67export const metadata: Metadata = {8 title: "Examples",9 description: "Eight practical recipes: text extraction, JSON APIs through a Canadian residential IP, sticky login flows, POST bodies, headers and cookies, handling blocks, concurrency and debugging.",10};1112const R1_JS = `import { Fetcha } from "@fetcha/sdk";13const fetcha = new Fetcha({ apiKey: process.env.FETCHA_API_KEY! });1415const r = await fetcha.fetch({16 url: "https://www.example.ca/products/42",17 country: "CA",18 format: "text",19});2021if (!r.success) throw new Error(\`target answered \${r.status} after \${r.metadata.attempts} attempts\`);22console.log(r.text); // readable text: scripts, styles and tags removed23console.log(r.headers["last-modified"]);`;2425const R1_PY = `from fetcha import Fetcha26client = Fetcha(api_key=os.environ["FETCHA_API_KEY"])2728r = client.fetch("https://www.example.ca/products/42", country="CA", format="text")2930if not r.success:31 raise RuntimeError(f"target answered {r.status} after {r.metadata['attempts']} attempts")32print(r.text) # readable text33print(r.headers.get("last-modified"))`;3435const R2_JS = `type Listing = { id: string; price: number; currency: string };3637const data = await fetcha.json<{ items: Listing[] }>(38 "https://api.example.ca/v2/listings?city=montreal&page=1",39 { country: "CA", network: "residential", headers: { Accept: "application/json" } },40);4142for (const item of data.items) console.log(item.id, item.price, item.currency);`;4344const R2_PY = `data = client.json(45 "https://api.example.ca/v2/listings?city=montreal&page=1",46 country="CA",47 network="residential",48 headers={"Accept": "application/json"},49)50for item in data["items"]:51 print(item["id"], item["price"], item["currency"])`;5253const R3_JS = `// 1. One session for the whole flow (15 minutes, Canadian exit)54const session = await fetcha.sessions.create({ country: "CA", ttl: 900, label: "login:user-42" }, "login:user-42");5556// 2. Log in. No retries: a replayed login POST could lock the account.57const login = await fetcha.fetch({58 url: "https://www.example.ca/login",59 method: "POST",60 session: session.id,61 headers: { "Content-Type": "application/x-www-form-urlencoded" },62 body: "email=me%40example.com&password=secret",63 retries: 0,64});65if (!login.success) throw new Error(\`login failed with \${login.status}\`);6667// 3. Carry the cookies forward and read the protected page with the same identity68const jar = Object.fromEntries(login.cookies.map((c) => [c.name, c.value]));69const account = await fetcha.fetch({70 url: "https://www.example.ca/account",71 session: session.id,72 cookies: jar,73 format: "text",74});75console.log(account.text);7677await fetcha.sessions.close(session.id);`;7879const R3_PY = `# 1. One session for the whole flow (15 minutes, Canadian exit)80session = client.sessions.create(country="CA", ttl=900, label="login:user-42")8182# 2. Log in. No retries: a replayed login POST could lock the account.83login = client.fetch(84 "https://www.example.ca/login",85 method="POST",86 session=session.id,87 headers={"Content-Type": "application/x-www-form-urlencoded"},88 body="email=me%40example.com&password=secret",89 retries=0,90)91if not login.success:92 raise RuntimeError(f"login failed with {login.status}")9394# 3. Carry the cookies forward and read the protected page with the same identity95jar = {c["name"]: c["value"] for c in login.cookies}96account = client.fetch("https://www.example.ca/account", session=session.id, cookies=jar, format="text")97print(account.text)9899client.sessions.close(session.id)`;100101const R4_JS = `const r = await fetcha.fetch({102 url: "https://api.example.com/v1/search",103 method: "POST",104 // Objects are JSON-serialised; Content-Type defaults to application/json105 body: { query: "standing desk", filters: { in_stock: true, max_price: 500 }, page: 1 },106 format: "json",107 retries: 0, // POST: decide about retries yourself108});109110if (r.success && r.json !== undefined) {111 console.log((r.json as { total: number }).total);112} else {113 console.log("status", r.status, "body", r.content?.slice(0, 300));114}`;115116const R4_PY = `r = client.fetch(117 "https://api.example.com/v1/search",118 method="POST",119 # dicts are JSON-serialised; Content-Type defaults to application/json120 body={"query": "standing desk", "filters": {"in_stock": True, "max_price": 500}, "page": 1},121 format="json",122 retries=0, # POST: decide about retries yourself123)124125if r.success and r.json is not None:126 print(r.json["total"])127else:128 print("status", r.status, "body", (r.content or "")[:300])`;129130const R5_JS = `const r = await fetcha.fetch({131 url: "https://www.example.com/pricing",132 headers: {133 // Overrides Fetcha's default desktop Chrome UA134 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36",135 Referer: "https://www.google.com/",136 "X-Requested-With": "XMLHttpRequest",137 },138 // Sent as a single Cookie header, appended to any Cookie header above139 cookies: { currency: "CAD", consent: "1" },140 locale: "fr-CA", // Accept-Language141 device: "mobile", // iPhone UA unless you set User-Agent yourself (as above)142});143144console.log(r.status, r.cookies); // cookies the target set in return`;145146const R5_PY = `r = client.fetch(147 "https://www.example.com/pricing",148 headers={149 # Overrides Fetcha's default desktop Chrome UA150 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36",151 "Referer": "https://www.google.com/",152 "X-Requested-With": "XMLHttpRequest",153 },154 # Sent as a single Cookie header, appended to any Cookie header above155 cookies={"currency": "CAD", "consent": "1"},156 locale="fr-CA", # Accept-Language157 device="mobile", # iPhone UA unless you set User-Agent yourself (as above)158)159print(r.status, r.cookies) # cookies the target set in return`;160161const R6_JS = `import { Fetcha, FetchaError } from "@fetcha/sdk";162163async function fetchOrExplain(url: string) {164 try {165 const r = await fetcha.fetch({ url, country: "US", debug: true });166167 if (r.success) return r;168169 // Fetcha reached the target but every attempt was blocked (or the origin returned an error).170 const blocked = r.metadata.attempts > 1 || [403, 429].includes(r.status);171 if (blocked) {172 console.warn(\`blocked: status \${r.status} after \${r.metadata.attempts} attempts\`, r.metadata.debug);173 // Options: retry later, try another country, or a fresh session.174 return await fetcha.fetch({ url, country: "CA" });175 }176 console.warn(\`origin error \${r.status}\`); // plain 404/500: not a block, not retried177 return r;178 } catch (e) {179 if (e instanceof FetchaError && e.code === "TARGET_TIMEOUT") {180 console.warn("timed out across all attempts", e.requestId);181 return null;182 }183 throw e;184 }185}`;186187const R6_PY = `from fetcha import FetchaError188189def fetch_or_explain(url: str):190 try:191 r = client.fetch(url, country="US", debug=True)192 except FetchaError as e:193 if e.code == "TARGET_TIMEOUT":194 print("timed out across all attempts", e.request_id)195 return None196 raise197198 if r.success:199 return r200201 # Fetcha reached the target but every attempt was blocked (or the origin returned an error).202 blocked = r.metadata["attempts"] > 1 or r.status in (403, 429)203 if blocked:204 print(f"blocked: status {r.status} after {r.metadata['attempts']} attempts", r.metadata.get("debug"))205 # Options: retry later, try another country, or a fresh session.206 return client.fetch(url, country="CA")207 print(f"origin error {r.status}") # plain 404/500: not a block, not retried208 return r`;209210const R7_JS = `// Plan concurrency: Free 5, Developer 25, Growth 100, Business 500.211// Read it once from the API instead of hardcoding it.212const usage = (await fetcha.usage.current()) as { plan: { concurrency: number } };213const limit = Math.max(1, usage.plan.concurrency - 1); // leave one slot for other workers214215async function mapWithConcurrency<T, R>(items: T[], n: number, fn: (t: T) => Promise<R>): Promise<R[]> {216 const out: R[] = new Array(items.length);217 let i = 0;218 await Promise.all(219 Array.from({ length: n }, async () => {220 while (i < items.length) {221 const idx = i++;222 out[idx] = await fn(items[idx]!);223 }224 }),225 );226 return out;227}228229const urls = ["https://example.com/1", "https://example.com/2" /* … */];230const results = await mapWithConcurrency(urls, limit, async (url) => {231 try {232 return await fetcha.fetch({ url, country: "CA", format: "text" });233 } catch (e) {234 if (e instanceof FetchaError && e.code === "CONCURRENCY_LIMIT") {235 // Another process is using slots: wait briefly and try once more236 await new Promise((r) => setTimeout(r, 500));237 return await fetcha.fetch({ url, country: "CA", format: "text" });238 }239 throw e;240 }241});`;242243const R7_PY = `from concurrent.futures import ThreadPoolExecutor244import time245from fetcha import FetchaError246247# Plan concurrency: Free 5, Developer 25, Growth 100, Business 500.248limit = max(1, client.usage()["plan"]["concurrency"] - 1) # leave one slot for other workers249250def one(url: str):251 for attempt in range(2):252 try:253 return client.fetch(url, country="CA", format="text")254 except FetchaError as e:255 if e.code == "CONCURRENCY_LIMIT" and attempt == 0:256 time.sleep(0.5) # another process is using slots257 continue258 raise259260urls = ["https://example.com/1", "https://example.com/2"] # …261with ThreadPoolExecutor(max_workers=limit) as pool:262 results = list(pool.map(one, urls))`;263264const R8_JS = `const r = await fetcha.fetch({ url: "https://www.example.com/", country: "US", debug: true });265266const m = r.metadata;267console.table({268 network: m.network, // class that served the final attempt269 country: m.country,270 attempts: m.attempts, // > 1 means a retry or escalation happened271 duration_ms: m.duration_ms, // whole request, all attempts272 bytes: m.bytes, // all attempts; basis for bandwidth pricing273 origin_ms: m.timing?.origin_ms, // time to first byte on the final attempt274 processing_ms: m.timing?.processing_ms,275});276277// One line per attempt: route alias, outcome, target status, duration278for (const a of (m.debug as { attempts: Array<Record<string, unknown>> } | undefined)?.attempts ?? []) {279 console.log(a.provider, a.network, a.outcome, a.status, \`\${a.duration_ms}ms\`);280}281// e.g. network-a residential blocked 403 1210ms282// network-b residential success 200 1888ms283284console.log("request id for support:", r.request_id);`;285286const R8_PY = `r = client.fetch("https://www.example.com/", country="US", debug=True)287288m = r.metadata289print({290 "network": m["network"], # class that served the final attempt291 "country": m["country"],292 "attempts": m["attempts"], # > 1 means a retry or escalation happened293 "duration_ms": m["duration_ms"], # whole request, all attempts294 "bytes": m["bytes"], # all attempts; basis for bandwidth pricing295 "origin_ms": m["timing"]["origin_ms"],296 "processing_ms": m["timing"]["processing_ms"],297})298299# One line per attempt: route alias, outcome, target status, duration300for a in m.get("debug", {}).get("attempts", []):301 print(a["provider"], a["network"], a["outcome"], a["status"], f"{a['duration_ms']}ms")302# e.g. network-a residential blocked 403 1210ms303# network-b residential success 200 1888ms304305print("request id for support:", r.request_id)`;306307function Recipe({ title, id, intro, js, py, children }: { title: string; id: string; intro: React.ReactNode; js: string; py: string; children?: React.ReactNode }) {308 return (309 <section>310 <H2 id={id}>{title}</H2>311 <P>{intro}</P>312 <CodeTabs313 tabs={[314 { label: "JavaScript", lang: "typescript", code: js },315 { label: "Python", lang: "python", code: py },316 ]}317 />318 {children}319 </section>320 );321}322323export default function ExamplesPage() {324 return (325 <DocPage path="/docs/examples" eyebrow="Tools" title="Examples" description="Practical recipes using the JavaScript and Python SDKs. Each one is complete: copy it, set FETCHA_API_KEY and run." status="Stable">326 <P>327 The snippets assume a client created as in the <A href="/docs/sdks">SDKs</A> page: <Code>const fetcha = new Fetcha({"{ apiKey }"})</Code> or <Code>client = Fetcha(api_key=…)</Code>. The328 SDKs are installed from source for now.329 </P>330331 <Recipe332 id="fetch-a-product-page-as-text"333 title="1. Fetch a product page as text"334 intro={335 <>336 The most common call. <Code>{`format: "text"`}</Code> strips scripts, styles and markup and returns readable text in <Code>text</Code>, which is what you want for indexing, summarising or337 feeding a model.338 </>339 }340 js={R1_JS}341 py={R1_PY}342 />343344 <Recipe345 id="json-api-through-a-canadian-residential-ip"346 title="2. JSON API through a Canadian residential IP"347 intro={348 <>349 Many sites load their data from a JSON endpoint. Call it directly with <Code>{`format: "json"`}</Code> and an explicit <Code>country</Code>; the <Code>json</Code> helper returns the parsed350 value. Pinning <Code>{`network: "residential"`}</Code> is optional today since <Code>auto</Code> resolves to it, but makes the intent explicit.351 </>352 }353 js={R2_JS}354 py={R2_PY}355 >356 <Callout variant="info">357 If the endpoint returns something that is not JSON (an HTML error page, for instance), <Code>json</Code> is <Code>undefined</Code>/<Code>None</Code>. Use <Code>fetch</Code> instead of{" "}358 <Code>json</Code> when you need <Code>status</Code> and <Code>content</Code> to diagnose it.359 </Callout>360 </Recipe>361362 <Recipe363 id="sticky-session-login-flow"364 title="3. Sticky session login flow"365 intro={366 <>367 Two requests that must look like one visitor: a POST to log in, then a GET of a protected page. A <A href="/docs/sessions">session</A> keeps the exit IP stable; the cookies from the368 first response are replayed in the second. The login uses <Code>retries: 0</Code> because replaying a credential POST is rarely what you want.369 </>370 }371 js={R3_JS}372 py={R3_PY}373 />374375 <Recipe376 id="post-with-a-json-body"377 title="4. POST with a JSON body"378 intro={379 <>380 Pass an object as <Code>body</Code> and Fetcha serialises it as JSON with <Code>Content-Type: application/json</Code>. Pass a string to send anything else (form-encoded, XML, GraphQL) and381 set the content type yourself. Bodies are ignored for <Code>GET</Code> and <Code>HEAD</Code>.382 </>383 }384 js={R4_JS}385 py={R4_PY}386 />387388 <Recipe389 id="custom-headers-and-cookies"390 title="5. Custom headers and cookies"391 intro={392 <>393 Your <Code>headers</Code> override Fetcha's defaults one by one. <Code>cookies</Code> is a convenience over the <Code>Cookie</Code> header, <Code>locale</Code> sets{" "}394 <Code>Accept-Language</Code>, and <Code>{`device: "mobile"`}</Code> switches to an iPhone User-Agent unless you provide your own.395 </>396 }397 js={R5_JS}398 py={R5_PY}399 />400401 <Recipe402 id="handling-target-blocked"403 title="6. Handling TARGET_BLOCKED"404 intro={405 <>406 A fully blocked target is <Strong>not</Strong> an exception: Fetcha returns <Code>200</Code> with <Code>success: false</Code>, the last blocked page and <Code>metadata.attempts</Code>{" "}407 greater than one (the request log records it as <Code>TARGET_BLOCKED</Code>). Distinguish it from a plain origin error, then decide: retry later, change geography or open a fresh session.408 </>409 }410 js={R6_JS}411 py={R6_PY}412 />413414 <Recipe415 id="concurrency-within-plan-limits"416 title="7. Concurrency within plan limits"417 intro={418 <>419 Concurrency is enforced per organization (5 / 25 / 100 / 500 in-flight requests depending on plan). Run a worker pool sized just below the limit instead of firing everything with a bare{" "}420 <Code>Promise.all</Code>, and treat <Code>CONCURRENCY_LIMIT</Code> as “wait for a slot” rather than a failure. See <A href="/docs/rate-limits">Rate limits</A>.421 </>422 }423 js={R7_JS}424 py={R7_PY}425 >426 <Ul>427 <Li>The plan concurrency comes from <Code>GET /v1/usage</Code>, so the pool adapts when you upgrade.</Li>428 <Li>429 If several services share one organization, split the budget between them or centralise fetching behind a queue.430 </Li>431 </Ul>432 </Recipe>433434 <Recipe435 id="using-debug-metadata"436 title="8. Using debug metadata"437 intro={438 <>439 <Code>debug: true</Code> adds a per-attempt trace to <Code>metadata.debug.attempts</Code>. Combined with <Code>timing</Code>, <Code>bytes</Code> and <Code>attempts</Code> it explains440 why a request was slow or expensive, and <Code>request_id</Code> lets support see the same trace on their side.441 </>442 }443 js={R8_JS}444 py={R8_PY}445 />446 </DocPage>447 );448}449