TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import type { Metadata } from "next";2import { PLAN_LIMITS } from "@fetcha/core";3import { CodeBlock } from "@/components/ui/code-block";4import { DocPage } from "@/components/docs/doc-page";5import { A, Code, H2, H3, Li, P, Strong, Table, TBody, Td, Th, THead, Tr, Ul } from "@/components/docs/prose";6import { Callout } from "@/components/docs/callout";7import { CodeTabs } from "@/components/docs/code-tabs";8import { ResponseExample } from "@/components/docs/response-example";9import { fetchTabs } from "@/components/docs/snippets";1011export const metadata: Metadata = {12 title: "Retries",13 description: "How Fetcha retries and escalates: what triggers a retry, the retries option and its maximum, the shared timeout, attempt metadata, bandwidth implications and idempotency.",14};1516const L = PLAN_LIMITS.unlimited;1718export default function RetriesPage() {19 return (20 <DocPage path="/docs/retries" eyebrow="Reliability" title="Retries" description="Retries are built into every fetch. When a target blocks or a route fails, Fetcha tries again with a new exit identity and, when it helps, a different route, all within a single request and a single timeout." status="Stable">21 <H2>How a retry works</H2>22 <P>23 Before executing, the routing engine produces an ordered list of candidate routes for the request (see <A href="/docs/networks">Network selection</A>). Fetcha walks that list:24 </P>25 <Ul>26 <Li>27 <Strong>New IP.</Strong> Each attempt asks the network for a fresh exit identity, so a second attempt on the same route already comes from a different address. The exception is a request28 pinned to a <A href="/docs/sessions">session</A>, which deliberately keeps its identity.29 </Li>30 <Li>31 <Strong>Alternate route.</Strong> Distinct routes are preferred before repeating one. If several routes serve the same class, the second attempt typically goes through another one.32 </Li>33 <Li>34 <Strong>Escalation.</Strong> In <Code>auto</Code> mode, routes are ordered by class from cheapest to most reliable, so retries climb toward premium classes as they launch. With a single35 live class today, escalation happens between routes inside that class.36 </Li>37 <Li>38 <Strong>Learning.</Strong> Every attempt, successful or not, updates the domain profile and the route's circuit breaker.39 </Li>40 </Ul>4142 <H3>What triggers a retry</H3>43 <Table dense>44 <THead>45 <Tr>46 <Th>Outcome of an attempt</Th>47 <Th>Retried?</Th>48 <Th>If all attempts fail</Th>49 </Tr>50 </THead>51 <TBody>52 <Tr>53 <Td>Block page: HTTP 403, 407, 429 or 999; 503 challenge; captcha, anti-bot or WAF signatures</Td>54 <Td className="text-success">Yes</Td>55 <Td>56 <Code>200</Code> with <Code>success: false</Code> and the last blocked page57 </Td>58 </Tr>59 <Tr>60 <Td>Timeout waiting for the target</Td>61 <Td className="text-success">Yes, while budget remains</Td>62 <Td mono>504 TARGET_TIMEOUT</Td>63 </Tr>64 <Tr>65 <Td>Connection failure to the target</Td>66 <Td className="text-success">Yes</Td>67 <Td mono>502 TARGET_UNAVAILABLE</Td>68 </Tr>69 <Tr>70 <Td>Network-side failure (gateway error, upstream auth, 5xx from the route)</Td>71 <Td className="text-success">Yes</Td>72 <Td mono>503 PROVIDER_UNAVAILABLE</Td>73 </Tr>74 <Tr>75 <Td>2xx/3xx response that is not a block</Td>76 <Td>No, returned immediately</Td>77 <Td>—</Td>78 </Tr>79 <Tr>80 <Td>4xx/5xx response that is not a block (plain 404, 500, …)</Td>81 <Td>No, returned immediately with <Code>success: false</Code></Td>82 <Td>—</Td>83 </Tr>84 <Tr>85 <Td>Response larger than the size cap</Td>86 <Td className="text-danger">No</Td>87 <Td mono>502 RESPONSE_TOO_LARGE</Td>88 </Tr>89 <Tr>90 <Td>91 More than <Code>max_redirects</Code> hops92 </Td>93 <Td className="text-danger">No</Td>94 <Td mono>502 TOO_MANY_REDIRECTS</Td>95 </Tr>96 <Tr>97 <Td>Redirect to a private or internal host</Td>98 <Td className="text-danger">No</Td>99 <Td mono>400 URL_NOT_ALLOWED</Td>100 </Tr>101 </TBody>102 </Table>103104 <H2>The retries option</H2>105 <P>106 <Code>retries</Code> is the number of <Strong>additional</Strong> attempts after the first; total attempts are <Code>retries + 1</Code>. Omit it to use the maximum. The schema accepts 0 to{" "}107 {L.max_retries}; the same maximum applies to every organization (single plan).108 </P>109 <Table>110 <THead>111 <Tr>112 <Th>Limit</Th>113 <Th>Value</Th>114 </Tr>115 </THead>116 <TBody>117 <Tr>118 <Td className="font-medium text-fg">Max retries</Td>119 <Td mono>{L.max_retries}</Td>120 </Tr>121 <Tr>122 <Td className="font-medium text-fg">Max attempts</Td>123 <Td mono>{L.max_retries + 1}</Td>124 </Tr>125 <Tr>126 <Td className="font-medium text-fg">Max timeout</Td>127 <Td mono>{L.max_timeout_ms / 1000} s</Td>128 </Tr>129 <Tr>130 <Td className="font-medium text-fg">Browser fallback</Td>131 <Td>132 On by default (<Code>browser_fallback: true</Code>): a JavaScript challenge on an HTTP attempt escalates to the managed browser within the same budget.133 </Td>134 </Tr>135 </TBody>136 </Table>137 <P>138 The candidate list is also bounded by how many distinct routes exist for the request. When there are fewer routes than attempts, the engine repeats the best route (with a new IP each time),139 so <Code>metadata.attempts</Code> may come out lower than <Code>retries + 1</Code> even on a fully blocked target.140 </P>141 <CodeTabs tabs={fetchTabs({ url: "https://shop.example.net/", retries: 0 }, { javascript: `console.log(data.metadata.attempts); // always 1 with retries: 0`, python: `print(data["metadata"]["attempts"]) # always 1 with retries: 0` }, ["curl", "javascript", "python"])} />142143 <H2>One timeout for everything</H2>144 <P>145 <Code>timeout</Code> is the budget for the <Strong>whole</Strong> request, not per attempt. Each attempt receives what is left; when fewer than 500 ms remain, Fetcha stops and returns{" "}146 <Code>TARGET_TIMEOUT</Code>. If you rely on several attempts against slow targets, size <Code>timeout</Code> accordingly (up to {L.max_timeout_ms / 1000} s) rather than raising <Code>retries</Code>.147 </P>148149 <H2>Attempts in metadata</H2>150 <P>151 <Code>metadata.attempts</Code> counts every attempt including the final one. <Code>metadata.duration_ms</Code> and <Code>metadata.bytes</Code> cover all of them. With <Code>debug: true</Code>,{" "}152 <Code>metadata.debug.attempts</Code> lists each attempt with the route alias, network, country, outcome, target status and duration.153 </P>154 <ResponseExample155 status={200}156 title="200 OK · metadata after one block and one success"157 body={{158 metadata: {159 network: "residential",160 country: "US",161 attempts: 2,162 duration_ms: 3140,163 bytes: 71822,164 session: null,165 cached: false,166 timing: { dns_ms: 17, proxy_connect_ms: 0, tls_ms: 0, origin_ms: 1402, processing_ms: 9, total_ms: 3140 },167 debug: {168 attempts: [169 { provider: "network-a", network: "residential", country: "US", outcome: "blocked", status: 403, duration_ms: 1210 },170 { provider: "network-b", network: "residential", country: "US", outcome: "success", status: 200, duration_ms: 1888 },171 ],172 },173 },174 }}175 />176177 <H2>Usage implications</H2>178 <P>Fetcha has no billing, but retries still show up in your usage figures and in the internal cost estimate behind optional spending limits:</P>179 <Ul>180 <Li>181 <Strong>Bandwidth is counted for every attempt.</Strong> <Code>metadata.bytes</Code> sums request and response bytes across attempts. A block page is usually small, but a large page that182 gets blocked after being fully downloaded counts its full size.183 </Li>184 <Li>185 <Strong>Every request counts in your usage</Strong>, including failed and blocked ones, but there is no monthly quota; see <A href="/docs/rate-limits">Rate limits</A>.186 </Li>187 <Li>188 Requests refused before any attempt (validation, limits, <Code>URL_NOT_ALLOWED</Code>, session errors) transfer no bytes and are not counted.189 </Li>190 </Ul>191192 <H2>Idempotency for POST and other writes</H2>193 <Callout variant="warning" title="Retries replay the request">194 A retry sends the <Strong>same method and body</Strong> again. If your <Code>POST</Code> creates an order, sends a message or otherwise has side effects on the target, an attempt that was195 processed by the origin but answered with something Fetcha classifies as a block (a 429 for instance) would be replayed.196 </Callout>197 <Ul>198 <Li>199 Set <Code>{`"retries": 0`}</Code> for non-idempotent writes and implement your own retry decision based on <Code>status</Code>.200 </Li>201 <Li>Where the target supports it, include an idempotency token in your body or headers so replays are harmless.</Li>202 <Li>203 Remember that a 301/302 after a POST, and any 303, is followed as a <Code>GET</Code> without the body (standard browser behaviour).204 </Li>205 <Li>206 For <Code>POST /v1/sessions</Code>, use the <Code>Idempotency-Key</Code> header to make session creation safe to retry on your side.207 </Li>208 </Ul>209210 <H2>Retrying on your side</H2>211 <P>212 Fetcha already retries target-side problems. Client-side retries are still appropriate for <Code>RATE_LIMITED</Code> (honour <Code>Retry-After</Code>), <Code>CONCURRENCY_LIMIT</Code> (when a213 slot frees up), <Code>PROVIDER_UNAVAILABLE</Code> and <Code>INTERNAL_ERROR</Code> (exponential backoff), and for a <Code>success: false</Code> block you may want to retry later with a different214 country or a fresh session.215 </P>216 <CodeBlock217 lang="typescript"218 code={`async function fetchWithBackoff(opts: FetchOptions, maxTries = 4) {219 for (let i = 0; ; i++) {220 try {221 return await fetcha.fetch(opts);222 } catch (e) {223 const transient = e instanceof FetchaError && ["RATE_LIMITED", "CONCURRENCY_LIMIT", "PROVIDER_UNAVAILABLE", "INTERNAL_ERROR"].includes(e.code);224 if (!transient || i + 1 >= maxTries) throw e;225 await new Promise((r) => setTimeout(r, Math.min(8000, 500 * 2 ** i) + Math.random() * 250));226 }227 }228}`}229 />230 </DocPage>231 );232}233