TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import { z } from "zod";23/** Network classes exposed to customers. Providers are never exposed. */4export const NETWORK_CLASSES = ["auto", "datacenter", "residential", "isp", "mobile"] as const;5export type NetworkClass = (typeof NETWORK_CLASSES)[number];6/** A concrete network class (never `auto`). */7export type ConcreteNetwork = Exclude<NetworkClass, "auto">;89export const HTTP_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"] as const;10export type HttpMethod = (typeof HTTP_METHODS)[number];1112export const OUTPUT_FORMATS = ["html", "text", "markdown", "json", "raw"] as const;13export type OutputFormat = (typeof OUTPUT_FORMATS)[number];1415export const DEVICES = ["desktop", "mobile", "tablet"] as const;1617const headerRecord = z.record(z.string().max(256), z.string().max(8192)).refine(18 (h) => Object.keys(h).length <= 64,19 "Too many headers (max 64)",20);2122export const fetchRequestSchema = z23 .object({24 url: z.string().min(1).max(8192),25 method: z.enum(HTTP_METHODS).default("GET"),26 headers: headerRecord.optional(),27 cookies: z.record(z.string().max(256), z.string().max(4096)).optional(),28 body: z.union([z.string().max(2_000_000), z.record(z.string(), z.unknown())]).optional(),29 timeout: z.number().int().min(1000).max(120_000).default(30_000),30 country: z31 .string()32 .length(2)33 .transform((s) => s.toUpperCase())34 .optional(),35 region: z.string().max(64).optional(),36 city: z.string().max(128).optional(),37 network: z.enum(NETWORK_CLASSES).default("auto"),38 session: z.string().max(64).optional(),39 /** Render in a managed headless browser (real Chromium) routed through the same network. */40 browser: z.boolean().default(false),41 /** When an HTTP attempt is blocked by a JavaScript challenge / anti-bot, automatically escalate to the browser. */42 browser_fallback: z.boolean().default(true),43 javascript: z.boolean().optional(),44 wait_for: z.string().max(512).optional(),45 wait_ms: z.number().int().min(0).max(30_000).optional(),46 wait_until: z.enum(["load", "domcontentloaded", "networkidle"]).default("domcontentloaded"),47 /** Browser: skip images, fonts and media to save bandwidth (default true). */48 block_resources: z.boolean().default(true),49 /** Browser: return a PNG screenshot (base64) in `screenshot`. */50 screenshot: z.boolean().default(false),51 /** Browser: use the captcha solver on Cloudflare Turnstile challenges when the platform has one configured (default true). */52 solve_captcha: z.boolean().default(true),53 /** Include the list of hyperlinks found in the page (`links`). */54 links: z.boolean().default(false),55 /** Referer strategy: "auto" (none on first try, search-engine referer on retries), "none", or a literal URL. */56 referer: z.union([z.enum(["auto", "none"]), z.string().url().max(2048)]).default("auto"),57 device: z.enum(DEVICES).optional(),58 locale: z.string().max(16).optional(),59 format: z.enum(OUTPUT_FORMATS).default("html"),60 follow_redirects: z.boolean().default(true),61 max_redirects: z.number().int().min(0).max(20).default(10),62 max_response_bytes: z.number().int().min(1024).max(50_000_000).optional(),63 cache: z64 .object({65 enabled: z.boolean().default(false),66 ttl: z.number().int().min(1).max(86_400).default(300),67 })68 .optional(),69 retries: z.number().int().min(0).max(5).optional(),70 debug: z.boolean().default(false),71 })72 .strict();7374export type FetchRequestInput = z.input<typeof fetchRequestSchema>;75export type FetchRequest = z.output<typeof fetchRequestSchema>;7677export const sessionCreateSchema = z78 .object({79 country: z80 .string()81 .length(2)82 .transform((s) => s.toUpperCase())83 .optional(),84 region: z.string().max(64).optional(),85 city: z.string().max(128).optional(),86 network: z.enum(NETWORK_CLASSES).default("auto"),87 ttl: z.number().int().min(60).max(1800).default(600),88 label: z.string().max(128).optional(),89 })90 .strict();91export type SessionCreateInput = z.output<typeof sessionCreateSchema>;9293export interface FetchTiming {94 dns_ms: number;95 proxy_connect_ms: number;96 tls_ms: number;97 origin_ms: number;98 processing_ms: number;99 total_ms: number;100}101102export interface PageMetadata {103 title: string | null;104 description: string | null;105 canonical: string | null;106 lang: string | null;107 og: Record<string, string>;108 /** Number of hyperlinks found (the list itself is only returned with `links: true`). */109 links_count: number;110}111112export interface PageLink {113 url: string;114 text: string;115 internal: boolean;116 nofollow: boolean;117}118119export interface FetchMetadata {120 network: ConcreteNetwork | "direct";121 country: string | null;122 /** "http" for a plain fetch, "browser" when the final attempt was rendered in the managed browser. */123 mode: "http" | "browser";124 attempts: number;125 duration_ms: number;126 bytes: number;127 session: string | null;128 cached: boolean;129 timing?: FetchTiming;130 /** Only populated when the caller enabled debug AND the organization has provider visibility enabled. */131 debug?: {132 attempts: Array<{133 provider: string;134 network: string;135 mode: "http" | "browser";136 country: string | null;137 outcome: string;138 block_reason?: string | null;139 /** True when a captcha solver token was needed to pass this attempt. */140 captcha_solved?: boolean;141 status: number | null;142 duration_ms: number;143 error?: string;144 }>;145 };146}147148export interface FetchResponseBody {149 request_id: string;150 success: boolean;151 status: number;152 url: string;153 final_url: string;154 content: string | null;155 content_type: string | null;156 headers: Record<string, string>;157 cookies: Array<{ name: string; value: string; domain?: string; path?: string }>;158 metadata: FetchMetadata;159 /** Present when `format` = text. */160 text?: string | null;161 /** Present when `format` = json and the body parsed. */162 json?: unknown;163 /** Present when `format` = markdown. */164 markdown?: string | null;165 /** Parsed page metadata (HTML responses only). */166 page?: PageMetadata | null;167 /** Present when `links: true` (HTML responses only). */168 links?: PageLink[];169 /** Present when `screenshot: true` in browser mode: PNG, base64. */170 screenshot?: string | null;171}172173/**174 * Fetcha is a private platform: there is a single plan and it is unlimited. Access is granted by175 * an administrator (signup allowlist). Legacy plan names from the public preview map to it.176 */177export const PLANS = ["unlimited"] as const;178export type Plan = (typeof PLANS)[number];179180export interface PlanLimits {181 plan: Plan;182 label: string;183 monthly_requests: number;184 concurrency: number;185 max_timeout_ms: number;186 max_retries: number;187 networks: ConcreteNetwork[];188 retention_days: number;189 price_usd_month: number;190 included_gb: number;191 overage_per_1k_requests_usd: number;192 residential_per_gb_usd: number;193 /** Managed browser rendering available. */194 browser: boolean;195 /** Max concurrent browser renders per organization. */196 browser_concurrency: number;197 /** Crawl jobs: max pages per job and concurrent jobs per organization. */198 crawl_max_pages: number;199 crawl_concurrent_jobs: number;200}201202export const PLAN_LIMITS: Record<Plan, PlanLimits> = {203 unlimited: {204 plan: "unlimited",205 label: "Unlimited",206 monthly_requests: Number.MAX_SAFE_INTEGER,207 concurrency: 200,208 max_timeout_ms: 120_000,209 max_retries: 5,210 networks: ["datacenter", "residential", "isp", "mobile"],211 retention_days: 90,212 price_usd_month: 0,213 included_gb: 0,214 overage_per_1k_requests_usd: 0,215 residential_per_gb_usd: 0,216 browser: true,217 browser_concurrency: 8,218 crawl_max_pages: 2000,219 crawl_concurrent_jobs: 5,220 },221};222223/** Map any stored plan value (including legacy free/developer/growth/business/enterprise) to the single plan. */224export function normalizePlan(_plan: string | null | undefined): Plan {225 return "unlimited";226}227228export function isUnlimited(limits: PlanLimits): boolean {229 return limits.monthly_requests >= Number.MAX_SAFE_INTEGER;230}231232// ---------------------------------------------------------------------------233// Crawl & map234// ---------------------------------------------------------------------------235export const CRAWL_FORMATS = ["markdown", "text", "html"] as const;236export type CrawlFormat = (typeof CRAWL_FORMATS)[number];237238export const crawlCreateSchema = z239 .object({240 url: z.string().min(1).max(8192),241 /** Maximum number of pages to fetch (the seed counts as one). */242 max_pages: z.number().int().min(1).max(5000).default(25),243 /** Maximum link depth from the seed (0 = seed only). */244 max_depth: z.number().int().min(0).max(10).default(2),245 /** Only follow links on the seed's registrable host (default true). */246 same_domain: z.boolean().default(true),247 /** Also follow links on subdomains of the seed host. */248 allow_subdomains: z.boolean().default(false),249 /** Only crawl URLs matching at least one of these patterns (glob with `*`, or /regex/). */250 include_patterns: z.array(z.string().max(512)).max(50).optional(),251 /** Never crawl URLs matching one of these patterns. */252 exclude_patterns: z.array(z.string().max(512)).max(50).optional(),253 /** Honour robots.txt disallow rules for the seed host (default true). */254 respect_robots: z.boolean().default(true),255 /** Also seed the frontier with URLs from the site's sitemap(s). */256 use_sitemap: z.boolean().default(false),257 /** Parallel page fetches within this job. */258 concurrency: z.number().int().min(1).max(10).default(3),259 /** Fixed pause between page fetches per worker (politeness). */260 delay_ms: z.number().int().min(0).max(30_000).default(0),261 /** Per-page timeout. */262 timeout: z.number().int().min(1000).max(120_000).default(30_000),263 format: z.enum(CRAWL_FORMATS).default("markdown"),264 /** Keep only the main content (article/main) when converting to markdown/text. */265 main_content: z.boolean().default(true),266 country: z267 .string()268 .length(2)269 .transform((s) => s.toUpperCase())270 .optional(),271 network: z.enum(NETWORK_CLASSES).default("auto"),272 browser: z.boolean().default(false),273 browser_fallback: z.boolean().default(true),274 headers: headerRecord.optional(),275 /** Optional POST-back URL called once when the job finishes. */276 webhook_url: z.string().url().max(2048).optional(),277 label: z.string().max(128).optional(),278 })279 .strict();280export type CrawlCreateInput = z.output<typeof crawlCreateSchema>;281282export const CRAWL_STATUSES = ["queued", "running", "completed", "failed", "cancelled"] as const;283export type CrawlStatus = (typeof CRAWL_STATUSES)[number];284285export const mapCreateSchema = z286 .object({287 url: z.string().min(1).max(8192),288 /** Maximum number of URLs to return. */289 limit: z.number().int().min(1).max(10_000).default(1000),290 /** Include sitemap.xml (and sitemaps listed in robots.txt). */291 use_sitemap: z.boolean().default(true),292 /** Include hyperlinks from the seed page. */293 use_links: z.boolean().default(true),294 same_domain: z.boolean().default(true),295 allow_subdomains: z.boolean().default(false),296 /** Filter results by substring / glob / regex. */297 search: z.string().max(256).optional(),298 country: z299 .string()300 .length(2)301 .transform((s) => s.toUpperCase())302 .optional(),303 network: z.enum(NETWORK_CLASSES).default("auto"),304 timeout: z.number().int().min(1000).max(120_000).default(30_000),305 })306 .strict();307export type MapCreateInput = z.output<typeof mapCreateSchema>;308309export const API_KEY_SCOPES = ["fetch:execute", "browser:use", "crawl:execute", "sessions:write", "usage:read"] as const;310export type ApiKeyScope = (typeof API_KEY_SCOPES)[number];311