import { z } from "zod"; /** Network classes exposed to customers. Providers are never exposed. */ export const NETWORK_CLASSES = ["auto", "datacenter", "residential", "isp", "mobile"] as const; export type NetworkClass = (typeof NETWORK_CLASSES)[number]; /** A concrete network class (never `auto`). */ export type ConcreteNetwork = Exclude; export const HTTP_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"] as const; export type HttpMethod = (typeof HTTP_METHODS)[number]; export const OUTPUT_FORMATS = ["html", "text", "markdown", "json", "raw"] as const; export type OutputFormat = (typeof OUTPUT_FORMATS)[number]; export const DEVICES = ["desktop", "mobile", "tablet"] as const; const headerRecord = z.record(z.string().max(256), z.string().max(8192)).refine( (h) => Object.keys(h).length <= 64, "Too many headers (max 64)", ); export const fetchRequestSchema = z .object({ url: z.string().min(1).max(8192), method: z.enum(HTTP_METHODS).default("GET"), headers: headerRecord.optional(), cookies: z.record(z.string().max(256), z.string().max(4096)).optional(), body: z.union([z.string().max(2_000_000), z.record(z.string(), z.unknown())]).optional(), timeout: z.number().int().min(1000).max(120_000).default(30_000), country: z .string() .length(2) .transform((s) => s.toUpperCase()) .optional(), region: z.string().max(64).optional(), city: z.string().max(128).optional(), network: z.enum(NETWORK_CLASSES).default("auto"), session: z.string().max(64).optional(), /** Render in a managed headless browser (real Chromium) routed through the same network. */ browser: z.boolean().default(false), /** When an HTTP attempt is blocked by a JavaScript challenge / anti-bot, automatically escalate to the browser. */ browser_fallback: z.boolean().default(true), javascript: z.boolean().optional(), wait_for: z.string().max(512).optional(), wait_ms: z.number().int().min(0).max(30_000).optional(), wait_until: z.enum(["load", "domcontentloaded", "networkidle"]).default("domcontentloaded"), /** Browser: skip images, fonts and media to save bandwidth (default true). */ block_resources: z.boolean().default(true), /** Browser: return a PNG screenshot (base64) in `screenshot`. */ screenshot: z.boolean().default(false), /** Browser: use the captcha solver on Cloudflare Turnstile challenges when the platform has one configured (default true). */ solve_captcha: z.boolean().default(true), /** Include the list of hyperlinks found in the page (`links`). */ links: z.boolean().default(false), /** Referer strategy: "auto" (none on first try, search-engine referer on retries), "none", or a literal URL. */ referer: z.union([z.enum(["auto", "none"]), z.string().url().max(2048)]).default("auto"), device: z.enum(DEVICES).optional(), locale: z.string().max(16).optional(), format: z.enum(OUTPUT_FORMATS).default("html"), follow_redirects: z.boolean().default(true), max_redirects: z.number().int().min(0).max(20).default(10), max_response_bytes: z.number().int().min(1024).max(50_000_000).optional(), cache: z .object({ enabled: z.boolean().default(false), ttl: z.number().int().min(1).max(86_400).default(300), }) .optional(), retries: z.number().int().min(0).max(5).optional(), debug: z.boolean().default(false), }) .strict(); export type FetchRequestInput = z.input; export type FetchRequest = z.output; export const sessionCreateSchema = z .object({ country: z .string() .length(2) .transform((s) => s.toUpperCase()) .optional(), region: z.string().max(64).optional(), city: z.string().max(128).optional(), network: z.enum(NETWORK_CLASSES).default("auto"), ttl: z.number().int().min(60).max(1800).default(600), label: z.string().max(128).optional(), }) .strict(); export type SessionCreateInput = z.output; export interface FetchTiming { dns_ms: number; proxy_connect_ms: number; tls_ms: number; origin_ms: number; processing_ms: number; total_ms: number; } export interface PageMetadata { title: string | null; description: string | null; canonical: string | null; lang: string | null; og: Record; /** Number of hyperlinks found (the list itself is only returned with `links: true`). */ links_count: number; } export interface PageLink { url: string; text: string; internal: boolean; nofollow: boolean; } export interface FetchMetadata { network: ConcreteNetwork | "direct"; country: string | null; /** "http" for a plain fetch, "browser" when the final attempt was rendered in the managed browser. */ mode: "http" | "browser"; attempts: number; duration_ms: number; bytes: number; session: string | null; cached: boolean; timing?: FetchTiming; /** Only populated when the caller enabled debug AND the organization has provider visibility enabled. */ debug?: { attempts: Array<{ provider: string; network: string; mode: "http" | "browser"; country: string | null; outcome: string; block_reason?: string | null; /** True when a captcha solver token was needed to pass this attempt. */ captcha_solved?: boolean; status: number | null; duration_ms: number; error?: string; }>; }; } export interface FetchResponseBody { request_id: string; success: boolean; status: number; url: string; final_url: string; content: string | null; content_type: string | null; headers: Record; cookies: Array<{ name: string; value: string; domain?: string; path?: string }>; metadata: FetchMetadata; /** Present when `format` = text. */ text?: string | null; /** Present when `format` = json and the body parsed. */ json?: unknown; /** Present when `format` = markdown. */ markdown?: string | null; /** Parsed page metadata (HTML responses only). */ page?: PageMetadata | null; /** Present when `links: true` (HTML responses only). */ links?: PageLink[]; /** Present when `screenshot: true` in browser mode: PNG, base64. */ screenshot?: string | null; } /** * Fetcha is a private platform: there is a single plan and it is unlimited. Access is granted by * an administrator (signup allowlist). Legacy plan names from the public preview map to it. */ export const PLANS = ["unlimited"] as const; export type Plan = (typeof PLANS)[number]; export interface PlanLimits { plan: Plan; label: string; monthly_requests: number; concurrency: number; max_timeout_ms: number; max_retries: number; networks: ConcreteNetwork[]; retention_days: number; price_usd_month: number; included_gb: number; overage_per_1k_requests_usd: number; residential_per_gb_usd: number; /** Managed browser rendering available. */ browser: boolean; /** Max concurrent browser renders per organization. */ browser_concurrency: number; /** Crawl jobs: max pages per job and concurrent jobs per organization. */ crawl_max_pages: number; crawl_concurrent_jobs: number; } export const PLAN_LIMITS: Record = { unlimited: { plan: "unlimited", label: "Unlimited", monthly_requests: Number.MAX_SAFE_INTEGER, concurrency: 200, max_timeout_ms: 120_000, max_retries: 5, networks: ["datacenter", "residential", "isp", "mobile"], retention_days: 90, price_usd_month: 0, included_gb: 0, overage_per_1k_requests_usd: 0, residential_per_gb_usd: 0, browser: true, browser_concurrency: 8, crawl_max_pages: 2000, crawl_concurrent_jobs: 5, }, }; /** Map any stored plan value (including legacy free/developer/growth/business/enterprise) to the single plan. */ export function normalizePlan(_plan: string | null | undefined): Plan { return "unlimited"; } export function isUnlimited(limits: PlanLimits): boolean { return limits.monthly_requests >= Number.MAX_SAFE_INTEGER; } // --------------------------------------------------------------------------- // Crawl & map // --------------------------------------------------------------------------- export const CRAWL_FORMATS = ["markdown", "text", "html"] as const; export type CrawlFormat = (typeof CRAWL_FORMATS)[number]; export const crawlCreateSchema = z .object({ url: z.string().min(1).max(8192), /** Maximum number of pages to fetch (the seed counts as one). */ max_pages: z.number().int().min(1).max(5000).default(25), /** Maximum link depth from the seed (0 = seed only). */ max_depth: z.number().int().min(0).max(10).default(2), /** Only follow links on the seed's registrable host (default true). */ same_domain: z.boolean().default(true), /** Also follow links on subdomains of the seed host. */ allow_subdomains: z.boolean().default(false), /** Only crawl URLs matching at least one of these patterns (glob with `*`, or /regex/). */ include_patterns: z.array(z.string().max(512)).max(50).optional(), /** Never crawl URLs matching one of these patterns. */ exclude_patterns: z.array(z.string().max(512)).max(50).optional(), /** Honour robots.txt disallow rules for the seed host (default true). */ respect_robots: z.boolean().default(true), /** Also seed the frontier with URLs from the site's sitemap(s). */ use_sitemap: z.boolean().default(false), /** Parallel page fetches within this job. */ concurrency: z.number().int().min(1).max(10).default(3), /** Fixed pause between page fetches per worker (politeness). */ delay_ms: z.number().int().min(0).max(30_000).default(0), /** Per-page timeout. */ timeout: z.number().int().min(1000).max(120_000).default(30_000), format: z.enum(CRAWL_FORMATS).default("markdown"), /** Keep only the main content (article/main) when converting to markdown/text. */ main_content: z.boolean().default(true), country: z .string() .length(2) .transform((s) => s.toUpperCase()) .optional(), network: z.enum(NETWORK_CLASSES).default("auto"), browser: z.boolean().default(false), browser_fallback: z.boolean().default(true), headers: headerRecord.optional(), /** Optional POST-back URL called once when the job finishes. */ webhook_url: z.string().url().max(2048).optional(), label: z.string().max(128).optional(), }) .strict(); export type CrawlCreateInput = z.output; export const CRAWL_STATUSES = ["queued", "running", "completed", "failed", "cancelled"] as const; export type CrawlStatus = (typeof CRAWL_STATUSES)[number]; export const mapCreateSchema = z .object({ url: z.string().min(1).max(8192), /** Maximum number of URLs to return. */ limit: z.number().int().min(1).max(10_000).default(1000), /** Include sitemap.xml (and sitemaps listed in robots.txt). */ use_sitemap: z.boolean().default(true), /** Include hyperlinks from the seed page. */ use_links: z.boolean().default(true), same_domain: z.boolean().default(true), allow_subdomains: z.boolean().default(false), /** Filter results by substring / glob / regex. */ search: z.string().max(256).optional(), country: z .string() .length(2) .transform((s) => s.toUpperCase()) .optional(), network: z.enum(NETWORK_CLASSES).default("auto"), timeout: z.number().int().min(1000).max(120_000).default(30_000), }) .strict(); export type MapCreateInput = z.output; export const API_KEY_SCOPES = ["fetch:execute", "browser:use", "crawl:execute", "sessions:write", "usage:read"] as const; export type ApiKeyScope = (typeof API_KEY_SCOPES)[number];