TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import { FetchaError, PLAN_LIMITS, extractDomain, fetchRequestSchema, newId, type FetchRequest } from "@fetcha/core";2import { db, eq, proxySessions, sql } from "@fetcha/db";3import type { ProviderId } from "@fetcha/providers";4import type { AttemptRecord, SerializedCookie } from "@fetcha/routing";5import type { ConcreteNetwork } from "@fetcha/core";6import type { ApiPrincipal } from "../auth";7import { config } from "../config";8import { acquireBrowserSlot, acquireConcurrency, checkMonthlyLimits, checkRateLimits, invalidateUsageCache } from "../limits";9import { getEngine } from "../services/engine";10import { completeRequest, createRequestRow, failRequest, recordAttempt, type RequestContext } from "../services/persist";11import { checkSpendAlerts, notifyHardLimit, recordAbuse } from "../services/alerts";1213export interface FetchHandlerInput {14 principal: ApiPrincipal;15 body: unknown;16 source: "api" | "playground" | "sdk" | "crawl";17 clientIp: string | null;18 userAgent: string | null;19}2021export interface FetchHandlerOutput {22 requestId: string;23 status: number;24 body: unknown;25 /** Internal: execution summary for callers that need more than the public document (crawler). */26 summary?: { mode: "http" | "browser"; bytes: number; costUsd: number; attempts: number; domain: string };27}2829/**30 * Shared /v1/fetch pipeline used by the public API, the dashboard playground and the crawler.31 * Order matters: validate → limits → SSRF (inside executor) → route → persist.32 */33export async function handleFetch(input: FetchHandlerInput): Promise<FetchHandlerOutput> {34 const { principal } = input;35 const requestId = newId("req");36 const started = performance.now();3738 const parsed = fetchRequestSchema.safeParse(input.body);39 if (!parsed.success) {40 throw new FetchaError("INVALID_REQUEST", "The request body is invalid.", {41 requestId,42 details: { issues: parsed.error.issues.map((i) => ({ path: i.path.join("."), message: i.message })) },43 });44 }45 const req: FetchRequest = parsed.data;46 const limits = PLAN_LIMITS[principal.plan];47 if (req.timeout > limits.max_timeout_ms) req.timeout = limits.max_timeout_ms;48 if (req.network !== "auto" && !limits.networks.includes(req.network)) {49 throw new FetchaError("NETWORK_UNAVAILABLE", `The "${req.network}" network is not included in the ${limits.label} plan.`, { requestId });50 }51 const engine = await getEngine();52 if (req.browser && (!limits.browser || !engine.browserEnabled)) throw new FetchaError("BROWSER_UNAVAILABLE", undefined, { requestId });5354 await checkRateLimits(principal, input.clientIp);55 try {56 await checkMonthlyLimits(principal);57 } catch (e) {58 if (e instanceof FetchaError && e.code === "USAGE_LIMIT_REACHED" && (principal.projectHardLimitUsd !== null || principal.orgHardLimitUsd !== null)) {59 const { monthlyUsage } = await import("../limits");60 notifyHardLimit(principal, await monthlyUsage(principal)).catch(() => {});61 }62 throw e;63 }64 const release = await acquireConcurrency(principal);65 let releaseBrowser: (() => Promise<void>) | null = null;66 if (req.browser) {67 try {68 releaseBrowser = await acquireBrowserSlot(principal);69 } catch (e) {70 await release();71 throw e;72 }73 }7475 const domain = extractDomain(req.url);76 const ctx: RequestContext = {77 requestId,78 organizationId: principal.organizationId,79 projectId: principal.projectId,80 apiKeyId: principal.keyId,81 source: input.source,82 plan: principal.plan,83 clientIp: input.clientIp,84 userAgent: input.userAgent,85 logLevel: principal.projectLogLevel,86 };8788 const attempts: AttemptRecord[] = [];89 try {90 await createRequestRow(ctx, req, domain);9192 // Resolve a Fetcha session (sess_…) into a provider-pinned sticky key (+ its cookie jar).93 let sessionKey: string | null = null;94 let sessionProvider: ProviderId | null = null;95 let sessionNetwork: ConcreteNetwork | null = null;96 let sessionCookies: SerializedCookie[] | null = null;97 let sessionId: string | null = null;98 if (req.session) {99 const [s] = await db.select().from(proxySessions).where(eq(proxySessions.id, req.session)).limit(1);100 if (!s || s.projectId !== principal.projectId) throw new FetchaError("SESSION_NOT_FOUND", undefined, { requestId });101 if (s.status !== "active" || s.expiresAt.getTime() < Date.now()) throw new FetchaError("SESSION_EXPIRED", undefined, { requestId });102 sessionKey = s.stickyKey;103 sessionId = s.id;104 sessionProvider = s.provider as ProviderId;105 sessionNetwork = s.network as ConcreteNetwork;106 sessionCookies = (s.cookies ?? []) as unknown as SerializedCookie[];107 if (!req.country && s.country) req.country = s.country;108 if (req.network === "auto") req.network = s.network as FetchRequest["network"];109 db.update(proxySessions)110 .set({ lastUsedAt: new Date(), requestCount: sql`${proxySessions.requestCount} + 1` })111 .where(eq(proxySessions.id, s.id))112 .catch(() => {});113 }114115 const knowledge = await engine.knowledge(domain);116 const result = await engine.executor.execute({117 requestId,118 plan: principal.plan,119 request: req,120 sessionKey,121 sessionProvider,122 sessionNetwork,123 sessionCookies,124 knowledge,125 providerVisibility: principal.providerVisibility,126 maxResponseBytes: Math.min(req.max_response_bytes ?? config.maxResponseBytesDefault, config.maxResponseBytesDefault),127 browserPool: engine.browserEnabled && limits.browser ? engine.browser : null,128 browserAllowed: limits.browser,129 onAttempt: async (a) => {130 attempts.push(a);131 await recordAttempt(requestId, a).catch(() => {});132 },133 });134 const latencyMs = Math.round(performance.now() - started);135 await completeRequest(ctx, req, result, latencyMs);136 if (sessionId && result.cookies.length) {137 db.update(proxySessions)138 .set({ cookies: result.cookies.slice(-200) as unknown as Array<{ name: string; value: string; domain?: string; path?: string }> })139 .where(eq(proxySessions.id, sessionId))140 .catch(() => {});141 }142 await invalidateUsageCache(principal.organizationId, principal.projectId);143 if (principal.projectSoftLimitUsd !== null || principal.orgSoftLimitUsd !== null) {144 const { monthlyUsage } = await import("../limits");145 monthlyUsage(principal).then((u) => checkSpendAlerts(principal, u)).catch(() => {});146 }147 return { requestId, status: 200, body: result.body, summary: { mode: result.mode, bytes: result.bytesIn + result.bytesOut, costUsd: result.costUsd, attempts: result.attempts.length, domain } };148 } catch (e) {149 const latencyMs = Math.round(performance.now() - started);150 const err = e instanceof FetchaError ? e : new FetchaError("INTERNAL_ERROR", undefined, { requestId, cause: e });151 await failRequest(ctx, err.code, err.message, attempts, latencyMs, domain).catch(() => {});152 if (err.code === "URL_NOT_ALLOWED") await recordAbuse(principal, requestId, "ssrf_attempt", `${req.method} ${req.url}`);153 if (!(e instanceof FetchaError)) console.error(`[fetch] ${requestId} internal error`, e);154 throw Object.assign(err, { requestId });155 } finally {156 await releaseBrowser?.();157 await release();158 }159}160