import { FetchaError, PLAN_LIMITS, extractDomain, fetchRequestSchema, newId, type FetchRequest } from "@fetcha/core"; import { db, eq, proxySessions, sql } from "@fetcha/db"; import type { ProviderId } from "@fetcha/providers"; import type { AttemptRecord, SerializedCookie } from "@fetcha/routing"; import type { ConcreteNetwork } from "@fetcha/core"; import type { ApiPrincipal } from "../auth"; import { config } from "../config"; import { acquireBrowserSlot, acquireConcurrency, checkMonthlyLimits, checkRateLimits, invalidateUsageCache } from "../limits"; import { getEngine } from "../services/engine"; import { completeRequest, createRequestRow, failRequest, recordAttempt, type RequestContext } from "../services/persist"; import { checkSpendAlerts, notifyHardLimit, recordAbuse } from "../services/alerts"; export interface FetchHandlerInput { principal: ApiPrincipal; body: unknown; source: "api" | "playground" | "sdk" | "crawl"; clientIp: string | null; userAgent: string | null; } export interface FetchHandlerOutput { requestId: string; status: number; body: unknown; /** Internal: execution summary for callers that need more than the public document (crawler). */ summary?: { mode: "http" | "browser"; bytes: number; costUsd: number; attempts: number; domain: string }; } /** * Shared /v1/fetch pipeline used by the public API, the dashboard playground and the crawler. * Order matters: validate → limits → SSRF (inside executor) → route → persist. */ export async function handleFetch(input: FetchHandlerInput): Promise { const { principal } = input; const requestId = newId("req"); const started = performance.now(); const parsed = fetchRequestSchema.safeParse(input.body); if (!parsed.success) { throw new FetchaError("INVALID_REQUEST", "The request body is invalid.", { requestId, details: { issues: parsed.error.issues.map((i) => ({ path: i.path.join("."), message: i.message })) }, }); } const req: FetchRequest = parsed.data; const limits = PLAN_LIMITS[principal.plan]; if (req.timeout > limits.max_timeout_ms) req.timeout = limits.max_timeout_ms; if (req.network !== "auto" && !limits.networks.includes(req.network)) { throw new FetchaError("NETWORK_UNAVAILABLE", `The "${req.network}" network is not included in the ${limits.label} plan.`, { requestId }); } const engine = await getEngine(); if (req.browser && (!limits.browser || !engine.browserEnabled)) throw new FetchaError("BROWSER_UNAVAILABLE", undefined, { requestId }); await checkRateLimits(principal, input.clientIp); try { await checkMonthlyLimits(principal); } catch (e) { if (e instanceof FetchaError && e.code === "USAGE_LIMIT_REACHED" && (principal.projectHardLimitUsd !== null || principal.orgHardLimitUsd !== null)) { const { monthlyUsage } = await import("../limits"); notifyHardLimit(principal, await monthlyUsage(principal)).catch(() => {}); } throw e; } const release = await acquireConcurrency(principal); let releaseBrowser: (() => Promise) | null = null; if (req.browser) { try { releaseBrowser = await acquireBrowserSlot(principal); } catch (e) { await release(); throw e; } } const domain = extractDomain(req.url); const ctx: RequestContext = { requestId, organizationId: principal.organizationId, projectId: principal.projectId, apiKeyId: principal.keyId, source: input.source, plan: principal.plan, clientIp: input.clientIp, userAgent: input.userAgent, logLevel: principal.projectLogLevel, }; const attempts: AttemptRecord[] = []; try { await createRequestRow(ctx, req, domain); // Resolve a Fetcha session (sess_…) into a provider-pinned sticky key (+ its cookie jar). let sessionKey: string | null = null; let sessionProvider: ProviderId | null = null; let sessionNetwork: ConcreteNetwork | null = null; let sessionCookies: SerializedCookie[] | null = null; let sessionId: string | null = null; if (req.session) { const [s] = await db.select().from(proxySessions).where(eq(proxySessions.id, req.session)).limit(1); if (!s || s.projectId !== principal.projectId) throw new FetchaError("SESSION_NOT_FOUND", undefined, { requestId }); if (s.status !== "active" || s.expiresAt.getTime() < Date.now()) throw new FetchaError("SESSION_EXPIRED", undefined, { requestId }); sessionKey = s.stickyKey; sessionId = s.id; sessionProvider = s.provider as ProviderId; sessionNetwork = s.network as ConcreteNetwork; sessionCookies = (s.cookies ?? []) as unknown as SerializedCookie[]; if (!req.country && s.country) req.country = s.country; if (req.network === "auto") req.network = s.network as FetchRequest["network"]; db.update(proxySessions) .set({ lastUsedAt: new Date(), requestCount: sql`${proxySessions.requestCount} + 1` }) .where(eq(proxySessions.id, s.id)) .catch(() => {}); } const knowledge = await engine.knowledge(domain); const result = await engine.executor.execute({ requestId, plan: principal.plan, request: req, sessionKey, sessionProvider, sessionNetwork, sessionCookies, knowledge, providerVisibility: principal.providerVisibility, maxResponseBytes: Math.min(req.max_response_bytes ?? config.maxResponseBytesDefault, config.maxResponseBytesDefault), browserPool: engine.browserEnabled && limits.browser ? engine.browser : null, browserAllowed: limits.browser, onAttempt: async (a) => { attempts.push(a); await recordAttempt(requestId, a).catch(() => {}); }, }); const latencyMs = Math.round(performance.now() - started); await completeRequest(ctx, req, result, latencyMs); if (sessionId && result.cookies.length) { db.update(proxySessions) .set({ cookies: result.cookies.slice(-200) as unknown as Array<{ name: string; value: string; domain?: string; path?: string }> }) .where(eq(proxySessions.id, sessionId)) .catch(() => {}); } await invalidateUsageCache(principal.organizationId, principal.projectId); if (principal.projectSoftLimitUsd !== null || principal.orgSoftLimitUsd !== null) { const { monthlyUsage } = await import("../limits"); monthlyUsage(principal).then((u) => checkSpendAlerts(principal, u)).catch(() => {}); } return { requestId, status: 200, body: result.body, summary: { mode: result.mode, bytes: result.bytesIn + result.bytesOut, costUsd: result.costUsd, attempts: result.attempts.length, domain } }; } catch (e) { const latencyMs = Math.round(performance.now() - started); const err = e instanceof FetchaError ? e : new FetchaError("INTERNAL_ERROR", undefined, { requestId, cause: e }); await failRequest(ctx, err.code, err.message, attempts, latencyMs, domain).catch(() => {}); if (err.code === "URL_NOT_ALLOWED") await recordAbuse(principal, requestId, "ssrf_attempt", `${req.method} ${req.url}`); if (!(e instanceof FetchaError)) console.error(`[fetch] ${requestId} internal error`, e); throw Object.assign(err, { requestId }); } finally { await releaseBrowser?.(); await release(); } }