TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import { newId, redactHeaders, type ConcreteNetwork, type FetchRequest, type Plan } from "@fetcha/core";2import { db, domainProfiles, fetchRequests, requestAttempts, routingMetrics, usageEvents, sql } from "@fetcha/db";3import { foldRouteStat, preferredRoute, routeKey, type AttemptRecord, type ExecutionResult } from "@fetcha/routing";4import { priceRequest } from "./pricing";56export interface RequestContext {7 requestId: string;8 organizationId: string;9 projectId: string;10 apiKeyId: string | null;11 source: "api" | "playground" | "sdk" | "crawl";12 plan: Plan;13 clientIp: string | null;14 userAgent: string | null;15 logLevel: "none" | "metadata" | "headers" | "full";16}1718export async function createRequestRow(ctx: RequestContext, req: FetchRequest, domain: string): Promise<void> {19 await db.insert(fetchRequests).values({20 id: ctx.requestId,21 organizationId: ctx.organizationId,22 projectId: ctx.projectId,23 apiKeyId: ctx.apiKeyId,24 source: ctx.source,25 url: req.url,26 domain,27 method: req.method,28 requestedNetwork: req.network,29 country: req.country ?? null,30 region: req.region ?? null,31 city: req.city ?? null,32 sessionId: req.session ?? null,33 browser: req.browser,34 format: req.format,35 status: "pending",36 requestHeaders: ctx.logLevel === "headers" || ctx.logLevel === "full" ? redactHeaders(req.headers) : null,37 clientIp: ctx.clientIp,38 userAgent: ctx.userAgent,39 });40}4142export async function recordAttempt(requestId: string, a: AttemptRecord): Promise<void> {43 await db.insert(requestAttempts).values({44 id: a.attemptId,45 requestId,46 attemptNo: a.attemptNo,47 provider: a.provider,48 network: a.network,49 mode: a.mode,50 country: a.country,51 sessionKey: a.sessionKey,52 outcome: a.outcome,53 httpStatus: a.httpStatus,54 errorCode: a.errorCode,55 errorDetail: a.errorDetail ?? (a.blockVendor ? `vendor=${a.blockVendor} profile=${a.profileId ?? "-"}` : a.profileId ? `profile=${a.profileId}` : null),56 blockReason: a.blockReason,57 durationMs: a.durationMs,58 bytesIn: a.bytesIn,59 bytesOut: a.bytesOut,60 unitPricePerGb: a.unitPricePerGb,61 costUsd: a.costUsd,62 routingScore: a.routingScore,63 timing: a.timing ? { ...a.timing } : null,64 });65}6667export interface CompletionSummary {68 priceUsd: number;69 costUsd: number;70}7172/** Persist a successful (or blocked-but-answered) execution: request row, usage ledger, domain intelligence, routing metrics. */73export async function completeRequest(ctx: RequestContext, req: FetchRequest, result: ExecutionResult, latencyMs: number): Promise<CompletionSummary> {74 const network = result.network;75 const price = priceRequest({ plan: ctx.plan, network, bytes: result.bytesIn + result.bytesOut, upstreamCostUsd: result.costUsd, attempts: result.attempts.length, success: result.body.success });76 const finalStatus = result.body.success ? "success" : "failed";7778 await db79 .update(fetchRequests)80 .set({81 status: finalStatus,82 httpStatus: result.body.status,83 errorCode: result.body.success ? null : "TARGET_BLOCKED",84 errorMessage: result.body.success ? null : "The target blocked every route we tried.",85 finalUrl: result.finalUrl,86 network,87 mode: result.mode,88 browser: result.mode === "browser",89 attempts: result.attempts.length,90 latencyMs,91 bytesIn: result.bytesIn,92 bytesOut: result.bytesOut,93 costUsd: result.costUsd,94 priceUsd: price.totalUsd,95 responseHeaders: ctx.logLevel === "headers" || ctx.logLevel === "full" ? redactHeaders(result.body.headers) : null,96 timing: result.body.metadata.timing ? { ...result.body.metadata.timing } : null,97 completedAt: new Date(),98 })99 .where(sql`${fetchRequests.id} = ${ctx.requestId}`);100101 await writeUsage(ctx, result, price.totalUsd, network);102 await Promise.all([updateDomainProfile(result.domain, result.attempts, result.body.success, result.browserRequired), updateRoutingMetrics(result.attempts)]);103 return { priceUsd: price.totalUsd, costUsd: result.costUsd };104}105106export async function failRequest(ctx: RequestContext, code: string, message: string, attempts: AttemptRecord[], latencyMs: number, domain: string): Promise<void> {107 const costUsd = attempts.reduce((s, a) => s + a.costUsd, 0);108 await db109 .update(fetchRequests)110 .set({ status: "failed", errorCode: code, errorMessage: message, attempts: attempts.length, latencyMs, costUsd, completedAt: new Date() })111 .where(sql`${fetchRequests.id} = ${ctx.requestId}`);112 // Failed requests still count against the request quota (abuse control) but are not priced.113 await db.insert(usageEvents).values({114 id: newId("usage"),115 organizationId: ctx.organizationId,116 projectId: ctx.projectId,117 requestId: ctx.requestId,118 metric: "request",119 quantity: 1,120 unit: "count",121 costUsd: 0,122 upstreamCostUsd: costUsd,123 });124 if (attempts.length) await Promise.all([updateDomainProfile(domain, attempts, false, false), updateRoutingMetrics(attempts)]);125}126127async function writeUsage(ctx: RequestContext, result: ExecutionResult, priceUsd: number, network: ConcreteNetwork | null) {128 const bytes = result.bytesIn + result.bytesOut;129 const rows = [130 { metric: "request", quantity: 1, unit: "count", costUsd: priceUsd, upstreamCostUsd: result.costUsd },131 { metric: "bandwidth", quantity: bytes, unit: "bytes", costUsd: 0, upstreamCostUsd: 0 },132 ];133 if (network === "residential" || network === "isp") rows.push({ metric: "residential_bandwidth", quantity: bytes, unit: "bytes", costUsd: 0, upstreamCostUsd: 0 });134 if (network === "mobile") rows.push({ metric: "mobile_bandwidth", quantity: bytes, unit: "bytes", costUsd: 0, upstreamCostUsd: 0 });135 await db.insert(usageEvents).values(136 rows.map((r) => ({ id: newId("usage"), organizationId: ctx.organizationId, projectId: ctx.projectId, requestId: ctx.requestId, ...r })),137 );138}139140async function updateDomainProfile(domain: string, attempts: AttemptRecord[], success: boolean, browserRequired: boolean) {141 if (!domain) return;142 const [existing] = await db.select().from(domainProfiles).where(sql`${domainProfiles.domain} = ${domain}`).limit(1);143 const stats = { ...(existing?.routeStats ?? {}) };144 let blocks = 0;145 let captchas = 0;146 for (const a of attempts) {147 foldRouteStat(stats, routeKey(a.provider, a.network), { ok: a.outcome === "success", blocked: a.outcome === "blocked", latencyMs: a.durationMs, costUsd: a.costUsd });148 if (a.outcome === "blocked") blocks++;149 if (a.blockReason === "captcha" || a.blockReason === "cloudflare_challenge") captchas++;150 }151 const browserInc = browserRequired ? 1 : 0;152 const pref = preferredRoute(stats);153 const totalLatency = attempts.reduce((s, a) => s + a.durationMs, 0);154 const n = (existing?.requests ?? 0) + 1;155 const avgLatency = existing ? existing.avgLatencyMs + (totalLatency - existing.avgLatencyMs) / n : totalLatency;156 await db157 .insert(domainProfiles)158 .values({159 domain,160 preferredNetwork: pref?.network ?? null,161 preferredProvider: pref?.provider ?? null,162 requests: 1,163 successes: success ? 1 : 0,164 blocks,165 captchas,166 browserRequired: browserInc,167 avgLatencyMs: totalLatency,168 routeStats: stats,169 lastSeenAt: new Date(),170 })171 .onConflictDoUpdate({172 target: domainProfiles.domain,173 set: {174 preferredNetwork: pref?.network ?? null,175 preferredProvider: pref?.provider ?? null,176 requests: sql`${domainProfiles.requests} + 1`,177 successes: sql`${domainProfiles.successes} + ${success ? 1 : 0}`,178 blocks: sql`${domainProfiles.blocks} + ${blocks}`,179 captchas: sql`${domainProfiles.captchas} + ${captchas}`,180 browserRequired: sql`${domainProfiles.browserRequired} + ${browserInc}`,181 avgLatencyMs: avgLatency,182 routeStats: stats,183 lastSeenAt: new Date(),184 updatedAt: new Date(),185 },186 });187}188189async function updateRoutingMetrics(attempts: AttemptRecord[]) {190 const bucket = new Date();191 bucket.setMinutes(0, 0, 0);192 for (const a of attempts) {193 const ok = a.outcome === "success" ? 1 : 0;194 const blocked = a.outcome === "blocked" ? 1 : 0;195 const errors = ok || blocked ? 0 : 1;196 await db197 .insert(routingMetrics)198 .values({199 id: newId("evt"),200 bucket,201 provider: a.provider,202 network: a.network,203 country: a.country ?? "",204 requests: 1,205 successes: ok,206 blocked,207 errors,208 latencySumMs: a.durationMs,209 bytes: a.bytesIn + a.bytesOut,210 costUsd: a.costUsd,211 })212 .onConflictDoUpdate({213 target: [routingMetrics.bucket, routingMetrics.provider, routingMetrics.network, routingMetrics.country],214 set: {215 requests: sql`${routingMetrics.requests} + 1`,216 successes: sql`${routingMetrics.successes} + ${ok}`,217 blocked: sql`${routingMetrics.blocked} + ${blocked}`,218 errors: sql`${routingMetrics.errors} + ${errors}`,219 latencySumMs: sql`${routingMetrics.latencySumMs} + ${a.durationMs}`,220 bytes: sql`${routingMetrics.bytes} + ${a.bytesIn + a.bytesOut}`,221 costUsd: sql`${routingMetrics.costUsd} + ${a.costUsd}`,222 },223 });224 }225}226