TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import Fastify from "fastify";2import cors from "@fastify/cors";3import { FetchaError, isFetchaError, newId } from "@fetcha/core";4import { closeDb, db, sql } from "@fetcha/db";5import { closeAllDispatchers } from "@fetcha/providers";6import { closeBrowserPool } from "@fetcha/browser";7import { authenticateApiKey, requireScope } from "./auth";8import { assertProductionConfig, config } from "./config";9import { closeKV, getKV } from "./redis";10import { handleFetch } from "./routes/fetch";11import { registerInternalRoutes } from "./routes/internal";12import { registerCrawlRoutes, registerInternalCrawlRoutes } from "./routes/crawl";13import { startCrawlWorker } from "./services/crawl";14import { meResponse, usageResponse } from "./routes/misc";15import { closeSession, createSession, getSession, listSessions } from "./routes/sessions";16import { getEngine, startHealthLoop } from "./services/engine";1718export async function buildServer() {19 const app = Fastify({20 logger: {21 level: config.logLevel,22 ...(config.env !== "production" ? { transport: { target: "pino-pretty", options: { colorize: true, translateTime: "HH:MM:ss" } } } : {}),23 redact: ["req.headers.authorization", "req.headers['x-api-key']", "req.headers.cookie"],24 },25 trustProxy: true,26 bodyLimit: 4 * 1024 * 1024,27 genReqId: () => newId("req"),28 disableRequestLogging: config.env === "production",29 });3031 await app.register(cors, {32 origin: true,33 methods: ["GET", "POST", "DELETE", "OPTIONS"],34 allowedHeaders: ["authorization", "content-type", "x-api-key", "idempotency-key"],35 exposedHeaders: ["x-fetcha-request-id", "x-request-id"],36 });3738 app.addHook("onSend", async (req, reply) => {39 if (!reply.getHeader("x-fetcha-request-id")) reply.header("x-fetcha-request-id", req.id);40 reply.header("x-fetcha-version", config.version);41 reply.header("cache-control", "no-store");42 });4344 app.setErrorHandler((err: unknown, req, reply) => {45 if (isFetchaError(err)) {46 const e = err as FetchaError;47 const rid = e.requestId ?? req.id;48 reply.header("x-fetcha-request-id", rid);49 if (e.code === "RATE_LIMITED" && e.details?.retry_after_ms) reply.header("retry-after", String(Math.ceil(Number(e.details.retry_after_ms) / 1000)));50 return reply.status(e.status).send(e.toJSON(rid));51 }52 const fe = err as { statusCode?: number; validation?: unknown; message?: string; code?: string };53 if (fe.statusCode === 400 || fe.validation || fe.code === "FST_ERR_CTP_INVALID_MEDIA_TYPE" || fe.code === "FST_ERR_CTP_EMPTY_JSON_BODY") {54 return reply.status(400).send(new FetchaError("INVALID_REQUEST", fe.code === "FST_ERR_CTP_INVALID_MEDIA_TYPE" ? "Send a JSON body with Content-Type: application/json." : "Malformed JSON body.").toJSON(req.id));55 }56 if (fe.statusCode === 413) return reply.status(413).send(new FetchaError("INVALID_REQUEST", "Request body too large.").toJSON(req.id));57 if (fe.statusCode === 404) return reply.status(404).send(new FetchaError("NOT_FOUND").toJSON(req.id));58 req.log.error({ err }, "unhandled error");59 return reply.status(500).send(new FetchaError("INTERNAL_ERROR").toJSON(req.id));60 });6162 app.setNotFoundHandler((req, reply) => {63 reply.status(404).send(new FetchaError("NOT_FOUND", `No route for ${req.method} ${req.url.split("?")[0]}.`).toJSON(req.id));64 });6566 // ---- Health ---------------------------------------------------------------67 app.get("/health", async () => ({ status: "ok", version: config.version, time: new Date().toISOString() }));68 app.get("/ready", async (_req, reply) => {69 const checks: Record<string, boolean> = {};70 try {71 await db.execute(sql`select 1`);72 checks.database = true;73 } catch {74 checks.database = false;75 }76 checks.cache = await getKV().ping();77 const engine = await getEngine().catch(() => null);78 checks.providers = Boolean(engine && engine.registry.available().length > 0);79 const ok = checks.database && checks.cache && checks.providers;80 return reply.status(ok ? 200 : 503).send({ status: ok ? "ready" : "degraded", checks, available_networks: engine?.registry.availableNetworks() ?? [], browser: engine ? { enabled: engine.browserEnabled, launched: engine.browser.status().launched } : null });81 });8283 // ---- Public API v1 ----------------------------------------------------------84 const clientIp = (req: { headers: Record<string, unknown>; ip: string }) => ((req.headers["x-forwarded-for"] as string | undefined)?.split(",")[0]?.trim() ?? req.ip) || null;8586 app.post("/v1/fetch", async (req, reply) => {87 const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);88 requireScope(principal, "fetch:execute");89 const source = /fetcha-sdk/i.test(String(req.headers["user-agent"] ?? "")) ? "sdk" : "api";90 const out = await handleFetch({ principal, body: req.body, source, clientIp: clientIp(req), userAgent: (req.headers["user-agent"] as string | undefined) ?? null });91 reply.header("x-fetcha-request-id", out.requestId);92 return reply.status(out.status).send(out.body);93 });9495 app.post("/v1/sessions", async (req) => {96 const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);97 requireScope(principal, "sessions:write");98 return createSession(principal, req.body, req.headers["idempotency-key"] as string | undefined);99 });100 app.get("/v1/sessions", async (req) => {101 const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);102 return listSessions(principal);103 });104 app.get<{ Params: { id: string } }>("/v1/sessions/:id", async (req) => {105 const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);106 return getSession(principal, req.params.id);107 });108 app.delete<{ Params: { id: string } }>("/v1/sessions/:id", async (req) => {109 const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);110 requireScope(principal, "sessions:write");111 return closeSession(principal, req.params.id);112 });113114 app.get("/v1/me", async (req) => {115 const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);116 return meResponse(principal);117 });118 app.get("/v1/usage", async (req) => {119 const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);120 requireScope(principal, "usage:read");121 return usageResponse(principal);122 });123124 await registerCrawlRoutes(app);125126 // Explicitly unavailable surfaces (never fake functionality).127 app.post("/v1/browser", async () => {128 throw new FetchaError("BROWSER_UNAVAILABLE", "Scripted browser actions are not available yet. Rendered fetches are: send `browser: true` to POST /v1/fetch.");129 });130 app.post("/v1/extract", async () => {131 throw new FetchaError("INVALID_REQUEST", "Structured extraction is not yet available. Use `format: \"markdown\"` on POST /v1/fetch in the meantime.");132 });133134 await registerInternalRoutes(app);135 await registerInternalCrawlRoutes(app);136 return app;137}138139async function main() {140 assertProductionConfig();141 const app = await buildServer();142 const engine = await getEngine();143 app.log.info({ providers: engine.registry.available().map((p) => p.id), networks: engine.registry.availableNetworks() }, "providers loaded");144 const stopHealth = startHealthLoop(engine, app.log);145 const stopCrawl = startCrawlWorker(app.log);146 app.log.info({ browser: engine.browser.status() }, "managed browser pool ready (lazy launch)");147148 const shutdown = async (signal: string) => {149 app.log.info(`${signal} received, shutting down`);150 stopHealth();151 await Promise.race([stopCrawl(), new Promise((r) => setTimeout(r, 8000))]);152 await app.close();153 await Promise.all([closeAllDispatchers(), closeBrowserPool(), closeKV(), closeDb()]);154 process.exit(0);155 };156 process.on("SIGINT", () => void shutdown("SIGINT"));157 process.on("SIGTERM", () => void shutdown("SIGTERM"));158159 await app.listen({ port: config.port, host: config.host });160 app.log.info(`Fetcha API listening on http://${config.host}:${config.port}`);161}162163const isMain = process.argv[1] && /server\.ts$|server\.js$/.test(process.argv[1]);164if (isMain) {165 main().catch((e) => {166 console.error(e);167 process.exit(1);168 });169}170