import Fastify from "fastify"; import cors from "@fastify/cors"; import { FetchaError, isFetchaError, newId } from "@fetcha/core"; import { closeDb, db, sql } from "@fetcha/db"; import { closeAllDispatchers } from "@fetcha/providers"; import { closeBrowserPool } from "@fetcha/browser"; import { authenticateApiKey, requireScope } from "./auth"; import { assertProductionConfig, config } from "./config"; import { closeKV, getKV } from "./redis"; import { handleFetch } from "./routes/fetch"; import { registerInternalRoutes } from "./routes/internal"; import { registerCrawlRoutes, registerInternalCrawlRoutes } from "./routes/crawl"; import { startCrawlWorker } from "./services/crawl"; import { meResponse, usageResponse } from "./routes/misc"; import { closeSession, createSession, getSession, listSessions } from "./routes/sessions"; import { getEngine, startHealthLoop } from "./services/engine"; export async function buildServer() { const app = Fastify({ logger: { level: config.logLevel, ...(config.env !== "production" ? { transport: { target: "pino-pretty", options: { colorize: true, translateTime: "HH:MM:ss" } } } : {}), redact: ["req.headers.authorization", "req.headers['x-api-key']", "req.headers.cookie"], }, trustProxy: true, bodyLimit: 4 * 1024 * 1024, genReqId: () => newId("req"), disableRequestLogging: config.env === "production", }); await app.register(cors, { origin: true, methods: ["GET", "POST", "DELETE", "OPTIONS"], allowedHeaders: ["authorization", "content-type", "x-api-key", "idempotency-key"], exposedHeaders: ["x-fetcha-request-id", "x-request-id"], }); app.addHook("onSend", async (req, reply) => { if (!reply.getHeader("x-fetcha-request-id")) reply.header("x-fetcha-request-id", req.id); reply.header("x-fetcha-version", config.version); reply.header("cache-control", "no-store"); }); app.setErrorHandler((err: unknown, req, reply) => { if (isFetchaError(err)) { const e = err as FetchaError; const rid = e.requestId ?? req.id; reply.header("x-fetcha-request-id", rid); if (e.code === "RATE_LIMITED" && e.details?.retry_after_ms) reply.header("retry-after", String(Math.ceil(Number(e.details.retry_after_ms) / 1000))); return reply.status(e.status).send(e.toJSON(rid)); } const fe = err as { statusCode?: number; validation?: unknown; message?: string; code?: string }; if (fe.statusCode === 400 || fe.validation || fe.code === "FST_ERR_CTP_INVALID_MEDIA_TYPE" || fe.code === "FST_ERR_CTP_EMPTY_JSON_BODY") { return reply.status(400).send(new FetchaError("INVALID_REQUEST", fe.code === "FST_ERR_CTP_INVALID_MEDIA_TYPE" ? "Send a JSON body with Content-Type: application/json." : "Malformed JSON body.").toJSON(req.id)); } if (fe.statusCode === 413) return reply.status(413).send(new FetchaError("INVALID_REQUEST", "Request body too large.").toJSON(req.id)); if (fe.statusCode === 404) return reply.status(404).send(new FetchaError("NOT_FOUND").toJSON(req.id)); req.log.error({ err }, "unhandled error"); return reply.status(500).send(new FetchaError("INTERNAL_ERROR").toJSON(req.id)); }); app.setNotFoundHandler((req, reply) => { reply.status(404).send(new FetchaError("NOT_FOUND", `No route for ${req.method} ${req.url.split("?")[0]}.`).toJSON(req.id)); }); // ---- Health --------------------------------------------------------------- app.get("/health", async () => ({ status: "ok", version: config.version, time: new Date().toISOString() })); app.get("/ready", async (_req, reply) => { const checks: Record = {}; try { await db.execute(sql`select 1`); checks.database = true; } catch { checks.database = false; } checks.cache = await getKV().ping(); const engine = await getEngine().catch(() => null); checks.providers = Boolean(engine && engine.registry.available().length > 0); const ok = checks.database && checks.cache && checks.providers; return reply.status(ok ? 200 : 503).send({ status: ok ? "ready" : "degraded", checks, available_networks: engine?.registry.availableNetworks() ?? [], browser: engine ? { enabled: engine.browserEnabled, launched: engine.browser.status().launched } : null }); }); // ---- Public API v1 ---------------------------------------------------------- const clientIp = (req: { headers: Record; ip: string }) => ((req.headers["x-forwarded-for"] as string | undefined)?.split(",")[0]?.trim() ?? req.ip) || null; app.post("/v1/fetch", async (req, reply) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); requireScope(principal, "fetch:execute"); const source = /fetcha-sdk/i.test(String(req.headers["user-agent"] ?? "")) ? "sdk" : "api"; const out = await handleFetch({ principal, body: req.body, source, clientIp: clientIp(req), userAgent: (req.headers["user-agent"] as string | undefined) ?? null }); reply.header("x-fetcha-request-id", out.requestId); return reply.status(out.status).send(out.body); }); app.post("/v1/sessions", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); requireScope(principal, "sessions:write"); return createSession(principal, req.body, req.headers["idempotency-key"] as string | undefined); }); app.get("/v1/sessions", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); return listSessions(principal); }); app.get<{ Params: { id: string } }>("/v1/sessions/:id", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); return getSession(principal, req.params.id); }); app.delete<{ Params: { id: string } }>("/v1/sessions/:id", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); requireScope(principal, "sessions:write"); return closeSession(principal, req.params.id); }); app.get("/v1/me", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); return meResponse(principal); }); app.get("/v1/usage", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); requireScope(principal, "usage:read"); return usageResponse(principal); }); await registerCrawlRoutes(app); // Explicitly unavailable surfaces (never fake functionality). app.post("/v1/browser", async () => { throw new FetchaError("BROWSER_UNAVAILABLE", "Scripted browser actions are not available yet. Rendered fetches are: send `browser: true` to POST /v1/fetch."); }); app.post("/v1/extract", async () => { throw new FetchaError("INVALID_REQUEST", "Structured extraction is not yet available. Use `format: \"markdown\"` on POST /v1/fetch in the meantime."); }); await registerInternalRoutes(app); await registerInternalCrawlRoutes(app); return app; } async function main() { assertProductionConfig(); const app = await buildServer(); const engine = await getEngine(); app.log.info({ providers: engine.registry.available().map((p) => p.id), networks: engine.registry.availableNetworks() }, "providers loaded"); const stopHealth = startHealthLoop(engine, app.log); const stopCrawl = startCrawlWorker(app.log); app.log.info({ browser: engine.browser.status() }, "managed browser pool ready (lazy launch)"); const shutdown = async (signal: string) => { app.log.info(`${signal} received, shutting down`); stopHealth(); await Promise.race([stopCrawl(), new Promise((r) => setTimeout(r, 8000))]); await app.close(); await Promise.all([closeAllDispatchers(), closeBrowserPool(), closeKV(), closeDb()]); process.exit(0); }; process.on("SIGINT", () => void shutdown("SIGINT")); process.on("SIGTERM", () => void shutdown("SIGTERM")); await app.listen({ port: config.port, host: config.host }); app.log.info(`Fetcha API listening on http://${config.host}:${config.port}`); } const isMain = process.argv[1] && /server\.ts$|server\.js$/.test(process.argv[1]); if (isMain) { main().catch((e) => { console.error(e); process.exit(1); }); }