import type { FastifyInstance } from "fastify"; import { z } from "zod"; import { FetchaError } from "@fetcha/core"; import { authenticateApiKey, principalForProject, requireScope } from "../auth"; import { cancelCrawl, createCrawl, getCrawl, listCrawlPages, listCrawls, mapSite } from "../services/crawl"; import { getEngine } from "../services/engine"; const pagesQuery = z.object({ cursor: z.string().optional(), limit: z.coerce.number().int().min(1).max(500).optional(), status: z.enum(["pending", "success", "blocked", "failed", "skipped"]).optional(), include_content: z .union([z.literal("true"), z.literal("false"), z.literal("1"), z.literal("0")]) .optional() .transform((v) => (v === undefined ? undefined : v === "true" || v === "1")), }); const clientIp = (req: { headers: Record; ip: string }) => ((req.headers["x-forwarded-for"] as string | undefined)?.split(",")[0]?.trim() ?? req.ip) || null; const sourceOf = (req: { headers: Record }) => (/fetcha-sdk/i.test(String(req.headers["user-agent"] ?? "")) ? "sdk" : "api") as "sdk" | "api"; /** Public crawl & map endpoints. */ export async function registerCrawlRoutes(app: FastifyInstance) { app.post("/v1/crawl", async (req, reply) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); requireScope(principal, "fetch:execute"); const job = await createCrawl(principal, req.body, sourceOf(req)); return reply.status(202).send(job); }); app.get<{ Querystring: { limit?: string } }>("/v1/crawl", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); return listCrawls(principal, Number(req.query.limit ?? 50) || 50); }); app.get<{ Params: { id: string } }>("/v1/crawl/:id", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); return getCrawl(principal, req.params.id); }); app.get<{ Params: { id: string }; Querystring: Record }>("/v1/crawl/:id/pages", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); const q = pagesQuery.safeParse(req.query); if (!q.success) throw new FetchaError("INVALID_REQUEST", "Invalid query parameters.", { details: { issues: q.error.issues.map((i) => ({ path: i.path.join("."), message: i.message })) } }); return listCrawlPages(principal, req.params.id, q.data); }); app.delete<{ Params: { id: string } }>("/v1/crawl/:id", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); requireScope(principal, "fetch:execute"); return cancelCrawl(principal, req.params.id); }); app.post("/v1/map", async (req) => { const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined); requireScope(principal, "fetch:execute"); return mapSite(principal, req.body, sourceOf(req), clientIp(req)); }); } /** Internal (dashboard) crawl routes; the service-token check is applied by the internal onRequest hook. */ export async function registerInternalCrawlRoutes(app: FastifyInstance) { const bodySchema = z.object({ project_id: z.string(), user_id: z.string(), options: z.unknown().optional() }); type Q = { project_id: string; user_id: string; limit?: string; cursor?: string; status?: string; include_content?: string }; app.post("/internal/crawls", async (req, reply) => { const parsed = bodySchema.safeParse(req.body); if (!parsed.success) throw new FetchaError("INVALID_REQUEST", "project_id, user_id and options are required."); const principal = await principalForProject(parsed.data.project_id, parsed.data.user_id); const job = await createCrawl(principal, parsed.data.options ?? {}, "playground"); return reply.status(202).send(job); }); app.get<{ Querystring: Q }>("/internal/crawls", async (req) => { const principal = await principalForProject(req.query.project_id, req.query.user_id); return listCrawls(principal, Number(req.query.limit ?? 50) || 50); }); app.get<{ Params: { id: string }; Querystring: Q }>("/internal/crawls/:id", async (req) => { const principal = await principalForProject(req.query.project_id, req.query.user_id); return getCrawl(principal, req.params.id); }); app.get<{ Params: { id: string }; Querystring: Q }>("/internal/crawls/:id/pages", async (req) => { const principal = await principalForProject(req.query.project_id, req.query.user_id); const q = pagesQuery.safeParse(req.query); if (!q.success) throw new FetchaError("INVALID_REQUEST", "Invalid query parameters."); return listCrawlPages(principal, req.params.id, q.data); }); app.delete<{ Params: { id: string }; Querystring: Q }>("/internal/crawls/:id", async (req) => { const principal = await principalForProject(req.query.project_id, req.query.user_id); return cancelCrawl(principal, req.params.id); }); app.post("/internal/map", async (req) => { const parsed = bodySchema.safeParse(req.body); if (!parsed.success) throw new FetchaError("INVALID_REQUEST", "project_id, user_id and options are required."); const principal = await principalForProject(parsed.data.project_id, parsed.data.user_id); return mapSite(principal, parsed.data.options ?? {}, "playground", (req.headers["x-forwarded-for"] as string | undefined)?.split(",")[0]?.trim() ?? req.ip); }); app.get("/internal/browser", async () => { const engine = await getEngine(); const s = engine.browser.status(); return { ...s, enabled: engine.browserEnabled, flag_enabled: engine.browserEnabled || !engine.browser.enabled ? engine.browserEnabled : false }; }); }