spb/fetcha

1import type { FastifyInstance } from "fastify";2import { z } from "zod";3import { FetchaError } from "@fetcha/core";4import { authenticateApiKey, principalForProject, requireScope } from "../auth";5import { cancelCrawl, createCrawl, getCrawl, listCrawlPages, listCrawls, mapSite } from "../services/crawl";6import { getEngine } from "../services/engine";78const pagesQuery = z.object({9  cursor: z.string().optional(),10  limit: z.coerce.number().int().min(1).max(500).optional(),11  status: z.enum(["pending", "success", "blocked", "failed", "skipped"]).optional(),12  include_content: z13    .union([z.literal("true"), z.literal("false"), z.literal("1"), z.literal("0")])14    .optional()15    .transform((v) => (v === undefined ? undefined : v === "true" || v === "1")),16});1718const clientIp = (req: { headers: Record<string, unknown>; ip: string }) => ((req.headers["x-forwarded-for"] as string | undefined)?.split(",")[0]?.trim() ?? req.ip) || null;19const sourceOf = (req: { headers: Record<string, unknown> }) => (/fetcha-sdk/i.test(String(req.headers["user-agent"] ?? "")) ? "sdk" : "api") as "sdk" | "api";2021/** Public crawl & map endpoints. */22export async function registerCrawlRoutes(app: FastifyInstance) {23  app.post("/v1/crawl", async (req, reply) => {24    const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);25    requireScope(principal, "fetch:execute");26    const job = await createCrawl(principal, req.body, sourceOf(req));27    return reply.status(202).send(job);28  });29  app.get<{ Querystring: { limit?: string } }>("/v1/crawl", async (req) => {30    const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);31    return listCrawls(principal, Number(req.query.limit ?? 50) || 50);32  });33  app.get<{ Params: { id: string } }>("/v1/crawl/:id", async (req) => {34    const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);35    return getCrawl(principal, req.params.id);36  });37  app.get<{ Params: { id: string }; Querystring: Record<string, string | undefined> }>("/v1/crawl/:id/pages", async (req) => {38    const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);39    const q = pagesQuery.safeParse(req.query);40    if (!q.success) throw new FetchaError("INVALID_REQUEST", "Invalid query parameters.", { details: { issues: q.error.issues.map((i) => ({ path: i.path.join("."), message: i.message })) } });41    return listCrawlPages(principal, req.params.id, q.data);42  });43  app.delete<{ Params: { id: string } }>("/v1/crawl/:id", async (req) => {44    const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);45    requireScope(principal, "fetch:execute");46    return cancelCrawl(principal, req.params.id);47  });48  app.post("/v1/map", async (req) => {49    const principal = await authenticateApiKey(req.headers.authorization, req.headers["x-api-key"] as string | undefined);50    requireScope(principal, "fetch:execute");51    return mapSite(principal, req.body, sourceOf(req), clientIp(req));52  });53}5455/** Internal (dashboard) crawl routes; the service-token check is applied by the internal onRequest hook. */56export async function registerInternalCrawlRoutes(app: FastifyInstance) {57  const bodySchema = z.object({ project_id: z.string(), user_id: z.string(), options: z.unknown().optional() });58  type Q = { project_id: string; user_id: string; limit?: string; cursor?: string; status?: string; include_content?: string };5960  app.post("/internal/crawls", async (req, reply) => {61    const parsed = bodySchema.safeParse(req.body);62    if (!parsed.success) throw new FetchaError("INVALID_REQUEST", "project_id, user_id and options are required.");63    const principal = await principalForProject(parsed.data.project_id, parsed.data.user_id);64    const job = await createCrawl(principal, parsed.data.options ?? {}, "playground");65    return reply.status(202).send(job);66  });67  app.get<{ Querystring: Q }>("/internal/crawls", async (req) => {68    const principal = await principalForProject(req.query.project_id, req.query.user_id);69    return listCrawls(principal, Number(req.query.limit ?? 50) || 50);70  });71  app.get<{ Params: { id: string }; Querystring: Q }>("/internal/crawls/:id", async (req) => {72    const principal = await principalForProject(req.query.project_id, req.query.user_id);73    return getCrawl(principal, req.params.id);74  });75  app.get<{ Params: { id: string }; Querystring: Q }>("/internal/crawls/:id/pages", async (req) => {76    const principal = await principalForProject(req.query.project_id, req.query.user_id);77    const q = pagesQuery.safeParse(req.query);78    if (!q.success) throw new FetchaError("INVALID_REQUEST", "Invalid query parameters.");79    return listCrawlPages(principal, req.params.id, q.data);80  });81  app.delete<{ Params: { id: string }; Querystring: Q }>("/internal/crawls/:id", async (req) => {82    const principal = await principalForProject(req.query.project_id, req.query.user_id);83    return cancelCrawl(principal, req.params.id);84  });85  app.post("/internal/map", async (req) => {86    const parsed = bodySchema.safeParse(req.body);87    if (!parsed.success) throw new FetchaError("INVALID_REQUEST", "project_id, user_id and options are required.");88    const principal = await principalForProject(parsed.data.project_id, parsed.data.user_id);89    return mapSite(principal, parsed.data.options ?? {}, "playground", (req.headers["x-forwarded-for"] as string | undefined)?.split(",")[0]?.trim() ?? req.ip);90  });91  app.get("/internal/browser", async () => {92    const engine = await getEngine();93    const s = engine.browser.status();94    return { ...s, enabled: engine.browserEnabled, flag_enabled: engine.browserEnabled || !engine.browser.enabled ? engine.browserEnabled : false };95  });96}97