// author: simon-pierre boucher import { randomUUID } from "node:crypto"; import { fileURLToPath } from "node:url"; import Fastify, { type FastifyInstance } from "fastify"; import compress from "@fastify/compress"; import fastifyStatic from "@fastify/static"; import { tendrilError } from "@tendril/shared"; import { mapSite } from "@tendril/frontier"; import { toApiError } from "./errors.js"; import { MapRequestSchema, ScrapeRequestSchema } from "./schemas.js"; import { runScrape } from "./scrape.js"; function requestId(): string { return `tdr_req_${randomUUID()}`; } export function buildApp(): FastifyInstance { const app = Fastify({ logger: { level: process.env["LOG_LEVEL"] ?? "info" }, genReqId: requestId, bodyLimit: 2 * 1024 * 1024, }); app.register(compress, { global: true, encodings: ["br", "gzip", "deflate"], threshold: 1024 }); app.register(fastifyStatic, { root: fileURLToPath(new URL("../public", import.meta.url)), prefix: "/", index: ["index.html"], cacheControl: true, maxAge: "5m", }); app.addHook("onSend", async (req, reply) => { reply.header("X-Request-Id", req.id); }); app.get("/healthz", async () => ({ success: true, data: { status: "ok" } })); app.post("/v1/scrape", async (req, reply) => { const parsed = ScrapeRequestSchema.safeParse(req.body); if (!parsed.success) { const { status, body } = toApiError( tendrilError("ERR_INVALID_URL", { message: "Invalid request body", details: { issues: parsed.error.issues.map((i) => ({ path: i.path, message: i.message })) }, }), req.id, ); return reply.status(status).send(body); } const result = await runScrape(parsed.data); if (!result.ok) { const { status, body } = toApiError(result.error, req.id); req.log.warn({ code: result.error.code, url: parsed.data.url }, "scrape failed"); return reply.status(status).send(body); } return reply.status(200).send({ success: true, data: result.value, requestId: req.id }); }); app.post("/v1/map", async (req, reply) => { const parsed = MapRequestSchema.safeParse(req.body); if (!parsed.success) { const { status, body } = toApiError( tendrilError("ERR_INVALID_URL", { message: "Invalid request body", details: { issues: parsed.error.issues.map((i) => ({ path: i.path, message: i.message })) }, }), req.id, ); return reply.status(status).send(body); } const { url, search, limit, includeSubdomains, sitemapOnly, ignoreSitemap } = parsed.data; const result = await mapSite(url, { ...(search !== undefined ? { search } : {}), limit, includeSubdomains, sitemapOnly, ignoreSitemap, }); if (!result.ok) { const { status, body } = toApiError(result.error, req.id); return reply.status(status).send(body); } return reply .status(200) .send({ success: true, data: { links: result.value.links, count: result.value.links.length }, requestId: req.id }); }); app.get("/v1/status", async () => ({ success: true, data: { tiers: { http: { available: true }, webkit: { available: false }, safari: { available: false } }, pipelineVersion: "0.1.0", }, })); app.setErrorHandler((error, req, reply) => { req.log.error({ err: error }, "unhandled error"); const { status, body } = toApiError( tendrilError("ERR_INTERNAL", { message: "Internal error" }), req.id, ); return reply.status(status).send(body); }); return app; }