spb/tendril Public

1// author: simon-pierre boucher <contact@spboucher.ai>2import { randomUUID } from "node:crypto";3import { fileURLToPath } from "node:url";4import Fastify, { type FastifyInstance } from "fastify";5import compress from "@fastify/compress";6import fastifyStatic from "@fastify/static";7import { tendrilError } from "@tendril/shared";8import { mapSite } from "@tendril/frontier";9import { toApiError } from "./errors.js";10import { MapRequestSchema, ScrapeRequestSchema } from "./schemas.js";11import { runScrape } from "./scrape.js";1213function requestId(): string {14  return `tdr_req_${randomUUID()}`;15}1617export function buildApp(): FastifyInstance {18  const app = Fastify({19    logger: { level: process.env["LOG_LEVEL"] ?? "info" },20    genReqId: requestId,21    bodyLimit: 2 * 1024 * 1024,22  });2324  app.register(compress, { global: true, encodings: ["br", "gzip", "deflate"], threshold: 1024 });2526  app.register(fastifyStatic, {27    root: fileURLToPath(new URL("../public", import.meta.url)),28    prefix: "/",29    index: ["index.html"],30    cacheControl: true,31    maxAge: "5m",32  });3334  app.addHook("onSend", async (req, reply) => {35    reply.header("X-Request-Id", req.id);36  });3738  app.get("/healthz", async () => ({ success: true, data: { status: "ok" } }));3940  app.post("/v1/scrape", async (req, reply) => {41    const parsed = ScrapeRequestSchema.safeParse(req.body);42    if (!parsed.success) {43      const { status, body } = toApiError(44        tendrilError("ERR_INVALID_URL", {45          message: "Invalid request body",46          details: { issues: parsed.error.issues.map((i) => ({ path: i.path, message: i.message })) },47        }),48        req.id,49      );50      return reply.status(status).send(body);51    }5253    const result = await runScrape(parsed.data);54    if (!result.ok) {55      const { status, body } = toApiError(result.error, req.id);56      req.log.warn({ code: result.error.code, url: parsed.data.url }, "scrape failed");57      return reply.status(status).send(body);58    }5960    return reply.status(200).send({ success: true, data: result.value, requestId: req.id });61  });6263  app.post("/v1/map", async (req, reply) => {64    const parsed = MapRequestSchema.safeParse(req.body);65    if (!parsed.success) {66      const { status, body } = toApiError(67        tendrilError("ERR_INVALID_URL", {68          message: "Invalid request body",69          details: { issues: parsed.error.issues.map((i) => ({ path: i.path, message: i.message })) },70        }),71        req.id,72      );73      return reply.status(status).send(body);74    }7576    const { url, search, limit, includeSubdomains, sitemapOnly, ignoreSitemap } = parsed.data;77    const result = await mapSite(url, {78      ...(search !== undefined ? { search } : {}),79      limit,80      includeSubdomains,81      sitemapOnly,82      ignoreSitemap,83    });84    if (!result.ok) {85      const { status, body } = toApiError(result.error, req.id);86      return reply.status(status).send(body);87    }88    return reply89      .status(200)90      .send({ success: true, data: { links: result.value.links, count: result.value.links.length }, requestId: req.id });91  });9293  app.get("/v1/status", async () => ({94    success: true,95    data: {96      tiers: { http: { available: true }, webkit: { available: false }, safari: { available: false } },97      pipelineVersion: "0.1.0",98    },99  }));100101  app.setErrorHandler((error, req, reply) => {102    req.log.error({ err: error }, "unhandled error");103    const { status, body } = toApiError(104      tendrilError("ERR_INTERNAL", { message: "Internal error" }),105      req.id,106    );107    return reply.status(status).send(body);108  });109110  return app;111}112