spb/tendril Public

1// author: simon-pierre boucher <contact@spboucher.ai>2import { performance } from "node:perf_hooks";3import { err, ok, tendrilError, type FetchTimings, type Result, type Tier } from "@tendril/shared";4import { shouldEscalate } from "@tendril/router";5import { getRobots, isAllowed } from "@tendril/frontier";6import { httpFetch } from "@tendril/fetcher-http";7import { extract, PIPELINE_VERSION, type ExtractResult } from "@tendril/extract";8import type { ScrapeRequest } from "./schemas.js";910export interface ScrapeData {11  markdown?: string;12  html?: string;13  rawHtml?: string;14  links?: ExtractResult["links"];15  structured?: ExtractResult["structured"];16  metadata: ExtractResult["metadata"] & { statusCode: number; sourceURL: string; pipelineVersion: string };17  tierUsed: Tier;18  cached: boolean;19  timings: FetchTimings;20}2122function nonHtmlMarkdown(contentType: string, body: string): Result<string> {23  const ct = contentType.toLowerCase();24  if (ct.includes("application/json")) {25    try {26      return ok("```json\n" + JSON.stringify(JSON.parse(body), null, 2) + "\n```");27    } catch {28      return ok(body);29    }30  }31  if (ct.includes("text/plain") || ct.includes("text/markdown") || ct.includes("text/csv") || ct.includes("xml")) {32    return ok(body);33  }34  return err(tendrilError("ERR_UNSUPPORTED_TYPE", { details: { contentType } }));35}3637/**38 * Orchestrate a single scrape (§14.1): Tier 0 fetch → escalation decision →39 * deterministic extraction. Tiers 1/2 are not yet built, so a page that demands40 * escalation under `tier: "auto"` fails closed with ERR_TARGET_BLOCKED and the41 * decisive reason, rather than returning a challenge page as if it were content.42 */43export async function runScrape(req: ScrapeRequest): Promise<Result<ScrapeData>> {44  const started = performance.now();45  const timings: FetchTimings = { total: 0, escalations: [] };4647  if (req.respectRobots) {48    let target: URL;49    try {50      target = new URL(req.url);51    } catch {52      return err(tendrilError("ERR_INVALID_URL", { details: { url: req.url } }));53    }54    const robots = await getRobots(target.origin);55    if (!isAllowed(robots, target.pathname + target.search)) {56      return err(tendrilError("ERR_ROBOTS_DENIED", { details: { url: req.url } }));57    }58  }5960  const fetched = await httpFetch(req.url, {61    timeout: req.timeout,62    ...(req.headers !== undefined ? { headers: req.headers } : {}),63    ...(req.maxBytes !== undefined ? { maxBytes: req.maxBytes } : {}),64  });65  if (!fetched.ok) return fetched;66  const page = fetched.value;6768  const decision = shouldEscalate({ status: page.status, contentType: page.contentType, body: page.body });6970  if (decision.action === "non-html") {71    const md = nonHtmlMarkdown(page.contentType, page.body);72    if (!md.ok) return md;73    timings.total = performance.now() - started;74    return ok({75      ...(req.formats.includes("markdown") ? { markdown: md.value } : {}),76      ...(req.formats.includes("rawHtml") ? { rawHtml: page.body } : {}),77      metadata: {78        statusCode: page.status,79        sourceURL: page.finalUrl,80        pipelineVersion: PIPELINE_VERSION,81      },82      tierUsed: "http",83      cached: false,84      timings,85    });86  }8788  if (decision.action === "escalate" && req.tier === "auto") {89    timings.escalations.push({ from: "http", to: "webkit", reason: decision.reason });90    return err(91      tendrilError("ERR_TARGET_BLOCKED", {92        message: "Tier 0 insufficient and higher tiers are not yet available",93        details: { reason: decision.reason, status: page.status },94      }),95    );96  }9798  const extractStart = performance.now();99  const extracted = extract(page.body, {100    url: page.finalUrl,101    onlyMainContent: req.onlyMainContent,102    ...(req.includeTags !== undefined ? { includeTags: req.includeTags } : {}),103    ...(req.excludeTags !== undefined ? { excludeTags: req.excludeTags } : {}),104  });105  if (!extracted.ok) return extracted;106  const e = extracted.value;107  timings.extract = performance.now() - extractStart;108  timings.total = performance.now() - started;109110  return ok({111    ...(req.formats.includes("markdown") ? { markdown: e.markdown } : {}),112    ...(req.formats.includes("html") ? { html: e.html } : {}),113    ...(req.formats.includes("rawHtml") ? { rawHtml: page.body } : {}),114    ...(req.formats.includes("links") ? { links: e.links } : {}),115    ...(req.formats.includes("structured") ? { structured: e.structured } : {}),116    metadata: {117      ...e.metadata,118      statusCode: page.status,119      sourceURL: page.finalUrl,120      pipelineVersion: PIPELINE_VERSION,121    },122    tierUsed: "http",123    cached: false,124    timings,125  });126}127