spb/tendril Public

1// author: simon-pierre boucher <contact@spboucher.ai>2import { err, normalizeUrl, ok, tendrilError, type Result } from "@tendril/shared";3import { httpFetch } from "@tendril/fetcher-http";4import { extractLinksFromHtml } from "@tendril/extract";5import { getRobots } from "./robots-cache.js";6import { parseSitemap } from "./sitemap.js";78export type MapSource = "sitemap" | "robots" | "homepage" | "llms" | "index";910export interface MappedLink {11  readonly url: string;12  readonly source: MapSource;13  readonly lastModified?: string;14}1516export interface MapOptions {17  readonly search?: string;18  readonly limit?: number;19  readonly includeSubdomains?: boolean;20  readonly sitemapOnly?: boolean;21  readonly ignoreSitemap?: boolean;22}2324const SITEMAP_URL_CAP = 50_000;25const DEFAULT_LIMIT = 5_000;26const MAX_SITEMAP_FETCHES = 60;2728function apexOf(host: string): string {29  const parts = host.split(".");30  return parts.length <= 2 ? host : parts.slice(-2).join(".");31}3233function hostMatches(host: string, baseHost: string, includeSubdomains: boolean): boolean {34  if (host === baseHost) return true;35  if (!includeSubdomains) return false;36  const apex = apexOf(baseHost);37  return host === apex || host.endsWith("." + apex);38}3940async function collectSitemaps(41  seeds: string[],42  baseHost: string,43  includeSubdomains: boolean,44  add: (url: string, source: MapSource, lastModified?: string) => void,45): Promise<void> {46  const queue = [...seeds];47  const visited = new Set<string>();48  let fetches = 0;49  let collected = 0;5051  while (queue.length > 0 && fetches < MAX_SITEMAP_FETCHES && collected < SITEMAP_URL_CAP) {52    const sm = queue.shift();53    if (sm === undefined) break;54    const key = normalizeUrl(sm);55    const nk = key.ok ? key.value : sm;56    if (visited.has(nk)) continue;57    visited.add(nk);5859    const res = await httpFetch(sm, { timeout: 10_000 });60    fetches++;61    if (!res.ok) continue;62    const parsed = parseSitemap(res.value.body);6364    for (const child of parsed.childSitemaps) queue.push(child);65    for (const u of parsed.urls) {66      if (collected >= SITEMAP_URL_CAP) break;67      let host: string;68      try {69        host = new URL(u.loc).hostname.toLowerCase();70      } catch {71        continue;72      }73      if (!hostMatches(host, baseHost, includeSubdomains)) continue;74      add(u.loc, "sitemap", u.lastModified);75      collected++;76    }77  }78}7980/**81 * Discover a site's URLs without rendering (§14.3): robots.txt Sitemap82 * directives, sitemap.xml (recursing through indexes, capped at 50k), homepage83 * links, and /llms.txt — merged and deduplicated on the normalized URL, with the84 * first-seen source preserved.85 */86export async function mapSite(rawUrl: string, options: MapOptions = {}): Promise<Result<{ links: MappedLink[] }>> {87  let base: URL;88  try {89    base = new URL(rawUrl);90  } catch {91    return err(tendrilError("ERR_INVALID_URL", { details: { url: rawUrl } }));92  }93  const origin = base.origin;94  const baseHost = base.hostname.toLowerCase();95  const includeSubdomains = options.includeSubdomains ?? false;9697  const byKey = new Map<string, MappedLink>();98  const add = (url: string, source: MapSource, lastModified?: string): void => {99    const norm = normalizeUrl(url);100    if (!norm.ok) return;101    if (byKey.has(norm.value)) return;102    byKey.set(norm.value, lastModified !== undefined ? { url, source, lastModified } : { url, source });103  };104105  if (options.ignoreSitemap !== true) {106    const robots = await getRobots(origin);107    const seeds = [...robots.sitemaps, `${origin}/sitemap.xml`];108    await collectSitemaps(seeds, baseHost, includeSubdomains, add);109  }110111  if (options.sitemapOnly !== true) {112    const home = await httpFetch(rawUrl, { timeout: 10_000 });113    if (home.ok) {114      for (const link of extractLinksFromHtml(home.value.body, home.value.finalUrl)) {115        let host: string;116        try {117          host = new URL(link.url).hostname.toLowerCase();118        } catch {119          continue;120        }121        if (hostMatches(host, baseHost, includeSubdomains)) add(link.url, "homepage");122      }123    }124125    const llms = await httpFetch(`${origin}/llms.txt`, { timeout: 8_000 });126    if (llms.ok && llms.value.status < 400) {127      const urlRe = /https?:\/\/[^\s)]+/g;128      let m: RegExpExecArray | null;129      while ((m = urlRe.exec(llms.value.body)) !== null) {130        try {131          if (hostMatches(new URL(m[0]).hostname.toLowerCase(), baseHost, includeSubdomains)) add(m[0], "llms");132        } catch {133          continue;134        }135      }136    }137  }138139  let links = [...byKey.values()];140141  const search = options.search?.trim().toLowerCase();142  if (search !== undefined && search !== "") {143    links = links144      .map((l) => ({ l, pos: l.url.toLowerCase().indexOf(search) }))145      .filter((x) => x.pos !== -1)146      .sort((a, b) => a.pos - b.pos)147      .map((x) => x.l);148  }149150  const limit = options.limit ?? DEFAULT_LIMIT;151  return ok({ links: links.slice(0, limit) });152}153