spb/tendril Public

1// author: simon-pierre boucher <contact@spboucher.ai>2import { httpFetch } from "@tendril/fetcher-http";3import { parseRobots, type Robots } from "./robots.js";45const ALLOW_ALL: Robots = { groups: [], sitemaps: [] };6const DISALLOW_ALL: Robots = { groups: [{ agents: ["*"], rules: [{ allow: false, pattern: "/" }] }], sitemaps: [] };78const OK_TTL_MS = 24 * 60 * 60 * 1000;9const DISALLOW_TTL_MS = 60 * 60 * 1000;1011interface Entry {12  robots: Robots;13  expiresAt: number;14}1516const cache = new Map<string, Entry>();1718/**19 * Fetch and cache robots.txt for an origin (§10.1). Cached 24h. A 5xx is treated20 * as disallow-all for 1h (per the spec's intent); a 4xx or network failure is21 * treated as allow-all. `origin` must be a scheme://host[:port] string.22 */23export async function getRobots(origin: string): Promise<Robots> {24  const now = Date.now();25  const cached = cache.get(origin);26  if (cached !== undefined && cached.expiresAt > now) return cached.robots;2728  const res = await httpFetch(`${origin}/robots.txt`, { timeout: 8_000 });29  let robots: Robots;30  let ttl = OK_TTL_MS;3132  if (!res.ok) {33    robots = ALLOW_ALL;34  } else if (res.value.status >= 500) {35    robots = DISALLOW_ALL;36    ttl = DISALLOW_TTL_MS;37  } else if (res.value.status >= 400) {38    robots = ALLOW_ALL;39  } else {40    robots = parseRobots(res.value.body);41  }4243  cache.set(origin, { robots, expiresAt: now + ttl });44  return robots;45}4647export function clearRobotsCache(): void {48  cache.clear();49}50