spb/tendril Public

1// author: simon-pierre boucher <contact@spboucher.ai>2export interface RobotsRule {3  readonly allow: boolean;4  readonly pattern: string;5}67export interface RobotsGroup {8  readonly agents: string[];9  rules: RobotsRule[];10  crawlDelay?: number;11}1213export interface Robots {14  readonly groups: RobotsGroup[];15  readonly sitemaps: string[];16}1718/**19 * Parse robots.txt (RFC 9309 subset, §10.1): User-agent groups, Allow/Disallow20 * with `*`/`$` wildcards, Crawl-delay, and global Sitemap directives. Consecutive21 * User-agent lines share the following rule block.22 */23export function parseRobots(text: string): Robots {24  const groups: RobotsGroup[] = [];25  const sitemaps: string[] = [];26  let current: RobotsGroup | null = null;27  let expectingAgent = false;2829  for (const rawLine of text.split(/\r?\n/)) {30    const line = rawLine.replace(/#.*$/, "").trim();31    if (line === "") continue;32    const idx = line.indexOf(":");33    if (idx === -1) continue;34    const field = line.slice(0, idx).trim().toLowerCase();35    const value = line.slice(idx + 1).trim();3637    switch (field) {38      case "user-agent": {39        if (current === null || !expectingAgent) {40          current = { agents: [], rules: [] };41          groups.push(current);42        }43        current.agents.push(value.toLowerCase());44        expectingAgent = true;45        break;46      }47      case "allow":48      case "disallow": {49        if (current === null) {50          current = { agents: ["*"], rules: [] };51          groups.push(current);52        }53        expectingAgent = false;54        current.rules.push({ allow: field === "allow", pattern: value });55        break;56      }57      case "crawl-delay": {58        if (current !== null) {59          const n = Number(value);60          if (Number.isFinite(n)) current.crawlDelay = n;61        }62        expectingAgent = false;63        break;64      }65      case "sitemap": {66        if (value !== "") sitemaps.push(value);67        break;68      }69      default:70        break;71    }72  }7374  return { groups, sitemaps };75}7677function patternToRegex(pattern: string): RegExp {78  let anchoredEnd = false;79  let p = pattern;80  if (p.endsWith("$")) {81    anchoredEnd = true;82    p = p.slice(0, -1);83  }84  const escaped = p85    .split("*")86    .map((seg) => seg.replace(/[.+?^${}()|[\]\\]/g, "\\$&"))87    .join(".*");88  return new RegExp("^" + escaped + (anchoredEnd ? "$" : ""));89}9091function selectGroup(robots: Robots, userAgent: string): RobotsGroup | null {92  const ua = userAgent.toLowerCase();93  let best: RobotsGroup | null = null;94  let bestLen = -1;95  let star: RobotsGroup | null = null;96  for (const group of robots.groups) {97    for (const agent of group.agents) {98      if (agent === "*") {99        star = group;100        continue;101      }102      if (ua.includes(agent) && agent.length > bestLen) {103        best = group;104        bestLen = agent.length;105      }106    }107  }108  return best ?? star;109}110111/**112 * Decide whether a path is crawlable for our agent. Longest matching rule wins;113 * ties resolve to Allow (RFC 9309). No matching group, or an empty rule set,114 * means allow-all.115 */116export function isAllowed(robots: Robots, path: string, userAgent = "Tendril"): boolean {117  const group = selectGroup(robots, userAgent);118  if (group === null || group.rules.length === 0) return true;119120  let decision = true;121  let bestLen = -1;122  for (const rule of group.rules) {123    if (rule.pattern === "") {124      if (!rule.allow) continue;125      continue;126    }127    if (patternToRegex(rule.pattern).test(path) && rule.pattern.length > bestLen) {128      bestLen = rule.pattern.length;129      decision = rule.allow;130    } else if (patternToRegex(rule.pattern).test(path) && rule.pattern.length === bestLen && rule.allow) {131      decision = true;132    }133  }134  return decision;135}136137export function crawlDelay(robots: Robots, userAgent = "Tendril"): number | undefined {138  return selectGroup(robots, userAgent)?.crawlDelay;139}140