spb/fetcha

1/** Minimal robots.txt parser (RFC 9309 semantics: longest-match, allow wins on tie). */2export interface RobotsRules {3  allow: string[];4  disallow: string[];5  crawlDelayMs: number | null;6  sitemaps: string[];7}89export function parseRobots(txt: string, userAgent = "fetchabot"): RobotsRules {10  const ua = userAgent.toLowerCase();11  const groups: Array<{ agents: string[]; allow: string[]; disallow: string[]; crawlDelay: number | null }> = [];12  const sitemaps: string[] = [];13  let cur: (typeof groups)[number] | null = null;14  let lastWasAgent = false;15  for (const rawLine of txt.split(/\r?\n/)) {16    const line = rawLine.replace(/#.*$/, "").trim();17    if (!line) continue;18    const idx = line.indexOf(":");19    if (idx === -1) continue;20    const key = line.slice(0, idx).trim().toLowerCase();21    const value = line.slice(idx + 1).trim();22    if (key === "sitemap") {23      if (value) sitemaps.push(value);24      continue;25    }26    if (key === "user-agent") {27      if (!cur || !lastWasAgent) {28        cur = { agents: [], allow: [], disallow: [], crawlDelay: null };29        groups.push(cur);30      }31      cur.agents.push(value.toLowerCase());32      lastWasAgent = true;33      continue;34    }35    lastWasAgent = false;36    if (!cur) continue;37    if (key === "allow") cur.allow.push(value);38    else if (key === "disallow") cur.disallow.push(value);39    else if (key === "crawl-delay") {40      const n = Number(value);41      if (Number.isFinite(n)) cur.crawlDelay = n;42    }43  }44  // Pick the most specific group: exact UA token match, else "*".45  let chosen = groups.find((g) => g.agents.some((a) => a !== "*" && ua.includes(a)));46  if (!chosen) chosen = groups.find((g) => g.agents.includes("*"));47  return {48    allow: chosen?.allow ?? [],49    disallow: chosen?.disallow.filter(Boolean) ?? [],50    crawlDelayMs: chosen?.crawlDelay !== null && chosen?.crawlDelay !== undefined ? Math.min(30_000, Math.round(chosen.crawlDelay * 1000)) : null,51    sitemaps,52  };53}5455function patternToRegex(p: string): RegExp {56  let anchored = false;57  let pat = p;58  if (pat.endsWith("$")) {59    anchored = true;60    pat = pat.slice(0, -1);61  }62  const esc = pat.replace(/[.+^${}()|[\]\\?]/g, "\\$&").replace(/\*/g, ".*");63  return new RegExp(`^${esc}${anchored ? "$" : ""}`);64}6566export function robotsAllows(rules: RobotsRules, url: string): boolean {67  let path: string;68  try {69    const u = new URL(url);70    path = u.pathname + u.search;71  } catch {72    return false;73  }74  let best: { allow: boolean; len: number } | null = null;75  for (const p of rules.allow) {76    if (p && patternToRegex(p).test(path) && (!best || p.length > best.len || (p.length === best.len && !best.allow))) best = { allow: true, len: p.length };77  }78  for (const p of rules.disallow) {79    if (p && patternToRegex(p).test(path) && (!best || p.length > best.len)) best = { allow: false, len: p.length };80  }81  return best ? best.allow : true;82}8384/** Extract <loc> URLs from a sitemap or sitemap index. Returns { urls, sitemaps }. */85export function parseSitemap(xml: string): { urls: string[]; sitemaps: string[] } {86  const urls: string[] = [];87  const sitemaps: string[] = [];88  const isIndex = /<sitemapindex/i.test(xml);89  for (const m of xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)) {90    const loc = m[1]!.replace(/&amp;/g, "&").trim();91    (isIndex ? sitemaps : urls).push(loc);92  }93  if (!urls.length && !sitemaps.length) {94    // plain-text sitemap95    for (const line of xml.split(/\r?\n/)) {96      const t = line.trim();97      if (/^https?:\/\//i.test(t)) urls.push(t);98    }99  }100  return { urls, sitemaps };101}102