/** Minimal robots.txt parser (RFC 9309 semantics: longest-match, allow wins on tie). */
export interface RobotsRules {
  allow: string[];
  disallow: string[];
  crawlDelayMs: number | null;
  sitemaps: string[];
}

export function parseRobots(txt: string, userAgent = "fetchabot"): RobotsRules {
  const ua = userAgent.toLowerCase();
  const groups: Array<{ agents: string[]; allow: string[]; disallow: string[]; crawlDelay: number | null }> = [];
  const sitemaps: string[] = [];
  let cur: (typeof groups)[number] | null = null;
  let lastWasAgent = false;
  for (const rawLine of txt.split(/\r?\n/)) {
    const line = rawLine.replace(/#.*$/, "").trim();
    if (!line) continue;
    const idx = line.indexOf(":");
    if (idx === -1) continue;
    const key = line.slice(0, idx).trim().toLowerCase();
    const value = line.slice(idx + 1).trim();
    if (key === "sitemap") {
      if (value) sitemaps.push(value);
      continue;
    }
    if (key === "user-agent") {
      if (!cur || !lastWasAgent) {
        cur = { agents: [], allow: [], disallow: [], crawlDelay: null };
        groups.push(cur);
      }
      cur.agents.push(value.toLowerCase());
      lastWasAgent = true;
      continue;
    }
    lastWasAgent = false;
    if (!cur) continue;
    if (key === "allow") cur.allow.push(value);
    else if (key === "disallow") cur.disallow.push(value);
    else if (key === "crawl-delay") {
      const n = Number(value);
      if (Number.isFinite(n)) cur.crawlDelay = n;
    }
  }
  // Pick the most specific group: exact UA token match, else "*".
  let chosen = groups.find((g) => g.agents.some((a) => a !== "*" && ua.includes(a)));
  if (!chosen) chosen = groups.find((g) => g.agents.includes("*"));
  return {
    allow: chosen?.allow ?? [],
    disallow: chosen?.disallow.filter(Boolean) ?? [],
    crawlDelayMs: chosen?.crawlDelay !== null && chosen?.crawlDelay !== undefined ? Math.min(30_000, Math.round(chosen.crawlDelay * 1000)) : null,
    sitemaps,
  };
}

function patternToRegex(p: string): RegExp {
  let anchored = false;
  let pat = p;
  if (pat.endsWith("$")) {
    anchored = true;
    pat = pat.slice(0, -1);
  }
  const esc = pat.replace(/[.+^${}()|[\]\\?]/g, "\\$&").replace(/\*/g, ".*");
  return new RegExp(`^${esc}${anchored ? "$" : ""}`);
}

export function robotsAllows(rules: RobotsRules, url: string): boolean {
  let path: string;
  try {
    const u = new URL(url);
    path = u.pathname + u.search;
  } catch {
    return false;
  }
  let best: { allow: boolean; len: number } | null = null;
  for (const p of rules.allow) {
    if (p && patternToRegex(p).test(path) && (!best || p.length > best.len || (p.length === best.len && !best.allow))) best = { allow: true, len: p.length };
  }
  for (const p of rules.disallow) {
    if (p && patternToRegex(p).test(path) && (!best || p.length > best.len)) best = { allow: false, len: p.length };
  }
  return best ? best.allow : true;
}

/** Extract <loc> URLs from a sitemap or sitemap index. Returns { urls, sitemaps }. */
export function parseSitemap(xml: string): { urls: string[]; sitemaps: string[] } {
  const urls: string[] = [];
  const sitemaps: string[] = [];
  const isIndex = /<sitemapindex/i.test(xml);
  for (const m of xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)) {
    const loc = m[1]!.replace(/&amp;/g, "&").trim();
    (isIndex ? sitemaps : urls).push(loc);
  }
  if (!urls.length && !sitemaps.length) {
    // plain-text sitemap
    for (const line of xml.split(/\r?\n/)) {
      const t = line.trim();
      if (/^https?:\/\//i.test(t)) urls.push(t);
    }
  }
  return { urls, sitemaps };
}