import { gunzipSync } from 'node:zlib';
import type { CrawlContext } from '../types.js';

/** Sitemap ingestion (SPEC §1): sitemap indexes, url sets, gzip, lastmod filtering. */
export interface SitemapEntry {
  loc: string;
  lastmod: string | null;
  changefreq?: string | null;
  priority?: number | null;
}

export function decodeXml(s: string): string {
  return s.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&apos;/g, "'");
}

/** Parse <urlset> entries. */
export function parseUrlset(xml: string): SitemapEntry[] {
  const out: SitemapEntry[] = [];
  const re = /<url>([\s\S]*?)<\/url>/g;
  let m: RegExpExecArray | null;
  while ((m = re.exec(xml))) {
    const loc = m[1]!.match(/<loc>\s*([^<\s]+)\s*<\/loc>/)?.[1];
    if (!loc) continue;
    const pr = m[1]!.match(/<priority>\s*([\d.]+)\s*<\/priority>/)?.[1];
    out.push({ loc: decodeXml(loc), lastmod: m[1]!.match(/<lastmod>\s*([^<\s]+)\s*<\/lastmod>/)?.[1] ?? null, changefreq: m[1]!.match(/<changefreq>\s*([^<\s]+)\s*<\/changefreq>/)?.[1] ?? null, priority: pr ? Number(pr) : null });
  }
  return out;
}

/** Parse <sitemapindex> entries (child sitemap locations). */
export function parseSitemapIndex(xml: string): SitemapEntry[] {
  const out: SitemapEntry[] = [];
  const re = /<sitemap>([\s\S]*?)<\/sitemap>/g;
  let m: RegExpExecArray | null;
  while ((m = re.exec(xml))) {
    const loc = m[1]!.match(/<loc>\s*([^<\s]+)\s*<\/loc>/)?.[1];
    if (!loc) continue;
    out.push({ loc: decodeXml(loc), lastmod: m[1]!.match(/<lastmod>\s*([^<\s]+)\s*<\/lastmod>/)?.[1] ?? null });
  }
  return out;
}

export function isSitemapIndex(xml: string): boolean {
  return /<sitemapindex[\s>]/i.test(xml.slice(0, 2000));
}

function toText(res: { html: string | null; buffer?: Uint8Array | null }, url: string): string | null {
  if (res.buffer && res.buffer.byteLength) {
    const b = Buffer.from(res.buffer);
    const gz = b.length >= 2 && b[0] === 0x1f && b[1] === 0x8b;
    return (gz || url.endsWith('.gz') ? gunzipSync(b) : b).toString('utf8');
  }
  return res.html;
}

export interface DiscoverOptions {
  /** keep only URLs matching this pattern */
  match?: RegExp;
  /** keep only child sitemaps whose URL matches (sitemap indexes) */
  matchSitemap?: RegExp;
  /** skip entries whose lastmod is older than this */
  since?: Date | null;
  /** cap on URLs returned */
  limit?: number;
  /** cap on child sitemaps fetched */
  maxSitemaps?: number;
  /** newest first when lastmod is present */
  newestFirst?: boolean;
}

/**
 * Discover page URLs from a sitemap (or sitemap index), following children, decompressing .gz and
 * filtering by pattern/date. Uses the connector's routed fetch (budget + stats + politeness).
 */
export async function discoverFromSitemap(ctx: Pick<CrawlContext, 'fetch' | 'anomaly' | 'signal'>, sitemapUrl: string, opts: DiscoverOptions = {}): Promise<SitemapEntry[]> {
  const out: SitemapEntry[] = [];
  const queue = [sitemapUrl];
  let fetched = 0;
  const maxSitemaps = opts.maxSitemaps ?? 50;
  while (queue.length && fetched < maxSitemaps) {
    if (ctx.signal?.aborted) break;
    const url = queue.shift()!;
    fetched++;
    const res = await ctx.fetch(url, { engines: ['api'], responseType: url.endsWith('.gz') ? 'binary' : 'text', minQuality: 0, force: true });
    const xml = res.success ? toText(res, url) : null;
    if (!xml) {
      ctx.anomaly('sitemap_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
      continue;
    }
    if (isSitemapIndex(xml)) {
      let children = parseSitemapIndex(xml);
      if (opts.matchSitemap) children = children.filter((c) => opts.matchSitemap!.test(c.loc));
      if (opts.since) children = children.filter((c) => !c.lastmod || new Date(c.lastmod) >= opts.since!);
      if (opts.newestFirst) children.sort((a, b) => (b.lastmod ?? '').localeCompare(a.lastmod ?? ''));
      queue.push(...children.map((c) => c.loc));
      continue;
    }
    let entries = parseUrlset(xml);
    if (opts.match) entries = entries.filter((e) => opts.match!.test(e.loc));
    if (opts.since) entries = entries.filter((e) => !e.lastmod || new Date(e.lastmod) >= opts.since!);
    out.push(...entries);
    if (opts.limit && out.length >= opts.limit) break;
  }
  if (opts.newestFirst) out.sort((a, b) => (b.lastmod ?? '').localeCompare(a.lastmod ?? ''));
  return opts.limit ? out.slice(0, opts.limit) : out;
}

/** robots.txt: Sitemap: directives + whether a path is disallowed for our UA / '*'. */
export function parseRobots(txt: string, agent = 'rareindexbot'): { sitemaps: string[]; disallow: string[]; allow: string[]; crawlDelay: number | null } {
  const lines = txt.split(/\r?\n/).map((l) => l.replace(/#.*$/, '').trim()).filter(Boolean);
  const sitemaps: string[] = [];
  const groups: Array<{ agents: string[]; allow: string[]; disallow: string[]; delay: number | null }> = [];
  let cur: (typeof groups)[number] | null = null;
  for (const l of lines) {
    const [kRaw, ...rest] = l.split(':');
    const k = kRaw!.trim().toLowerCase();
    const v = rest.join(':').trim();
    if (k === 'sitemap') sitemaps.push(v);
    else if (k === 'user-agent') {
      if (!cur || cur.allow.length || cur.disallow.length) {
        cur = { agents: [], allow: [], disallow: [], delay: null };
        groups.push(cur);
      }
      cur.agents.push(v.toLowerCase());
    } else if (cur && k === 'disallow') cur.disallow.push(v);
    else if (cur && k === 'allow') cur.allow.push(v);
    else if (cur && k === 'crawl-delay') cur.delay = Number(v) || null;
  }
  const g = groups.find((x) => x.agents.some((a) => agent.includes(a) && a !== '*')) ?? groups.find((x) => x.agents.includes('*'));
  return { sitemaps, disallow: g?.disallow ?? [], allow: g?.allow ?? [], crawlDelay: g?.delay ?? null };
}

export function robotsAllows(rules: { allow: string[]; disallow: string[] }, path: string): boolean {
  const match = (rule: string) => {
    if (!rule) return false;
    const re = new RegExp('^' + rule.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\\\$$/, '$'));
    return re.test(path);
  };
  const longest = (rules_: string[]) => rules_.filter(match).sort((a, b) => b.length - a.length)[0] ?? '';
  const a = longest(rules.allow);
  const d = longest(rules.disallow);
  if (!d) return true;
  return a.length >= d.length;
}