/**
 * Dependency-free HTML → Markdown conversion, page metadata and link extraction.
 *
 * The converter builds a lightweight DOM from a forgiving tokenizer, optionally isolates the main
 * content (readability-style density scoring), then serialises to Markdown. It is intended for
 * LLM/RAG pipelines and crawling: stable, compact output rather than perfect fidelity.
 */
import { decodeEntities } from "./text";
import type { PageLink, PageMetadata } from "./schema";

// ---------------------------------------------------------------------------
// Tokenizer / DOM-lite
// ---------------------------------------------------------------------------
export interface HNode {
  type: "element" | "text";
  tag: string;
  attrs: Record<string, string>;
  children: HNode[];
  text: string;
  parent: HNode | null;
}

const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]);
const RAW_TEXT = new Set(["script", "style", "noscript", "template", "svg", "math", "iframe", "canvas", "object", "textarea"]);
const BLOCK = new Set([
  "address", "article", "aside", "blockquote", "body", "center", "dd", "details", "dialog", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6",
  "header", "hr", "html", "li", "main", "nav", "ol", "p", "pre", "section", "summary", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul",
]);
/** Elements whose content is closed implicitly by a new block start. */
const AUTO_CLOSE: Record<string, Set<string>> = {
  p: BLOCK,
  li: new Set(["li"]),
  dt: new Set(["dt", "dd"]),
  dd: new Set(["dt", "dd"]),
  tr: new Set(["tr"]),
  td: new Set(["td", "th", "tr"]),
  th: new Set(["td", "th", "tr"]),
  option: new Set(["option"]),
};

function makeNode(type: HNode["type"], tag: string, parent: HNode | null): HNode {
  return { type, tag, attrs: {}, children: [], text: "", parent };
}

const ATTR_RE = /([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;

export function parseHtml(html: string): HNode {
  const root = makeNode("element", "#root", null);
  let cur = root;
  let i = 0;
  const n = html.length;
  const pushText = (t: string) => {
    if (!t) return;
    const node = makeNode("text", "#text", cur);
    node.text = t;
    cur.children.push(node);
  };
  while (i < n) {
    const lt = html.indexOf("<", i);
    if (lt === -1) {
      pushText(html.slice(i));
      break;
    }
    if (lt > i) pushText(html.slice(i, lt));
    if (html.startsWith("<!--", lt)) {
      const end = html.indexOf("-->", lt + 4);
      i = end === -1 ? n : end + 3;
      continue;
    }
    if (html.startsWith("<![CDATA[", lt)) {
      const end = html.indexOf("]]>", lt);
      i = end === -1 ? n : end + 3;
      continue;
    }
    if (html[lt + 1] === "!" || html[lt + 1] === "?") {
      const end = html.indexOf(">", lt);
      i = end === -1 ? n : end + 1;
      continue;
    }
    const gt = findTagEnd(html, lt);
    if (gt === -1) {
      pushText(html.slice(lt));
      break;
    }
    const raw = html.slice(lt + 1, gt);
    i = gt + 1;
    if (raw.startsWith("/")) {
      const tag = raw.slice(1).trim().toLowerCase().split(/\s/)[0]!;
      // close up to matching open element
      let p: HNode | null = cur;
      while (p && p !== root && p.tag !== tag) p = p.parent;
      if (p && p !== root) cur = p.parent ?? root;
      continue;
    }
    const m = raw.match(/^([a-zA-Z][a-zA-Z0-9:-]*)/);
    if (!m) {
      pushText("<" + raw + ">");
      continue;
    }
    const tag = m[1]!.toLowerCase();
    const selfClosing = raw.endsWith("/");
    const attrs: Record<string, string> = {};
    const attrStr = raw.slice(m[0].length, selfClosing ? -1 : undefined);
    for (const am of attrStr.matchAll(ATTR_RE)) {
      const k = am[1]!.toLowerCase();
      attrs[k] = decodeEntities(am[2] ?? am[3] ?? am[4] ?? "");
    }
    // implicit closes
    let p: HNode | null = cur;
    while (p && p !== root) {
      const ac = AUTO_CLOSE[p.tag];
      if (ac && ac.has(tag)) {
        cur = p.parent ?? root;
        break;
      }
      if (p.tag === "p" && BLOCK.has(tag)) {
        cur = p.parent ?? root;
        break;
      }
      p = p.parent;
    }
    const node = makeNode("element", tag, cur);
    node.attrs = attrs;
    cur.children.push(node);
    if (RAW_TEXT.has(tag) && !selfClosing) {
      const close = html.toLowerCase().indexOf(`</${tag}`, i);
      const end = close === -1 ? n : close;
      const t = makeNode("text", "#text", node);
      t.text = html.slice(i, end);
      node.children.push(t);
      i = close === -1 ? n : html.indexOf(">", close) + 1 || n;
      continue;
    }
    if (!VOID.has(tag) && !selfClosing) cur = node;
  }
  return root;
}

function findTagEnd(html: string, from: number): number {
  let q: string | null = null;
  for (let j = from + 1; j < html.length; j++) {
    const c = html[j]!;
    if (q) {
      if (c === q) q = null;
    } else if (c === '"' || c === "'") {
      // only treat as quote when inside attribute area (after a space or =)
      const prev = html[j - 1];
      if (prev === "=" || prev === " " || prev === "\t" || prev === "\n") q = c;
    } else if (c === ">") return j;
    else if (c === "<" && j > from + 1) return -1 + 0 * j; // malformed: give up on this tag
  }
  return -1;
}

// ---------------------------------------------------------------------------
// Queries
// ---------------------------------------------------------------------------
export function walk(node: HNode, fn: (n: HNode) => boolean | void): void {
  if (fn(node) === false) return;
  for (const c of node.children) walk(c, fn);
}

export function findAll(root: HNode, pred: (n: HNode) => boolean): HNode[] {
  const out: HNode[] = [];
  walk(root, (n) => {
    if (n.type === "element" && pred(n)) out.push(n);
  });
  return out;
}

export function textOf(node: HNode): string {
  let s = "";
  walk(node, (n) => {
    if (n.type === "element" && RAW_TEXT.has(n.tag)) return false;
    if (n.type === "text") s += n.text;
  });
  return decodeEntities(s).replace(/\s+/g, " ").trim();
}

// ---------------------------------------------------------------------------
// Main content extraction (readability-lite)
// ---------------------------------------------------------------------------
const NOISE_TAGS = new Set(["nav", "header", "footer", "aside", "form", "button", "select", "input", "label", "dialog", "menu"]);
const NOISE_RE = /(^|[\s_-])(nav|menu|sidebar|side-bar|footer|header|banner|cookie|consent|gdpr|modal|popup|newsletter|subscribe|social|share|sharing|related|recommend|promo|advert|ad-|ads|sponsor|breadcrumb|comment|widget|toolbar|skip|masthead|sitemap|legal|copyright|login|signup|search)([\s_-]|$)/i;
const CONTENT_RE = /(^|[\s_-])(article|content|main|post|entry|body|story|text|blog|product|description|prose|markdown|documentation|docs)([\s_-]|$)/i;

function classId(n: HNode): string {
  return `${n.attrs["class"] ?? ""} ${n.attrs["id"] ?? ""}`;
}

export function isNoise(n: HNode): boolean {
  if (n.type !== "element") return false;
  if (RAW_TEXT.has(n.tag)) return true;
  if (NOISE_TAGS.has(n.tag)) return true;
  if (n.attrs["hidden"] !== undefined || /display\s*:\s*none/i.test(n.attrs["style"] ?? "")) return true;
  if (n.attrs["aria-hidden"] === "true") return true;
  const role = n.attrs["role"];
  if (role && /navigation|banner|contentinfo|complementary|dialog|search|menu/i.test(role)) return true;
  if (n.tag === "div" || n.tag === "section" || n.tag === "ul" || n.tag === "span") {
    const ci = classId(n);
    if (NOISE_RE.test(ci) && !CONTENT_RE.test(ci)) return true;
  }
  return false;
}

/** Pick the element that most likely holds the primary content. Falls back to <body>/root. */
export function mainContent(root: HNode): HNode {
  const explicit = findAll(root, (n) => n.tag === "article" || n.tag === "main" || n.attrs["role"] === "main" || n.attrs["itemprop"] === "articleBody");
  const body = findAll(root, (n) => n.tag === "body")[0] ?? root;
  const scored: Array<{ node: HNode; score: number }> = [];
  const candidates = explicit.length ? explicit : findAll(body, (n) => ["div", "section", "td", "article", "main"].includes(n.tag));
  const bodyLen = Math.max(1, textOf(body).length);
  for (const c of candidates) {
    if (isNoise(c)) continue;
    const text = textOf(c);
    if (text.length < 200) continue;
    const paragraphs = findAll(c, (n) => n.tag === "p" || n.tag === "h1" || n.tag === "h2" || n.tag === "h3" || n.tag === "li" || n.tag === "pre").length;
    const links = findAll(c, (n) => n.tag === "a").reduce((s, a) => s + textOf(a).length, 0);
    const linkDensity = links / Math.max(1, text.length);
    let score = text.length / bodyLen + Math.min(paragraphs, 40) / 40;
    score *= 1 - Math.min(0.9, linkDensity);
    if (c.tag === "article" || c.tag === "main" || c.attrs["role"] === "main") score *= 1.6;
    if (CONTENT_RE.test(classId(c))) score *= 1.2;
    scored.push({ node: c, score });
  }
  if (!scored.length) return body;
  scored.sort((a, b) => b.score - a.score);
  const best = scored[0]!;
  // Guard against choosing a tiny fragment: require the pick to hold ≥ 25% of the body text.
  if (textOf(best.node).length < bodyLen * 0.25 && !explicit.length) return body;
  return best.node;
}

// ---------------------------------------------------------------------------
// Markdown serialisation
// ---------------------------------------------------------------------------
export interface MarkdownOptions {
  baseUrl?: string;
  /** Isolate the main content (default true). */
  mainContent?: boolean;
  /** Keep images as ![alt](src) (default true). */
  images?: boolean;
  /** Keep hyperlinks as [text](href) (default true); false renders plain text. */
  links?: boolean;
  /** Drop navigation/footers/asides etc. even inside the main content (default true). */
  stripNoise?: boolean;
  /** Maximum output length in characters (default 2,000,000). */
  maxLength?: number;
}

function absolutize(href: string | undefined, base?: string): string | null {
  if (!href) return null;
  const h = href.trim();
  if (!h || h.startsWith("javascript:") || h.startsWith("data:") || h.startsWith("mailto:") || h.startsWith("tel:") || h === "#") return h.startsWith("mailto:") || h.startsWith("tel:") ? h : null;
  try {
    return base ? new URL(h, base).toString() : new URL(h).toString();
  } catch {
    return null;
  }
}

function esc(s: string): string {
  return s.replace(/([\\`*_{}[\]<>])/g, "\\$1");
}

type WriterOpts = Required<Pick<MarkdownOptions, "images" | "links" | "stripNoise">> & { baseUrl?: string; /** Text length of the rendered scope, used to keep "noise" landmarks that actually hold the content (e.g. a <nav> of language links on a portal). */ scopeLen?: number };

class MdWriter {
  out: string[] = [];
  constructor(private readonly opts: WriterOpts) {}

  /** Noise check that spares landmark elements carrying a large share of the scope's text. */
  private noise(n: HNode): boolean {
    if (!this.opts.stripNoise || !isNoise(n)) return false;
    if (n.type === "element" && (n.tag === "nav" || n.tag === "aside" || n.tag === "header" || n.tag === "footer" || n.tag === "form") && this.opts.scopeLen) {
      const t = textOf(n).length;
      if (t >= 120 && t >= this.opts.scopeLen * 0.35) return false;
    }
    return true;
  }

  block(s: string) {
    const t = s.replace(/\n{3,}/g, "\n\n").trim();
    if (!t) return;
    this.out.push(t);
  }

  render(node: HNode): string {
    this.out = [];
    this.renderChildren(node, "");
    return this.out.join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
  }

  private inline(node: HNode): string {
    let s = "";
    for (const c of node.children) {
      if (c.type === "text") {
        s += decodeEntities(c.text).replace(/\s+/g, " ");
        continue;
      }
      if (this.noise(c)) continue;
      switch (c.tag) {
        case "br":
          s += "  \n";
          break;
        case "strong":
        case "b": {
          const t = this.inline(c).trim();
          s += t ? `**${t}**` : "";
          break;
        }
        case "em":
        case "i": {
          const t = this.inline(c).trim();
          s += t ? `*${t}*` : "";
          break;
        }
        case "del":
        case "s":
        case "strike": {
          const t = this.inline(c).trim();
          s += t ? `~~${t}~~` : "";
          break;
        }
        case "code":
        case "kbd":
        case "samp":
        case "var": {
          const t = textOf(c);
          s += t ? `\`${t.replace(/`/g, "\\`")}\`` : "";
          break;
        }
        case "a": {
          const t = this.inline(c).trim();
          const href = absolutize(c.attrs["href"], this.opts.baseUrl);
          if (!t && !href) break;
          if (!this.opts.links || !href) s += t;
          else s += `[${t || href}](${href})`;
          break;
        }
        case "img": {
          if (!this.opts.images) break;
          const src = absolutize(c.attrs["src"] ?? c.attrs["data-src"] ?? (c.attrs["srcset"] ?? "").split(/[\s,]/)[0], this.opts.baseUrl);
          if (!src) break;
          const alt = (c.attrs["alt"] ?? "").replace(/\s+/g, " ").trim();
          s += `![${alt}](${src})`;
          break;
        }
        case "sup":
          s += `^${this.inline(c).trim()}`;
          break;
        case "sub":
          s += `_${this.inline(c).trim()}`;
          break;
        case "q":
          s += `"${this.inline(c).trim()}"`;
          break;
        case "input": {
          if (c.attrs["type"] === "checkbox") s += c.attrs["checked"] !== undefined ? "[x] " : "[ ] ";
          break;
        }
        default:
          if (BLOCK.has(c.tag)) {
            // block inside inline context: flush as text with breaks
            s += "\n" + this.inlineBlock(c) + "\n";
          } else s += this.inline(c);
      }
    }
    return s;
  }

  private inlineBlock(node: HNode): string {
    const w = new MdWriter(this.opts);
    return w.render(node);
  }

  private renderChildren(node: HNode, prefix: string) {
    let inlineBuf = "";
    const flush = () => {
      const t = inlineBuf.replace(/[ \t]+/g, " ").replace(/ *\n */g, "\n").trim();
      if (t) this.block(prefix + t.split("\n").join("\n" + prefix));
      inlineBuf = "";
    };
    for (const c of node.children) {
      if (c.type === "text") {
        inlineBuf += decodeEntities(c.text).replace(/\s+/g, " ");
        continue;
      }
      if (this.noise(c)) continue;
      if (!BLOCK.has(c.tag) && !["table", "ul", "ol", "dl"].includes(c.tag)) {
        inlineBuf += this.inline({ ...c, children: [c] } as HNode);
        continue;
      }
      flush();
      this.renderBlock(c, prefix);
    }
    flush();
  }

  private renderBlock(c: HNode, prefix: string) {
    switch (c.tag) {
      case "h1":
      case "h2":
      case "h3":
      case "h4":
      case "h5":
      case "h6": {
        const level = Number(c.tag[1]);
        const t = this.inline(c).replace(/\s+/g, " ").trim();
        if (t) this.block(`${prefix}${"#".repeat(level)} ${t}`);
        return;
      }
      case "p":
      case "div":
      case "section":
      case "article":
      case "main":
      case "header":
      case "footer":
      case "aside":
      case "nav":
      case "address":
      case "center":
      case "details":
      case "summary":
      case "figure":
      case "dialog":
      case "fieldset":
      case "form":
      case "body":
      case "html": {
        const hasBlocks = c.children.some((k) => k.type === "element" && (BLOCK.has(k.tag) || ["table", "ul", "ol", "dl"].includes(k.tag)));
        if (hasBlocks) this.renderChildren(c, prefix);
        else {
          const t = this.inline(c).replace(/[ \t]+/g, " ").trim();
          if (t) this.block(prefix + t.split("\n").map((l) => l.trim()).join("\n" + prefix));
        }
        return;
      }
      case "figcaption": {
        const t = this.inline(c).trim();
        if (t) this.block(`${prefix}*${t}*`);
        return;
      }
      case "blockquote": {
        const inner = new MdWriter(this.opts).render(c);
        if (inner) this.block(inner.split("\n").map((l) => `${prefix}> ${l}`).join("\n"));
        return;
      }
      case "pre": {
        const codeEl = c.children.find((k) => k.type === "element" && k.tag === "code");
        const lang = ((codeEl?.attrs["class"] ?? c.attrs["class"] ?? "").match(/(?:language|lang)-([a-z0-9+#-]+)/i) ?? [])[1] ?? "";
        let code = "";
        walk(c, (n) => {
          if (n.type === "text") code += decodeEntities(n.text);
        });
        code = code.replace(/^\n+|\n+$/g, "");
        const fence = code.includes("```") ? "````" : "```";
        this.block(`${prefix}${fence}${lang}\n${code.split("\n").map((l) => prefix + l).join("\n")}\n${prefix}${fence}`);
        return;
      }
      case "hr":
        this.block(`${prefix}---`);
        return;
      case "ul":
      case "ol": {
        const ordered = c.tag === "ol";
        let idx = Number(c.attrs["start"] ?? 1) || 1;
        const items: string[] = [];
        for (const li of c.children) {
          if (li.type !== "element" || li.tag !== "li") continue;
          const marker = ordered ? `${idx++}. ` : "- ";
          const w = new MdWriter(this.opts);
          const inner = w.render(li);
          if (!inner) continue;
          const lines = inner.split("\n");
          const pad = " ".repeat(marker.length);
          items.push(prefix + marker + lines[0] + (lines.length > 1 ? "\n" + lines.slice(1).map((l) => (l ? prefix + pad + l : "")).join("\n") : ""));
        }
        if (items.length) this.block(items.join("\n"));
        return;
      }
      case "li": {
        const w = new MdWriter(this.opts);
        const inner = w.render(c);
        if (inner) this.block(prefix + "- " + inner.split("\n").join("\n" + prefix + "  "));
        return;
      }
      case "dl": {
        const lines: string[] = [];
        for (const k of c.children) {
          if (k.type !== "element") continue;
          if (k.tag === "dt") lines.push(`${prefix}**${this.inline(k).trim()}**`);
          if (k.tag === "dd") lines.push(`${prefix}: ${this.inline(k).trim()}`);
        }
        if (lines.length) this.block(lines.join("\n"));
        return;
      }
      case "table": {
        const rows: string[][] = [];
        let headerRow: string[] | null = null;
        const trs = findAll(c, (n) => n.tag === "tr").filter((tr) => {
          // exclude nested tables' rows
          let p = tr.parent;
          while (p && p !== c) {
            if (p.tag === "table") return false;
            p = p.parent;
          }
          return true;
        });
        for (const tr of trs) {
          const cells = tr.children.filter((k) => k.type === "element" && (k.tag === "td" || k.tag === "th"));
          if (!cells.length) continue;
          const vals = cells.map((cell) => this.inline(cell).replace(/\s*\n\s*/g, " ").replace(/\|/g, "\\|").trim());
          const isHeader = !headerRow && cells.every((cell) => cell.tag === "th");
          if (isHeader) headerRow = vals;
          else rows.push(vals);
        }
        if (!headerRow && !rows.length) return;
        const width = Math.max(headerRow?.length ?? 0, ...rows.map((r) => r.length));
        const pad = (r: string[]) => [...r, ...Array(Math.max(0, width - r.length)).fill("")];
        const head = headerRow ? pad(headerRow) : Array(width).fill(" ");
        const lines = [`${prefix}| ${head.join(" | ")} |`, `${prefix}| ${Array(width).fill("---").join(" | ")} |`, ...rows.map((r) => `${prefix}| ${pad(r).join(" | ")} |`)];
        this.block(lines.join("\n"));
        return;
      }
      case "tr":
      case "td":
      case "th":
      case "thead":
      case "tbody":
      case "tfoot":
      case "dd":
      case "dt": {
        const t = this.inline(c).trim();
        if (t) this.block(prefix + t);
        return;
      }
      default: {
        this.renderChildren(c, prefix);
      }
    }
  }
}

export function htmlToMarkdown(html: string, opts: MarkdownOptions = {}): string {
  const root = parseHtml(html);
  const scope = opts.mainContent === false ? (findAll(root, (n) => n.tag === "body")[0] ?? root) : mainContent(root);
  const writer = new MdWriter({ images: opts.images ?? true, links: opts.links ?? true, stripNoise: opts.stripNoise ?? true, baseUrl: opts.baseUrl, scopeLen: textOf(scope).length });
  let md = writer.render(scope);
  // Prepend the document title when the content does not already start with a heading.
  const title = findAll(root, (n) => n.tag === "title")[0];
  const t = title ? textOf(title) : "";
  if (t && !/^#\s/.test(md) && !md.startsWith(`# ${t}`)) md = `# ${esc(t)}\n\n${md}`;
  const max = opts.maxLength ?? 2_000_000;
  return md.length > max ? md.slice(0, max) : md;
}

/** Readable text (main content, boilerplate removed). Falls back to whole document. */
export function htmlToMainText(html: string): string {
  const root = parseHtml(html);
  const scope = mainContent(root);
  const w = new MdWriter({ images: false, links: false, stripNoise: true, scopeLen: textOf(scope).length });
  return w
    .render(scope)
    .replace(/^#+\s*/gm, "")
    .replace(/\*\*|~~|(?<!\\)\*/g, "")
    .replace(/\\([\\`*_{}[\]<>])/g, "$1")
    .trim();
}

// ---------------------------------------------------------------------------
// Metadata & links
// ---------------------------------------------------------------------------
export function registrableHost(hostname: string): string {
  const parts = hostname.toLowerCase().replace(/^www\./, "").split(".");
  if (parts.length <= 2) return parts.join(".");
  const sld = new Set(["co", "com", "org", "net", "gov", "edu", "ac", "gc", "qc", "on", "bc"]);
  if (parts.length >= 3 && sld.has(parts[parts.length - 2]!) && parts[parts.length - 1]!.length === 2) return parts.slice(-3).join(".");
  return parts.slice(-2).join(".");
}

export function extractPageMetadata(html: string, baseUrl: string): { page: PageMetadata; links: PageLink[] } {
  const root = parseHtml(html.length > 3_000_000 ? html.slice(0, 3_000_000) : html);
  const head = findAll(root, (n) => n.tag === "head")[0] ?? root;
  const titleEl = findAll(head, (n) => n.tag === "title")[0] ?? findAll(root, (n) => n.tag === "title")[0];
  const og: Record<string, string> = {};
  let description: string | null = null;
  let canonical: string | null = null;
  for (const m of findAll(root, (n) => n.tag === "meta")) {
    const name = (m.attrs["name"] ?? m.attrs["property"] ?? "").toLowerCase();
    const content = (m.attrs["content"] ?? "").trim();
    if (!name || !content) continue;
    if (name === "description" && !description) description = content.slice(0, 1000);
    if (name.startsWith("og:") || name.startsWith("twitter:") || name.startsWith("article:")) og[name] = content.slice(0, 1000);
  }
  for (const l of findAll(root, (n) => n.tag === "link")) {
    if ((l.attrs["rel"] ?? "").toLowerCase().split(/\s+/).includes("canonical")) {
      canonical = absolutize(l.attrs["href"], baseUrl);
      break;
    }
  }
  const htmlEl = findAll(root, (n) => n.tag === "html")[0];
  const lang = htmlEl?.attrs["lang"]?.trim().slice(0, 16) || og["og:locale"]?.slice(0, 16) || null;

  let baseHref = baseUrl;
  const baseEl = findAll(head, (n) => n.tag === "base")[0];
  if (baseEl?.attrs["href"]) baseHref = absolutize(baseEl.attrs["href"], baseUrl) ?? baseUrl;

  const seen = new Set<string>();
  const links: PageLink[] = [];
  let host = "";
  try {
    host = registrableHost(new URL(baseUrl).hostname);
  } catch {
    /* ignore */
  }
  for (const a of findAll(root, (n) => n.tag === "a" || n.tag === "area")) {
    const url = absolutize(a.attrs["href"], baseHref);
    if (!url || !/^https?:/i.test(url)) continue;
    const clean = url.replace(/#.*$/, "");
    if (!clean || seen.has(clean)) continue;
    seen.add(clean);
    let internal = false;
    try {
      internal = registrableHost(new URL(clean).hostname) === host;
    } catch {
      /* ignore */
    }
    const rel = (a.attrs["rel"] ?? "").toLowerCase();
    links.push({ url: clean, text: textOf(a).slice(0, 200) || (a.attrs["title"] ?? a.attrs["aria-label"] ?? "").slice(0, 200), internal, nofollow: /\bnofollow\b/.test(rel) });
    if (links.length >= 5000) break;
  }
  return {
    page: {
      title: titleEl ? textOf(titleEl).slice(0, 300) || null : og["og:title"]?.slice(0, 300) ?? null,
      description: description ?? og["og:description"] ?? null,
      canonical,
      lang,
      og,
      links_count: links.length,
    },
    links,
  };
}

/** Glob (`*`, `**`) or `/regex/flags` pattern matcher for URL filtering. */
export function urlPatternMatcher(patterns: string[] | undefined): ((url: string) => boolean) | null {
  if (!patterns?.length) return null;
  const res: RegExp[] = [];
  for (const raw of patterns) {
    const p = raw.trim();
    if (!p) continue;
    const rx = p.match(/^\/(.+)\/([a-z]*)$/);
    if (rx) {
      try {
        res.push(new RegExp(rx[1]!, rx[2]!.replace(/[^gimsuy]/g, "")));
        continue;
      } catch {
        /* fall through to glob */
      }
    }
    const escaped = p.replace(/[.+^${}()|[\]\\?]/g, "\\$&").replace(/\*\*/g, "\u0000").replace(/\*/g, "[^/]*").replace(/\u0000/g, ".*");
    res.push(new RegExp(p.includes("://") || p.startsWith("/") ? `^${escaped}$` : escaped, "i"));
  }
  if (!res.length) return null;
  return (url: string) => res.some((r) => r.test(url));
}

/** Normalise a URL for crawl de-duplication: strip fragment, tracking params, trailing slash, sort query. */
export function normalizeCrawlUrl(url: string): string | null {
  try {
    const u = new URL(url);
    if (!/^https?:$/.test(u.protocol)) return null;
    u.hash = "";
    u.hostname = u.hostname.toLowerCase();
    const drop = ["utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", "utm_id", "gclid", "fbclid", "mc_cid", "mc_eid", "ref", "ref_", "_ga", "yclid", "msclkid", "igshid", "spm"];
    for (const k of drop) u.searchParams.delete(k);
    u.searchParams.sort();
    if (u.pathname.length > 1 && u.pathname.endsWith("/")) u.pathname = u.pathname.slice(0, -1);
    if ((u.protocol === "http:" && u.port === "80") || (u.protocol === "https:" && u.port === "443")) u.port = "";
    return u.toString();
  } catch {
    return null;
  }
}

const NON_HTML_EXT = /\.(jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|mp4|mp3|wav|ogg|webm|mov|avi|zip|gz|tgz|rar|7z|tar|pdf|docx?|xlsx?|pptx?|exe|dmg|apk|css|js|mjs|json|xml|rss|atom|woff2?|ttf|eot|otf)(\?.*)?$/i;

/** Whether a URL is likely to be an HTML page worth crawling. */
export function looksLikePage(url: string): boolean {
  try {
    const u = new URL(url);
    return !NON_HTML_EXT.test(u.pathname);
  } catch {
    return false;
  }
}