import { load, type CheerioAPI } from "cheerio";
import { getDocumentProxy } from "unpdf";
import { parseMw, parseAreaSqm, parseAreaHa, parseMoney, parsePartialDate, normalizeStatus, inferFacilityType, countryFromText, cleanText, htmlToText } from "@dci/core";
import type { SelectorRule } from "./config.js";
import type { RawDocument } from "./types.js";

/** All JSON-LD objects on a page (flattened @graph), optionally filtered by @type. */
export function jsonLd(html: string, type?: string): Record<string, unknown>[] {
  const $ = load(html);
  const out: Record<string, unknown>[] = [];
  $('script[type="application/ld+json"]').each((_, el) => {
    const raw = $(el).contents().text();
    try {
      const parsed = JSON.parse(raw.replace(/^\s*<!--|-->\s*$/g, ""));
      const arr = Array.isArray(parsed) ? parsed : [parsed];
      for (const p of arr) {
        if (p && typeof p === "object" && Array.isArray((p as { "@graph"?: unknown[] })["@graph"])) out.push(...((p as { "@graph": Record<string, unknown>[] })["@graph"]));
        else if (p && typeof p === "object") out.push(p as Record<string, unknown>);
      }
    } catch { /* malformed */ }
  });
  if (!type) return out;
  const t = type.toLowerCase();
  return out.filter((o) => { const ty = o["@type"]; const list = Array.isArray(ty) ? ty : [ty]; return list.some((x) => String(x).toLowerCase() === t); });
}

/** Embedded JSON blobs: __NEXT_DATA__, window.__INITIAL_STATE__ = {...}, application/json scripts, Nuxt/Drupal settings. */
export function embeddedJson(html: string): Array<{ id: string; data: unknown }> {
  const $ = load(html);
  const out: Array<{ id: string; data: unknown }> = [];
  $('script[type="application/json"], script#__NEXT_DATA__, script[data-drupal-selector="drupal-settings-json"]').each((_, el) => {
    const id = $(el).attr("id") ?? $(el).attr("data-drupal-selector") ?? "json";
    try { out.push({ id, data: JSON.parse($(el).contents().text()) }); } catch { /* skip */ }
  });
  const re = /(?:window\.|var\s+|const\s+|let\s+)?(__INITIAL_STATE__|__NUXT__|__PRELOADED_STATE__|__APOLLO_STATE__|__DATA__|__INITIAL_DATA__|initialData|pageData|__NEXT_DATA__)\s*=\s*(\{[\s\S]*?\})\s*;?\s*(?:<\/script>|\n)/g;
  for (const m of html.matchAll(re)) { try { out.push({ id: m[1]!, data: JSON.parse(m[2]!) }); } catch { /* not plain JSON */ } }
  return out;
}

/** Tiny JSONPath subset: "$.a.b[0].c", "a.b", "[*]" not supported (use walk). */
export function jsonPath(obj: unknown, path: string): unknown {
  const parts = path.replace(/^\$\.?/, "").split(/\.|\[(\d+)\]/).filter((p) => p !== undefined && p !== "");
  let cur: unknown = obj;
  for (const p of parts) {
    if (cur == null) return undefined;
    cur = (cur as Record<string, unknown>)[p];
  }
  return cur;
}

/** Depth-first walk collecting objects satisfying a predicate (used to find facility-like nodes in embedded JSON). */
export function walkJson(obj: unknown, pred: (o: Record<string, unknown>) => boolean, out: Record<string, unknown>[] = [], depth = 0): Record<string, unknown>[] {
  if (depth > 12 || obj == null) return out;
  if (Array.isArray(obj)) { for (const x of obj) walkJson(x, pred, out, depth + 1); return out; }
  if (typeof obj === "object") {
    const o = obj as Record<string, unknown>;
    if (pred(o)) out.push(o);
    for (const v of Object.values(o)) walkJson(v, pred, out, depth + 1);
  }
  return out;
}

export function applyTransform(v: unknown, t: SelectorRule extends string ? never : Exclude<SelectorRule, string>["transform"]): unknown {
  if (v == null) return null;
  const s = typeof v === "string" ? v : String(v);
  switch (t) {
    case "mw": return parseMw(s);
    case "sqm": return parseAreaSqm(s);
    case "ha": return parseAreaHa(s);
    case "money": return parseMoney(s);
    case "date": return parsePartialDate(s);
    case "status": return normalizeStatus(s);
    case "type": return inferFacilityType(s);
    case "country": return /^[A-Z]{2}$/.test(s.trim()) ? s.trim() : countryFromText(s);
    case "int": { const n = parseInt(s.replace(/[^\d-]/g, ""), 10); return Number.isFinite(n) ? n : null; }
    case "float": { const n = parseFloat(s.replace(/[^\d.-]/g, "")); return Number.isFinite(n) ? n : null; }
    case "trim": return cleanText(s);
    case "lower": return s.trim().toLowerCase();
    case "url": return s.trim();
    case "bool": return /^(true|yes|1|y)$/i.test(s.trim());
    default: return typeof v === "string" ? cleanText(v) : v;
  }
}

/** Evaluate one declarative selector rule against a page. */
export function evalRule($: CheerioAPI, html: string, rule: SelectorRule, scope?: ReturnType<CheerioAPI>): { value: unknown; method: string } {
  const r: Exclude<SelectorRule, string> = typeof rule === "string" ? { selector: rule } : rule;
  let value: unknown = null;
  let method = "";
  if (r.jsonld) {
    const objs = jsonLd(html, r.jsonld);
    value = objs.length && r.jsonPath ? jsonPath(objs[0], r.jsonPath) : objs[0] ?? null;
    method = `json-ld:${r.jsonld}${r.jsonPath ? "." + r.jsonPath : ""}`;
  } else if (r.meta) {
    value = $(`meta[property="${r.meta}"], meta[name="${r.meta}"]`).first().attr("content") ?? null;
    method = `meta:${r.meta}`;
  } else if (r.jsonPath && !r.selector) {
    for (const b of embeddedJson(html)) { const v = jsonPath(b.data, r.jsonPath); if (v != null) { value = v; method = `embedded-json:${b.id}.${r.jsonPath}`; break; } }
  } else if (r.selector) {
    const sel = scope ? scope.find(r.selector) : $(r.selector);
    if (r.all) value = sel.map((_, el) => (r.attr ? $(el).attr(r.attr) : $(el).text())).get().map((x) => cleanText(x)).filter(Boolean);
    else value = r.attr ? sel.first().attr(r.attr) ?? null : cleanText(sel.first().text());
    method = `selector:${r.selector}${r.attr ? "@" + r.attr : ""}`;
  } else if (r.regex) {
    value = htmlToText(html);
    method = "regex";
  }
  if (r.regex && value != null) {
    const re = new RegExp(r.regex, r.regexFlags ?? "i");
    const apply = (s: string) => { const m = s.match(re); return m ? (m[r.group ?? 1] ?? m[0]) : null; };
    value = Array.isArray(value) ? value.map((x) => apply(String(x))).filter(Boolean) : apply(String(value));
    method += `+regex:${r.regex.slice(0, 40)}`;
  }
  if (r.transform && value != null) { value = Array.isArray(value) ? value.map((x) => applyTransform(x, r.transform)) : applyTransform(value, r.transform); method += `>${r.transform}`; }
  if ((value == null || value === "") && r.default !== undefined) value = r.default;
  return { value, method };
}

export function pageTitle(html: string): string | null {
  const $ = load(html);
  return cleanText($('meta[property="og:title"]').attr("content") ?? $("title").first().text() ?? $("h1").first().text());
}

export function metaDescription(html: string): string | null {
  const $ = load(html);
  return cleanText($('meta[name="description"]').attr("content") ?? $('meta[property="og:description"]').attr("content") ?? null);
}

/** Blocks that are never the story: chrome, related / recommended / trending teasers, sidebars, comments. */
export const NON_CONTENT_SELECTOR = "script,style,noscript,nav,header,footer,aside,form,iframe,svg,[role=navigation],[role=complementary],[class*=related],[class*=recommend],[class*=trending],[class*=popular],[class*=sidebar],[class*=read-more],[class*=readmore],[class*=more-stories],[class*=also-like],[class*=comments],[id*=related],[id*=comments]";
/** Heading that opens a trailing teaser section inside the story element itself ("More in Construction & Site Selection", "Related articles", "Tags"). */
export const TEASER_HEADING_RE = /^\s*(?:More (?:in|from|on|like this)\b.*|Related(?: (?:articles?|stories|news|content|posts?|coverage|reading|links?))?|Recommended(?: for you| reading| stories| articles)?|You (?:may|might) also like|Most (?:read|popular|viewed)|Trending(?: now| stories)?|Popular (?:now|posts|stories|articles)|Latest (?:news|stories|posts|articles)|Read (?:more|next|also)|See also|Further reading|Editor'?s picks|Sponsored(?: content)?|Tags|Comments|Leave a (?:comment|reply))\s*:?\s*$/i;
/** Minimum story length before a teaser heading is allowed to cut the text (a short lead titled "Related" is not a teaser section). */
const TEASER_CUT_MIN_CHARS = 300;

/** Drop everything from the first teaser heading onwards (teasers are the last thing in a story element). */
export function cutTeasers(text: string): string {
  const lines = text.split("\n");
  let consumed = 0;
  for (let i = 0; i < lines.length; i++) {
    const line = lines[i]!;
    if (consumed >= TEASER_CUT_MIN_CHARS && line.trim().length <= 60 && TEASER_HEADING_RE.test(line)) return lines.slice(0, i).join("\n").trimEnd();
    consumed += line.length;
  }
  return text;
}

/**
 * Main text of an article-like page: the largest <article> / <main> / content block, after removing navigation, footer,
 * aside, related / recommended / trending / popular / sidebar blocks, and cut at the first trailing teaser heading — so a
 * "More in …" list of other headlines never leaks into the story (and never names its operator).
 */
export function mainText(html: string): string {
  const $ = load(html);
  $(NON_CONTENT_SELECTOR).remove();
  const candidates = ["article", "main", '[role="main"]', ".article-body", ".press-release", ".entry-content", ".post-content", ".content", "#content", "body"];
  for (const c of candidates) {
    // several matches (the story plus teaser <article class="card"> cards): the largest one is the story
    const texts = $(c).toArray().map((el) => htmlToText($(el).html() ?? ""));
    if (!texts.length) continue;
    const t = cutTeasers(texts.sort((a, b) => b.length - a.length)[0]!);
    if (t.length > 300) return t;
  }
  return cutTeasers(htmlToText($("body").html() ?? html));
}

/** Publication date from meta tags / JSON-LD / <time>. */
export function publishedDate(html: string): string | null {
  const $ = load(html);
  const cands = [$('meta[property="article:published_time"]').attr("content"), $('meta[name="pubdate"]').attr("content"), $('meta[name="publish-date"]').attr("content"), $('meta[name="date"]').attr("content"), $('meta[itemprop="datePublished"]').attr("content"), $("time[datetime]").first().attr("datetime"), (jsonLd(html, "NewsArticle")[0]?.datePublished as string | undefined), (jsonLd(html, "Article")[0]?.datePublished as string | undefined)];
  for (const c of cands) if (c) { const d = parsePartialDate(String(c)); if (d) return d; }
  return null;
}

/** PDFs above this size are not parsed at all (planning documents are a few MB; bigger files are scans or bombs). */
export const PDF_MAX_BYTES = 15 * 1024 * 1024;
/** Hard ceiling on PDF text extraction; a pathological file must never stall a run. */
export const PDF_TIMEOUT_MS = 20_000;

/**
 * Text of the first `maxPages` pages of a PDF (page by page, so a 3 000-page file costs 60 pages), bounded by
 * `maxBytes` on input and `timeoutMs` on wall time. Throws on oversize / timeout — callers log and skip the document.
 */
export async function pdfText(buf: Buffer, maxPages = 60, opts: { timeoutMs?: number; maxBytes?: number } = {}): Promise<{ text: string; pages: number; title: string | null; truncated: boolean }> {
  const maxBytes = opts.maxBytes ?? PDF_MAX_BYTES;
  const timeoutMs = opts.timeoutMs ?? PDF_TIMEOUT_MS;
  if (buf.length > maxBytes) throw new Error(`pdf too large: ${buf.length} bytes > ${maxBytes}`);
  let pdf: Awaited<ReturnType<typeof getDocumentProxy>> | null = null;
  const work = async () => {
    pdf = await getDocumentProxy(new Uint8Array(buf));
    const pages = pdf.numPages;
    const n = Math.min(pages, Math.max(1, maxPages));
    const parts: string[] = [];
    for (let i = 1; i <= n; i++) {
      const page = await pdf.getPage(i);
      const content = await page.getTextContent();
      parts.push(content.items.map((it) => ("str" in it ? it.str + (it.hasEOL ? "\n" : " ") : "")).join(""));
      page.cleanup();
    }
    const meta = await pdf.getMetadata().catch(() => null);
    const title = (meta?.info as { Title?: string } | undefined)?.Title ?? null;
    return { text: parts.join("\n").replace(/[ \t]+/g, " ").replace(/\n\s*\n+/g, "\n").trim(), pages, title: cleanText(title), truncated: pages > n };
  };
  let timer: ReturnType<typeof setTimeout> | undefined;
  const timeout = new Promise<never>((_, reject) => { timer = setTimeout(() => reject(new Error(`pdf extraction timed out after ${timeoutMs} ms`)), timeoutMs); });
  try {
    return await Promise.race([work(), timeout]);
  } finally {
    clearTimeout(timer);
    // release pdf.js worker memory (also aborts the in-flight parse after a timeout)
    try { void (pdf as unknown as { destroy?: () => Promise<void> } | null)?.destroy?.()?.catch(() => undefined); } catch { /* ignore */ }
  }
}

export function isPdf(doc: RawDocument): boolean { return /pdf/i.test(doc.contentType ?? "") || doc.body.subarray(0, 5).toString("latin1") === "%PDF-"; }
export function isJson(doc: RawDocument): boolean { return /json/i.test(doc.contentType ?? "") || /^\s*[\[{]/.test(doc.text.slice(0, 20)); }

/** Loose geo extraction from any HTML: JSON-LD geo, meta geo.position / ICBM, data-lat attributes, Google Maps links. */
export function extractGeo(html: string): { lat: number; lng: number; method: string } | null {
  for (const o of jsonLd(html)) {
    const g = (o.geo ?? (o.location as Record<string, unknown> | undefined)?.geo) as { latitude?: unknown; longitude?: unknown } | undefined;
    if (g && g.latitude != null && g.longitude != null) { const lat = Number(g.latitude), lng = Number(g.longitude); if (Number.isFinite(lat) && Number.isFinite(lng)) return { lat, lng, method: "json-ld:geo" }; }
  }
  const $ = load(html);
  const pos = $('meta[name="geo.position"]').attr("content") ?? $('meta[name="ICBM"]').attr("content");
  if (pos) { const m = pos.match(/(-?\d+(?:\.\d+)?)[;,\s]+(-?\d+(?:\.\d+)?)/); if (m) return { lat: Number(m[1]), lng: Number(m[2]), method: "meta:geo.position" }; }
  const el = $("[data-lat][data-lng], [data-latitude][data-longitude]").first();
  if (el.length) { const lat = Number(el.attr("data-lat") ?? el.attr("data-latitude")), lng = Number(el.attr("data-lng") ?? el.attr("data-longitude")); if (Number.isFinite(lat) && Number.isFinite(lng)) return { lat, lng, method: "attr:data-lat" }; }
  const gm = html.match(/maps\.google\.[a-z.]+\/(?:maps)?\?q=(-?\d+\.\d+),(-?\d+\.\d+)|google\.com\/maps[^"']*?@(-?\d+\.\d+),(-?\d+\.\d+)|!3d(-?\d+\.\d+)!4d(-?\d+\.\d+)/);
  if (gm) { const lat = Number(gm[1] ?? gm[3] ?? gm[5]), lng = Number(gm[2] ?? gm[4] ?? gm[6]); if (Number.isFinite(lat) && Number.isFinite(lng)) return { lat, lng, method: "link:google-maps" }; }
  return null;
}

/** Postal address from JSON-LD (PostalAddress) on any page. */
export function extractAddress(html: string): { street: string | null; city: string | null; region: string | null; postal: string | null; country: string | null } | null {
  for (const o of jsonLd(html)) {
    const a = (o.address ?? (o.location as Record<string, unknown> | undefined)?.address) as Record<string, unknown> | string | undefined;
    if (!a) continue;
    if (typeof a === "string") return { street: cleanText(a), city: null, region: null, postal: null, country: countryFromText(a) };
    return { street: cleanText(String(a.streetAddress ?? "")), city: cleanText(String(a.addressLocality ?? "")), region: cleanText(String(a.addressRegion ?? "")), postal: cleanText(String(a.postalCode ?? "")), country: a.addressCountry ? (typeof a.addressCountry === "string" ? (/^[A-Z]{2}$/.test(a.addressCountry) ? a.addressCountry : countryFromText(a.addressCountry)) : countryFromText(String((a.addressCountry as Record<string, unknown>).name ?? ""))) : null };
  }
  return null;
}