/** Minimal, dependency-free HTML → readable text extraction. */
export function htmlToText(html: string): string {
  let s = html;
  s = s.replace(/<script[\s\S]*?<\/script>/gi, " ");
  s = s.replace(/<style[\s\S]*?<\/style>/gi, " ");
  s = s.replace(/<noscript[\s\S]*?<\/noscript>/gi, " ");
  s = s.replace(/<template[\s\S]*?<\/template>/gi, " ");
  s = s.replace(/<svg[\s\S]*?<\/svg>/gi, " ");
  s = s.replace(/<!--[\s\S]*?-->/g, " ");
  s = s.replace(/<(br|\/p|\/div|\/li|\/h[1-6]|\/tr|\/section|\/article|\/header|\/footer|\/blockquote|\/pre|\/table|\/ul|\/ol|\/dd|\/dt)[^>]*>/gi, "\n");
  s = s.replace(/<\/t[dh]>/gi, "\t");
  s = s.replace(/<[^>]+>/g, " ");
  s = decodeEntities(s);
  s = s.replace(/[ \t\f\v]+/g, " ");
  s = s.replace(/\s*\n\s*/g, "\n");
  s = s.replace(/\n{3,}/g, "\n\n");
  return s.trim();
}

const ENTITIES: Record<string, string> = {
  amp: "&",
  lt: "<",
  gt: ">",
  quot: '"',
  apos: "'",
  nbsp: " ",
  copy: "©",
  reg: "®",
  trade: "™",
  hellip: "…",
  mdash: "—",
  ndash: "–",
  laquo: "«",
  raquo: "»",
  lsquo: "‘",
  rsquo: "’",
  ldquo: "“",
  rdquo: "”",
  bull: "•",
  middot: "·",
  euro: "€",
  pound: "£",
  yen: "¥",
  cent: "¢",
  deg: "°",
  times: "×",
  divide: "÷",
  plusmn: "±",
  frac12: "½",
  frac14: "¼",
  frac34: "¾",
  eacute: "é",
  egrave: "è",
  ecirc: "ê",
  euml: "ë",
  agrave: "à",
  aacute: "á",
  acirc: "â",
  auml: "ä",
  ccedil: "ç",
  iuml: "ï",
  icirc: "î",
  ocirc: "ô",
  ouml: "ö",
  ugrave: "ù",
  ucirc: "û",
  uuml: "ü",
  ntilde: "ñ",
  szlig: "ß",
  oelig: "œ",
  aelig: "æ",
  Eacute: "É",
  Agrave: "À",
  Ccedil: "Ç",
};

export function decodeEntities(s: string): string {
  return s
    .replace(/&#x([0-9a-f]+);/gi, (_, h) => safeChar(parseInt(h, 16)))
    .replace(/&#(\d+);/g, (_, d) => safeChar(parseInt(d, 10)))
    .replace(/&([a-z0-9]+);/gi, (m, name) => ENTITIES[name] ?? ENTITIES[name.toLowerCase()] ?? m);
}

function safeChar(code: number): string {
  try {
    return String.fromCodePoint(code);
  } catch {
    return "";
  }
}

export function extractTitle(html: string): string | null {
  const m = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
  return m ? decodeEntities(m[1]!).replace(/\s+/g, " ").trim().slice(0, 300) || null : null;
}

// ---------------------------------------------------------------------------
// Block / anti-bot detection
// ---------------------------------------------------------------------------

export type BlockVendor =
  | "cloudflare"
  | "datadome"
  | "perimeterx"
  | "akamai"
  | "kasada"
  | "imperva"
  | "aws_waf"
  | "vercel"
  | "shape"
  | "distil"
  | "fastly"
  | "google"
  | "sucuri"
  | "generic";

export interface BlockVerdict {
  blocked: boolean;
  /** Stable reason code: http_403, http_429, cloudflare_challenge, captcha, anti_bot, waf, soft_block, empty_html, rate_limited… */
  reason?: string;
  vendor?: BlockVendor;
  /** True when the block is a JavaScript challenge that a real browser can typically pass. */
  challenge?: boolean;
  /** Retry-After hint in milliseconds, when the origin sent one. */
  retryAfterMs?: number;
}

const HTML_CT = /text\/html|application\/xhtml/i;

function parseRetryAfter(v: string | undefined): number | undefined {
  if (!v) return undefined;
  const n = Number(v);
  if (Number.isFinite(n)) return Math.max(0, Math.round(n * 1000));
  const d = Date.parse(v);
  if (!Number.isNaN(d)) return Math.max(0, d - Date.now());
  return undefined;
}

/**
 * Heuristic block-page detection used by the retry engine. Looks at the status, the response
 * headers (WAF signatures) and the first 40 KB of the body (challenge markup, captcha vendors,
 * "access denied" pages served with a 200).
 */
export function looksBlocked(status: number, body: string, headers: Record<string, string>): BlockVerdict {
  const h = lowerKeys(headers);
  const server = (h["server"] ?? "").toLowerCase();
  const ct = h["content-type"] ?? "";
  const isHtml = !ct || HTML_CT.test(ct);
  const head = body.slice(0, 40_000);
  const retryAfterMs = parseRetryAfter(h["retry-after"]);

  // --- header-level signals (strongest) -------------------------------------
  if (h["cf-mitigated"] === "challenge") return { blocked: true, reason: "cloudflare_challenge", vendor: "cloudflare", challenge: true, retryAfterMs };
  if (h["x-datadome"] || h["x-dd-b"] || /datadome/i.test(h["set-cookie"] ?? "")) {
    if (status === 403 || status === 401 || status === 429 || /captcha-delivery|geo\.captcha|dd\.js|DataDome/i.test(head)) return { blocked: true, reason: "anti_bot", vendor: "datadome", challenge: true, retryAfterMs };
  }
  if (h["x-kpsdk-ct"] || h["x-kpsdk-c"] || (/x-kpsdk|kpsdk-/i.test(head) && status >= 400)) return { blocked: true, reason: "anti_bot", vendor: "kasada", challenge: true, retryAfterMs };
  if (h["x-amzn-waf-action"] === "challenge" || h["x-amzn-waf-action"] === "captcha") return { blocked: true, reason: "waf", vendor: "aws_waf", challenge: true, retryAfterMs };
  if (h["x-vercel-mitigated"] === "challenge" || h["x-vercel-protection-bypass"] !== undefined && status === 403) return { blocked: true, reason: "anti_bot", vendor: "vercel", challenge: true, retryAfterMs };
  if (status === 403 && (h["x-iinfo"] || /incap_ses|visid_incap/i.test(h["set-cookie"] ?? ""))) return { blocked: true, reason: "waf", vendor: "imperva", challenge: /_Incapsula_Resource/i.test(head), retryAfterMs };
  if (status === 403 && /_abck|ak_bmsc|bm_sz/i.test(h["set-cookie"] ?? "")) return { blocked: true, reason: "anti_bot", vendor: "akamai", challenge: false, retryAfterMs };

  // --- status-level signals --------------------------------------------------
  if (status === 407) return { blocked: true, reason: "http_407", vendor: "generic" };
  if (status === 429) return { blocked: true, reason: "rate_limited", vendor: vendorFrom(server, head), retryAfterMs };
  if (status === 999) return { blocked: true, reason: "http_999", vendor: "generic" };
  if (status === 403) return { blocked: true, reason: "http_403", vendor: vendorFrom(server, head), challenge: /cf-chl|challenge-platform|__cf_chl|Just a moment/i.test(head), retryAfterMs };
  if (status === 401 && /captcha|challenge|bot|automated/i.test(head)) return { blocked: true, reason: "http_401", vendor: vendorFrom(server, head) };
  if (status === 503 && /cloudflare|just a moment|attention required|checking your browser|ddos-guard|checking if the site connection is secure/i.test(head)) {
    return { blocked: true, reason: "cloudflare_challenge", vendor: "cloudflare", challenge: true, retryAfterMs };
  }
  if ((status === 503 || status === 520 || status === 521 || status === 522 || status === 523 || status === 524 || status === 525 || status === 526 || status === 530) && /cloudflare/i.test(server + " " + head)) {
    return { blocked: true, reason: `origin_${status}`, vendor: "cloudflare", challenge: false, retryAfterMs };
  }
  if (status === 405 && /akamai/i.test(server)) return { blocked: true, reason: "waf", vendor: "akamai" };
  if (status === 202 && /datadome|captcha-delivery/i.test(head)) return { blocked: true, reason: "anti_bot", vendor: "datadome", challenge: true };

  // --- body-level signals (only meaningful for HTML) -------------------------
  // Vendor beacons (Cloudflare JSD, DataDome tags.js, PerimeterX, Imperva, Kasada ips.js, AWS WAF
  // challenge.js) are present on EVERY page of a protected site, so on a 2xx only interstitial-specific
  // markers count; on 4xx/5xx the mere presence of a vendor script is enough.
  if (!isHtml) return { blocked: false };
  const denied = status >= 400;
  if (/cf-chl-bypass|__cf_chl_f_tk|__cf_chl_rt_tk|__cf_chl_tk|cf_chl_opt|<title>\s*Just a moment|cf-chl-widget|challenge-error-text|cf-challenge-running|id="challenge-(running|stage|form)"|Checking your browser before accessing|Verify you are human by completing the action/i.test(head) || (denied && /cf-chl|challenge-platform|cf-turnstile|challenges\.cloudflare\.com/i.test(head))) {
    return { blocked: true, reason: "cloudflare_challenge", vendor: "cloudflare", challenge: true, retryAfterMs };
  }
  if (/geo\.captcha-delivery\.com|<title>\s*DataDome|dd\.js\?|ddCaptcha/i.test(head) || (denied && /datadome/i.test(head))) return { blocked: true, reason: "anti_bot", vendor: "datadome", challenge: true };
  if (/px-captcha|human-challenge|<title>\s*Access to this page has been denied/i.test(head) || (denied && /perimeterx|_pxhd|_pxvid/i.test(head))) return { blocked: true, reason: "anti_bot", vendor: "perimeterx", challenge: true };
  if (/Reference&#32;#\d|Reference #\d+\.[0-9a-f]+\.\d+\.[0-9a-f]+|errors\.edgesuite\.net|akamai\.com\/us\/en\/policies/i.test(head)) return { blocked: true, reason: "anti_bot", vendor: "akamai", challenge: false };
  if (/Incapsula incident|Request unsuccessful\. Incapsula/i.test(head) || (denied && /_Incapsula_Resource/i.test(head))) return { blocked: true, reason: "waf", vendor: "imperva", challenge: true };
  if (/<title>\s*Human Verification|aws-waf-token.*captcha|awswaf.*captcha/i.test(head) || (denied && /awswaf|challenge\.js\?/i.test(head))) return { blocked: true, reason: "waf", vendor: "aws_waf", challenge: true };
  if (/<title>\s*Kasada/i.test(head) || (denied && /kpsdk|ips\.js/i.test(head))) return { blocked: true, reason: "anti_bot", vendor: "kasada", challenge: true };
  if (/Pardon Our Interruption|distil_r_captcha|distil_referrer/i.test(head)) return { blocked: true, reason: "anti_bot", vendor: "distil", challenge: true };
  if (/Vercel Security Checkpoint|_vercel_challenge/i.test(head)) return { blocked: true, reason: "anti_bot", vendor: "vercel", challenge: true };
  if (/<title>\s*Blocked by Shape|_imp_apg_r_/i.test(head) || (denied && /shape-security/i.test(head))) return { blocked: true, reason: "anti_bot", vendor: "shape" };
  if (/Sucuri WebSite Firewall/i.test(head) || (denied && /sucuri\.net/i.test(head))) return { blocked: true, reason: "waf", vendor: "sucuri" };
  if (/www\.google\.com\/sorry\/|Our systems have detected unusual traffic/i.test(head)) return { blocked: true, reason: "captcha", vendor: "google", challenge: false };
  if (/recaptcha\/api\.js|g-recaptcha|hcaptcha\.com|h-captcha|arkoselabs|funcaptcha|<title>\s*[^<]*captcha/i.test(head) && /verify|robot|human|unusual traffic|security check|confirm you are/i.test(head)) {
    return { blocked: true, reason: "captcha", vendor: vendorFrom(server, head), challenge: false };
  }
  if (/<title>\s*(Access Denied|Request Blocked|Forbidden|Blocked|Bot Detected|Security Check|Attention Required|Verification Required|Are you a robot)/i.test(head)) {
    return { blocked: true, reason: "soft_block", vendor: vendorFrom(server, head) };
  }
  if (/enable javascript and cookies to continue|please enable javascript|checking your browser before accessing|verify you are human|verifying you are human|are you a robot|unusual traffic from your (computer|network)|automated access to this (site|page)|suspected bot|request could not be satisfied.*(bot|blocked)/i.test(head) && body.length < 60_000) {
    return { blocked: true, reason: "soft_block", vendor: vendorFrom(server, head), challenge: /enable javascript|checking your browser|verifying/i.test(head) };
  }
  if (status >= 400 && /akamai|imperva|sucuri|cloudflare|awselb|varnish.*block/i.test(server)) return { blocked: true, reason: "waf", vendor: vendorFrom(server, head) };
  // A 200 whose document is a script-only shell with no visible text is typically a JS gate.
  if (status === 200 && /<html/i.test(head) && /<script/i.test(head) && body.length < 20_000) {
    const visible = body.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, "").replace(/\s+/g, "");
    if (visible.length < 20 && !/<(img|a|form|input|p|h1|h2|main|article)\b/i.test(head)) return { blocked: true, reason: "empty_html", vendor: "generic", challenge: true };
  }
  return { blocked: false };
}

function vendorFrom(server: string, head: string): BlockVendor {
  if (/cloudflare/i.test(server) || /cloudflare/i.test(head)) return "cloudflare";
  if (/akamai/i.test(server) || /akamai/i.test(head)) return "akamai";
  if (/imperva|incapsula/i.test(server + head)) return "imperva";
  if (/datadome/i.test(head)) return "datadome";
  if (/perimeterx|_px/i.test(head)) return "perimeterx";
  if (/kasada|kpsdk/i.test(head)) return "kasada";
  if (/awselb|aws/i.test(server) && /waf/i.test(head)) return "aws_waf";
  if (/vercel/i.test(server)) return "vercel";
  if (/sucuri/i.test(server + head)) return "sucuri";
  if (/fastly|varnish/i.test(server)) return "fastly";
  return "generic";
}

function lowerKeys(h: Record<string, string>): Record<string, string> {
  const out: Record<string, string> = {};
  for (const [k, v] of Object.entries(h)) out[k.toLowerCase()] = v;
  return out;
}

/** Whether an origin status is worth retrying on another route (transient upstream/edge errors). */
export function isTransientStatus(status: number): boolean {
  return status === 502 || status === 503 || status === 504 || status === 408 || (status >= 520 && status <= 530);
}