import { promises as dns } from "node:dns";
import { canonicalizeHtml, jaccard, registrableDomain, sha256, shingles, type SensorType, type Tier } from "@websensor/core";
import { httpFetch } from "./fetcher";
import { parseFeed } from "./rss";
import { parseSitemap } from "./sitemap";
import { detectFlavor } from "./statusjson";

/**
 * Deep discovery — the acquisition stage of the Source Factory. Starting from an organization (domain +
 * optional hints) it walks robots.txt → sitemaps → feeds (`<link rel=alternate>`, well-known paths, official
 * sub-domains) → the homepage's own navigation (newsroom, press, investors, changelog, security, pricing,
 * legal, careers, docs, status) → known status-page providers → GitHub organization → EDGAR → Hugging Face →
 * OpenAPI documents → web posture, and validates EVERY candidate by fetching and parsing it exactly as the
 * connector would. Nothing is assumed from a URL pattern.
 *
 * The result is a ranked list of candidate sensors with the evidence the Factory needs to score them:
 * page class, first-party confidence, expected change frequency, fetch cost and a content fingerprint used
 * to drop duplicates (two feeds exposing the same items, two pages with the same text).
 */
export type CandidateKind = "news" | "press" | "blog" | "ir" | "filings" | "changelog" | "releases" | "security" | "pricing" | "legal" | "careers" | "leadership" | "docs" | "api" | "status" | "sitemap" | "models" | "data" | "posture" | "other";

export interface DeepCandidate {
  url: string;
  type: SensorType;
  connector: string;
  kind: CandidateKind;
  name: string;
  config: Record<string, unknown>;
  suggestedTier: Tier;
  evidence: string;
  /** rough information value 0–1 (before the Factory's importance-aware scoring) */
  value: number;
  itemCount?: number;
  /** items per day observed in the feed/list window (null when unknown) */
  itemsPerDay?: number | null;
  /** 1 = the organization's own registrable domain; 0.9 = official channel on a platform (GitHub, HF, status provider) */
  firstPartyConfidence: number;
  fetchCost: { bytes: number; ms: number };
  /** content fingerprint for de-duplication (sorted item keys, or canonical text hash) */
  fingerprint?: string;
  itemKeys?: string[];
  textShingles?: Set<string>;
  title?: string;
}

export interface DeepDiscoveryHints {
  cik?: string | number;
  github_org?: string;
  github_repos?: string[];
  hf_author?: string;
  status_url?: string;
  hosts?: string[];
  urls?: { url: string; kind?: string; connector?: string; type?: string; tier?: Tier; config?: Record<string, unknown>; name?: string }[];
  posture?: boolean;
}

export interface DeepDiscoveryOptions {
  hints?: DeepDiscoveryHints;
  /** maximum number of HTTP requests for this organization (default 90) */
  budget?: number;
  /** wall-clock deadline for the whole organization (default 120 s); optional stages are skipped once reached */
  deadlineMs?: number;
  concurrency?: number;
  sensorIdForLogs?: string;
  probePages?: boolean;
  probeSubdomains?: boolean;
  probePosture?: boolean;
  probeOpenApi?: boolean;
}

export interface DeepDiscoveryResult {
  domain: string;
  /** origin actually used after following the homepage redirect (may be another registrable domain) */
  origin: string;
  canonicalDomain: string;
  homeStatus: number;
  /** anti-bot / challenge detected on the homepage */
  blocked: boolean;
  requests: number;
  candidates: DeepCandidate[];
  rejected: { url: string; reason: string }[];
  notes: string[];
  durationMs: number;
}

// ------------------------------------------------------------------------------------------------
// Vocabularies
// ------------------------------------------------------------------------------------------------

const FEED_PATHS = ["/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml", "/feed/", "/feeds/all.atom.xml", "/blog/feed", "/blog/rss", "/blog/rss.xml", "/blog/feed.xml", "/blog/atom.xml", "/blog/index.xml", "/news/feed", "/news/rss", "/news/rss.xml", "/news/feed.xml", "/news.rss", "/news.xml", "/newsroom/rss", "/newsroom/feed", "/newsroom/rss.xml", "/press/rss", "/press/feed", "/press-releases/rss", "/press-releases/feed", "/rss/news", "/rss/press-releases.xml", "/rss/news-releases.xml", "/rss/pressreleases.xml", "/investors/rss", "/investors/feed", "/en/feed", "/en/rss", "/en/rss.xml", "/en/news/rss", "/changelog/feed", "/changelog.rss", "/changelog/rss.xml", "/changelog.xml", "/releases.atom", "/releases/feed", "/security/feed", "/security/rss", "/advisories/rss", "/advisories.rss", "/updates/feed", "/updates/rss", "/whats-new/feed", "/?feed=rss2", "/feed/atom", "/feeds/posts/default", "/feed.json"];
const SITEMAP_PATHS = ["/sitemap.xml", "/sitemap_index.xml", "/sitemap-index.xml", "/sitemaps/sitemap.xml", "/news-sitemap.xml", "/sitemap/news.xml", "/sitemap-news.xml", "/sitemap.txt"];
const OPENAPI_PATHS = ["/openapi.json", "/swagger.json", "/api/openapi.json", "/api/swagger.json", "/v1/openapi.json", "/api/v1/openapi.json", "/.well-known/openapi.json", "/swagger/v1/swagger.json", "/api-docs", "/openapi.yaml", "/api/openapi.yaml"];
const SUBDOMAIN_KINDS: [string, CandidateKind][] = [["blog", "blog"], ["news", "news"], ["newsroom", "news"], ["press", "press"], ["media", "press"], ["investors", "ir"], ["investor", "ir"], ["ir", "ir"], ["status", "status"], ["docs", "docs"], ["developer", "docs"], ["developers", "docs"], ["security", "security"], ["trust", "security"], ["careers", "careers"], ["jobs", "careers"], ["changelog", "changelog"], ["updates", "changelog"], ["engineering", "blog"], ["research", "blog"], ["about", "other"], ["corporate", "news"], ["api", "api"]];

/** Page classes probed from navigation links and well-known paths. `paths` are fallbacks when the navigation has no link. */
const PAGE_KINDS: { kind: CandidateKind; label: string; hrefRe: RegExp; textRe: RegExp; paths: string[]; tier: Tier; value: number; max: number }[] = [
  { kind: "pricing", label: "pricing", hrefRe: /\/(pricing|plans|plans-and-pricing|tarifs|preise|precios)(\/|$|\?)/i, textRe: /^(pricing|plans|plans & pricing|tarifs|preise|precios)$/i, paths: ["/pricing", "/plans", "/pricing/"], tier: "C", value: 0.75, max: 1 },
  { kind: "changelog", label: "changelog", hrefRe: /\/(changelog|change-log|release-notes|releasenotes|whats-new|what-s-new|updates|product-updates)(\/|$|\?)/i, textRe: /^(changelog|change log|release notes|what'?s new|product updates|updates)$/i, paths: ["/changelog", "/release-notes", "/whats-new", "/updates"], tier: "B", value: 0.75, max: 1 },
  { kind: "security", label: "security advisories", hrefRe: /\/(security|security-advisories|advisories|psirt|trust|security-bulletins|bulletins|vulnerabilit)/i, textRe: /^(security|security advisories|advisories|trust center|trust|psirt|security bulletins)$/i, paths: ["/security", "/security/advisories", "/trust", "/.well-known/security.txt"], tier: "B", value: 0.7, max: 2 },
  { kind: "news", label: "newsroom", hrefRe: /\/(news|newsroom|news-room|media|media-center|media-centre|actualites|actualit%C3%A9s|nouvelles|aktuelles|noticias|notizie)(\/|$|\?)/i, textRe: /^(news|newsroom|news ?room|media|media cent(er|re)|latest news|in the news|actualit[ée]s|nouvelles|aktuelles|noticias)$/i, paths: ["/news", "/newsroom", "/news/", "/en/news", "/media"], tier: "B", value: 0.65, max: 1 },
  { kind: "press", label: "press releases", hrefRe: /\/(press|press-releases|pressreleases|press-room|pressroom|press-center|communiques|communiqu%C3%A9s|presse|prensa)(\/|$|\?)/i, textRe: /^(press|press releases|press ?room|press cent(er|re)|communiqu[ée]s( de presse)?|presse|prensa)$/i, paths: ["/press", "/press-releases", "/press/", "/newsroom/press-releases"], tier: "B", value: 0.65, max: 1 },
  { kind: "ir", label: "investor relations", hrefRe: /\/(investors|investor-relations|investor|ir|relations-investisseurs|investor-center)(\/|$|\?)/i, textRe: /^(investors|investor relations|investor cent(er|re)|ir|relations investisseurs|shareholders)$/i, paths: ["/investors", "/investor-relations", "/investors/", "/investor"], tier: "C", value: 0.6, max: 1 },
  { kind: "legal", label: "terms of service", hrefRe: /\/(terms|terms-of-service|terms-of-use|tos|legal\/terms|conditions|cgu|cgv|legal)(\/|$|\?|\.)/i, textRe: /^(terms|terms of (service|use)|terms & conditions|terms and conditions|legal|conditions g[ée]n[ée]rales|cgu|cgv)$/i, paths: ["/terms", "/legal/terms", "/terms-of-service", "/legal"], tier: "C", value: 0.6, max: 1 },
  { kind: "legal", label: "privacy policy", hrefRe: /\/(privacy|privacy-policy|privacypolicy|legal\/privacy|politique-de-confidentialite|datenschutz|privacidad)(\/|$|\?|\.)/i, textRe: /^(privacy|privacy policy|privacy notice|politique de confidentialit[ée]|datenschutz|privacidad)$/i, paths: ["/privacy", "/legal/privacy", "/privacy-policy"], tier: "C", value: 0.55, max: 1 },
  { kind: "careers", label: "careers", hrefRe: /\/(careers|jobs|join-us|work-with-us|carrieres|carri%C3%A8res|karriere|empleo)(\/|$|\?)/i, textRe: /^(careers|jobs|join us|work with us|carri[èe]res|karriere|empleo|we'?re hiring)$/i, paths: ["/careers", "/jobs", "/careers/"], tier: "D", value: 0.35, max: 1 },
  { kind: "leadership", label: "leadership", hrefRe: /\/(leadership|management|executive-team|our-team|team|board-of-directors|board|about\/leadership|company\/leadership|direction|governance)(\/|$|\?)/i, textRe: /^(leadership|leadership team|management|management team|executive team|our team|board of directors|governance|direction)$/i, paths: ["/leadership", "/about/leadership", "/company/leadership", "/about/team"], tier: "D", value: 0.45, max: 1 },
  { kind: "docs", label: "documentation", hrefRe: /\/(docs|documentation|developers|developer|api-docs|reference)(\/|$|\?)/i, textRe: /^(docs|documentation|developers|developer|api|api reference|api docs)$/i, paths: ["/docs", "/documentation", "/developers"], tier: "D", value: 0.4, max: 1 },
  { kind: "data", label: "open data", hrefRe: /\/(open-data|opendata|data|statistics|statistiques|datasets|publications)(\/|$|\?)/i, textRe: /^(open data|data|statistics|statistiques|datasets|publications)$/i, paths: [], tier: "C", value: 0.4, max: 1 },
];

const STATUS_HOST_RE = /^(https?:\/\/)?((status|statuspage|uptime|health|trust|system-status|servicestatus)\.[^/\s"']+|[^/\s"']*\.(statuspage\.io|instatus\.com|betteruptime\.com|betterstack\.com|incident\.io|status\.io|hund\.io|statuspal\.io|freshstatus\.io|status\.page|statusgator\.com))/i;
const CHALLENGE_RE = /cf-browser-verification|challenge-platform|__cf_chl|Attention Required!|Just a moment\.\.\.|Access Denied|Request unsuccessful\. Incapsula|akamai.*bot|_Incapsula_Resource|Please enable JavaScript to view|captcha/i;

// ------------------------------------------------------------------------------------------------

export async function discoverOrganization(domain: string, opts: DeepDiscoveryOptions = {}): Promise<DeepDiscoveryResult> {
  const started = Date.now();
  const id = opts.sensorIdForLogs ?? `factory_${domain}`;
  const budget = opts.budget ?? 90;
  const hints = opts.hints ?? {};
  const notes: string[] = [];
  const rejected: { url: string; reason: string }[] = [];
  const found = new Map<string, DeepCandidate>();
  let requests = 0;

  const add = (c: DeepCandidate): void => {
    const k = c.url.replace(/\/$/, "");
    const prev = found.get(k);
    if (!prev || prev.value < c.value) found.set(k, c);
  };
  const deadline = started + (opts.deadlineMs ?? 120_000);
  const budgetLeft = (): boolean => requests < budget && Date.now() < deadline;
  const get = async (url: string, o: { timeoutMs?: number; maxBytes?: number; accept?: string; headers?: Record<string, string>; userAgent?: string } = {}) => {
    requests++;
    return httpFetch(id, url, { timeoutMs: o.timeoutMs ?? 15_000, maxBytes: o.maxBytes ?? 3 * 1024 * 1024, accept: o.accept, headers: o.headers, userAgent: o.userAgent });
  };

  // ---- 1. robots.txt --------------------------------------------------------------------------------------
  let base = `https://${domain}`;
  const sitemapsFromRobots: string[] = [];
  let robotsOk = false;
  let robotsMentionsAi = false;
  const robots = await get(`${base}/robots.txt`, { timeoutMs: 12_000, maxBytes: 512 * 1024 }).catch(() => null);
  if (robots?.body && robots.meta.status === 200 && !/^\s*<!doctype html|<html/i.test(robots.body.toString("utf8").slice(0, 300))) {
    robotsOk = true;
    const text = robots.body.toString("utf8");
    for (const line of text.split(/\r?\n/)) {
      const m = line.match(/^\s*sitemap:\s*(\S+)/i);
      if (m) sitemapsFromRobots.push(m[1]!);
    }
    robotsMentionsAi = /GPTBot|ClaudeBot|Google-Extended|CCBot|anthropic-ai|PerplexityBot|Bytespider|Applebot-Extended|meta-externalagent/i.test(text);
  }

  // ---- 2. homepage ------------------------------------------------------------------------------------------
  const home = await get(`${base}/`, { timeoutMs: 20_000, maxBytes: 4 * 1024 * 1024 });
  const homeStatus = home.meta.status;
  let blocked = false;
  const homeHtml = home.body && home.meta.status < 400 ? home.body.toString("utf8") : "";
  if ([403, 429, 503].includes(homeStatus) || (homeHtml && CHALLENGE_RE.test(homeHtml.slice(0, 20_000)) && homeHtml.length < 60_000)) {
    blocked = true;
    notes.push(`homepage ${homeStatus}${homeHtml ? " (anti-bot challenge)" : ""}`);
  }
  let origin = base;
  try {
    const fu = new URL(home.meta.finalUrl || base);
    origin = fu.origin;
    if (registrableDomain(fu.hostname) !== registrableDomain(domain)) notes.push(`homepage redirects to ${fu.hostname}`);
  } catch {
    // keep base
  }
  base = origin;
  const canonicalDomain = registrableDomain(new URL(base).hostname);
  const ownHosts = new Set<string>([registrableDomain(domain), canonicalDomain, ...(hints.hosts ?? []).map((h) => registrableDomain(h))]);
  const isOwn = (u: string): boolean => {
    try {
      return ownHosts.has(registrableDomain(new URL(u).hostname));
    } catch {
      return false;
    }
  };

  const homeCanon = homeHtml ? canonicalizeHtml(homeHtml, home.meta.finalUrl || base) : null;
  const homeShingles = homeCanon ? shingles(homeCanon.text) : null;
  const alternates: string[] = [];
  const statusLinks = new Set<string>();
  const githubOrgs = new Set<string>();
  const hfAuthors = new Set<string>();
  const pageLinks = new Map<CandidateKind, string[]>();
  if (homeHtml) {
    for (const m of homeHtml.matchAll(/<link[^>]+rel=["']alternate["'][^>]*>/gi)) {
      const tag = m[0];
      if (!/application\/(rss|atom)\+xml|application\/feed\+json/i.test(tag)) continue;
      const href = tag.match(/href=["']([^"']+)["']/i)?.[1];
      const abs = href ? safeAbs(href, home.meta.finalUrl || base) : null;
      if (abs) alternates.push(abs);
    }
    // Navigation links: classify by href pattern first, then by anchor text.
    const anchors = [...homeHtml.matchAll(/<a\b[^>]*href=["']([^"'#]+)["'][^>]*>([\s\S]{0,300}?)<\/a>/gi)].map((m) => ({ href: safeAbs(m[1]!, home.meta.finalUrl || base), text: m[2]!.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim() }));
    for (const a of anchors) {
      if (!a.href) continue;
      const gh = a.href.match(/^https?:\/\/(?:www\.)?github\.com\/([A-Za-z0-9_.-]+)\/?(?:$|\?)/);
      if (gh && !/^(features|about|pricing|topics|login|signup|sponsors|marketplace|explore|apps|orgs|enterprise|solutions|security|team|customer-stories|readme|collections|events|trending|search|settings|notifications|new|organizations|site|contact)$/i.test(gh[1]!)) githubOrgs.add(gh[1]!);
      const hf = a.href.match(/^https?:\/\/huggingface\.co\/([A-Za-z0-9_.-]+)\/?$/);
      if (hf && !/^(models|datasets|spaces|docs|pricing|papers|blog|join|login|enterprise|posts|tasks|learn|chat)$/i.test(hf[1]!)) hfAuthors.add(hf[1]!);
      if (STATUS_HOST_RE.test(a.href)) statusLinks.add(a.href);
      if (!isOwn(a.href)) continue;
      let path = "";
      try {
        path = new URL(a.href).pathname;
      } catch {
        continue;
      }
      if (path.split("/").filter(Boolean).length > 3) continue; // deep article links are not sections
      for (const pk of PAGE_KINDS) {
        if (pk.hrefRe.test(path) || pk.textRe.test(a.text)) {
          const arr = pageLinks.get(pk.kind) ?? [];
          if (!arr.includes(a.href) && arr.length < 4) arr.push(a.href);
          pageLinks.set(pk.kind, arr);
        }
      }
    }
  }
  if (hints.github_org) githubOrgs.add(hints.github_org);
  if (hints.hf_author) hfAuthors.add(hints.hf_author);
  if (hints.status_url) statusLinks.add(hints.status_url);

  // ---- 3. official sub-domains (cheap DNS first) ------------------------------------------------------------
  const subHosts: { host: string; kind: CandidateKind }[] = [];
  if (opts.probeSubdomains !== false) {
    const rd = registrableDomain(domain);
    // Wildcard DNS (everything resolves) makes existence checks meaningless: only probe the core labels then.
    const wildcard = await resolves(`ws-probe-${Date.now().toString(36)}.${rd}`);
    if (wildcard) notes.push("wildcard DNS");
    const labels = wildcard ? ["blog", "news", "status", "investors", "docs", "security"] : [...new Set([...SUBDOMAIN_KINDS.map(([l]) => l)])];
    await parallel(labels, 8, async (label) => {
      const host = `${label}.${rd}`;
      if (host === new URL(base).hostname) return;
      if (wildcard || (await resolves(host))) subHosts.push({ host, kind: SUBDOMAIN_KINDS.find(([l]) => l === label)![1] });
    });
    if (wildcard) {
      // keep only sub-domains that actually answer with a distinct page
      const alive: typeof subHosts = [];
      await parallel(subHosts, 4, async (s) => {
        if (!budgetLeft()) return;
        const o = await get(`https://${s.host}/`, { timeoutMs: 10_000, maxBytes: 512 * 1024 });
        const fh = o.meta.finalUrl ? new URL(o.meta.finalUrl).hostname : "";
        if (o.meta.status === 200 && fh === s.host) alive.push(s);
      });
      subHosts.splice(0, subHosts.length, ...alive);
    }
    for (const h of hints.hosts ?? []) if (!subHosts.some((s) => s.host === h)) subHosts.push({ host: h, kind: "other" });
    if (subHosts.length) notes.push(`sub-domains: ${subHosts.map((s) => s.host).join(", ")}`);
  }

  // ---- 4. feeds ---------------------------------------------------------------------------------------------
  const feedUrls = new Set<string>(alternates);
  for (const p of FEED_PATHS) feedUrls.add(base + p);
  for (const s of subHosts) {
    if (s.kind === "status" || s.kind === "api") continue;
    for (const p of ["/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml", "/feed/", "/?feed=rss2"]) feedUrls.add(`https://${s.host}${p}`);
  }
  // Feeds are cheap and high-value: spend up to 55 % of the budget here.
  const feedList = [...feedUrls].slice(0, Math.max(10, Math.floor(budget * 0.55)));
  const feeds: DeepCandidate[] = [];
  await parallel(feedList, opts.concurrency ?? 4, async (url) => {
    if (!budgetLeft()) return;
    const o = await get(url, { timeoutMs: 12_000, maxBytes: 4 * 1024 * 1024, accept: "application/rss+xml, application/atom+xml, application/feed+json, application/xml;q=0.9, */*;q=0.5" });
    if (!o.body || o.meta.status !== 200) return;
    const text = o.body.toString("utf8");
    if (/^\s*<!doctype html|<html/i.test(text.slice(0, 400))) return;
    try {
      const f = parseFeed(text, o.meta.finalUrl);
      if (!f.items.length) return rejected.push({ url, reason: "empty feed" });
      const dates = f.items.map((i) => (i.publishedAt ? new Date(i.publishedAt).getTime() : NaN)).filter((t) => !Number.isNaN(t));
      const dated = dates.length / f.items.length;
      const itemsPerDay = itemsPerDayOf(dates);
      const finalUrl = o.meta.finalUrl || url;
      const kind = feedKind(finalUrl, f.title);
      const keys = f.items.map((i) => i.key).sort();
      const stale = dates.length && Math.max(...dates) < Date.now() - 3 * 365 * 86400e3;
      if (stale) return rejected.push({ url, reason: "feed abandoned (> 3 years)" });
      feeds.push({
        url: finalUrl,
        type: f.kind === "atom" ? "ATOM" : "RSS",
        connector: "rss",
        kind,
        name: feedName(kind, finalUrl),
        config: {},
        suggestedTier: itemsPerDay !== null && itemsPerDay < 0.05 ? "C" : "B",
        evidence: `${alternates.includes(url) ? "link[rel=alternate]" : "well-known path"} · ${f.items.length} items${itemsPerDay !== null ? ` · ${itemsPerDay}/day` : ""}`,
        value: 0.75 + 0.15 * dated + (kind === "news" || kind === "press" ? 0.1 : kind === "security" || kind === "changelog" ? 0.08 : 0),
        itemCount: f.items.length,
        itemsPerDay,
        firstPartyConfidence: isOwn(finalUrl) ? 1 : 0.7,
        fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs },
        fingerprint: sha256(keys.join("\n")),
        itemKeys: keys,
        title: f.title,
      });
    } catch {
      // not a feed
    }
  });
  // De-duplicate feeds sharing ≥ 70 % of their items (rss + atom of the same channel, /feed vs /rss.xml).
  feeds.sort((a, b) => b.value - a.value || (b.itemCount ?? 0) - (a.itemCount ?? 0));
  const keptFeeds: DeepCandidate[] = [];
  for (const f of feeds) {
    const dup = keptFeeds.find((k) => k.itemKeys && f.itemKeys && jaccard(new Set(k.itemKeys), new Set(f.itemKeys)) >= 0.7);
    if (dup) rejected.push({ url: f.url, reason: `duplicate of ${dup.url}` });
    else keptFeeds.push(f);
  }
  for (const f of keptFeeds) add(f);

  // ---- 5. sitemaps -------------------------------------------------------------------------------------------
  const sitemapCandidates = [...new Set([...sitemapsFromRobots.filter((u) => !/image|video/i.test(u)).sort((a, b) => Number(/news/i.test(b)) - Number(/news/i.test(a))).slice(0, 6), ...SITEMAP_PATHS.map((p) => base + p)])];
  const sitemaps: DeepCandidate[] = [];
  await parallel(sitemapCandidates, 3, async (url) => {
    if (!budgetLeft() || sitemaps.length >= 3) return;
    const o = await get(url, { timeoutMs: 15_000, maxBytes: 20 * 1024 * 1024, accept: "application/xml, text/xml, text/plain;q=0.8, */*;q=0.5" });
    if (!o.body || o.meta.status !== 200) return;
    const text = o.body.toString("utf8");
    if (/^\s*<!doctype html|<html/i.test(text.slice(0, 400))) return;
    try {
      const s = parseSitemap(text);
      const n = s.kind === "sitemapindex" ? s.children.length : s.entries.length;
      if (!n) return;
      const lastmods = s.entries.filter((e) => e.lastmod).length;
      const isNews = /news/i.test(url) || s.entries.some((e) => e.publishedAt);
      sitemaps.push({ url: o.meta.finalUrl || url, type: "SITEMAP", connector: "sitemap", kind: "sitemap", name: isNews ? "news sitemap" : "sitemap", config: { maxChildren: 4, maxUrls: 3000 }, suggestedTier: isNews ? "B" : "C", evidence: `${sitemapsFromRobots.includes(url) ? "robots.txt" : "well-known path"} · ${n} ${s.kind === "sitemapindex" ? "children" : "urls"}`, value: 0.45 + (s.entries.length ? 0.3 * (lastmods / s.entries.length) : 0.15) + (isNews ? 0.2 : 0), itemCount: n, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });
    } catch {
      // not a sitemap
    }
  });
  sitemaps.sort((a, b) => b.value - a.value);
  if (sitemaps[0]) add(sitemaps[0]);

  // ---- 6. status pages ----------------------------------------------------------------------------------------
  const statusHosts = new Set<string>();
  if (hints.status_url) {
    try {
      statusHosts.add(new URL(hints.status_url).origin);
    } catch {
      // ignore
    }
  }
  for (const s of statusLinks) {
    try {
      statusHosts.add(new URL(s).origin);
    } catch {
      // ignore
    }
  }
  for (const s of subHosts) if (s.kind === "status") statusHosts.add(`https://${s.host}`);
  let statusFound = false;
  for (const root of statusHosts) {
    if (statusFound || !budgetLeft()) break;
    for (const [path, connector] of [["/api/v2/summary.json", "statuspage"], ["/summary.json", "statusjson"], ["/api/v1/summary", "statusjson"]] as const) {
      if (statusFound || !budgetLeft()) break;
      const o = await get(root + path, { timeoutMs: 12_000, accept: "application/json" });
      if (!o.body || o.meta.status !== 200) continue;
      const text = o.body.toString("utf8");
      try {
        const j = JSON.parse(text) as Record<string, unknown>;
        if (connector === "statuspage" && Array.isArray(j.incidents) && (j.components || j.status)) {
          add({ url: root + path, type: "STATUSPAGE", connector: "statuspage", kind: "status", name: "status", config: {}, suggestedTier: "S", evidence: `Atlassian Statuspage (${new URL(root).host})`, value: 0.95, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });
          statusFound = true;
        } else if (connector === "statusjson") {
          const flavor = detectFlavor(j);
          if (flavor) {
            add({ url: root + path, type: "STATUSPAGE", connector: "statusjson", kind: "status", name: "status", config: { flavor }, suggestedTier: "S", evidence: `${flavor} status JSON (${new URL(root).host})`, value: 0.93, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });
            statusFound = true;
          }
        }
      } catch {
        // not JSON
      }
    }
  }

  // ---- 7. pages (server-rendered sections) --------------------------------------------------------------------
  if (opts.probePages !== false && !blocked) {
    const pageJobs: { kind: CandidateKind; label: string; tier: Tier; value: number; urls: string[]; max: number }[] = [];
    for (const pk of PAGE_KINDS) {
      const linked = pageLinks.get(pk.kind) ?? [];
      // feeds already cover news/press/blog for this org → the HTML index page adds little
      if ((pk.kind === "news" || pk.kind === "press") && keptFeeds.some((f) => f.kind === pk.kind || f.kind === "news")) continue;
      const shallowFirst = (a: string, b: string): number => safePath(a).split("/").length - safePath(b).split("/").length || a.length - b.length;
      const urls = [...new Set([...linked.filter((u) => pk.hrefRe.test(safePath(u)) || linked.length <= 2).sort(shallowFirst), ...pk.paths.map((p) => base + p)])].slice(0, 4);
      if (!urls.length) continue;
      pageJobs.push({ kind: pk.kind, label: pk.label, tier: pk.tier, value: pk.value, urls, max: pk.max });
    }
    const pageCanon: { url: string; sh: Set<string>; hash: string }[] = [];
    const probed = new Set<string>();
    await parallel(pageJobs, 3, async (job) => {
      let kept = 0;
      for (const url of job.urls) {
        if (kept >= job.max || !budgetLeft()) break;
        const pk = url.replace(/\/$/, "");
        if (probed.has(pk) || found.has(pk)) continue;
        probed.add(pk);
        const o = await get(url, { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024, accept: "text/html, application/xhtml+xml;q=0.9, */*;q=0.5" });
        if (o.meta.status !== 200 || !o.body) continue;
        const finalPath = safePath(o.meta.finalUrl);
        if (finalPath === "/" || finalPath === "") continue; // redirected home
        if (!isOwn(o.meta.finalUrl || url)) continue;
        if (!/text\/html|xhtml/i.test(o.meta.contentType ?? "") && /^\s*[{[]/.test(o.body.toString("utf8").slice(0, 50))) continue;
        const c = canonicalizeHtml(o.body.toString("utf8"), o.meta.finalUrl || url);
        if (c.text.length < 300) {
          rejected.push({ url, reason: `thin (${c.text.length} chars)` });
          continue;
        }
        const sh = shingles(c.text);
        if (homeCanon && (c.canonicalHash === homeCanon.canonicalHash || (homeShingles && jaccard(sh, homeShingles) > 0.8))) {
          rejected.push({ url, reason: "same as homepage" });
          continue;
        }
        const dup = pageCanon.find((p) => p.hash === c.canonicalHash || jaccard(p.sh, sh) > 0.85);
        if (dup) {
          rejected.push({ url, reason: `duplicate of ${dup.url}` });
          continue;
        }
        pageCanon.push({ url: o.meta.finalUrl || url, sh, hash: c.canonicalHash });
        const isSecurityTxt = /security\.txt$/.test(url);
        add({ url: o.meta.finalUrl || url, type: "HTML", connector: "http", kind: job.kind, name: isSecurityTxt ? "security.txt" : job.label, config: {}, suggestedTier: isSecurityTxt ? "D" : job.tier, evidence: `${pageLinks.get(job.kind)?.includes(url) ? "linked from homepage" : "well-known path"} · GET 200 · ${c.text.length} chars · distinct from homepage`, value: isSecurityTxt ? 0.3 : job.value, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs }, fingerprint: c.canonicalHash, textShingles: sh, title: c.title ?? undefined });
        kept++;
      }
    });
  }

  // ---- 8. GitHub organization → repositories' releases ----------------------------------------------------------
  const repos = new Set<string>(hints.github_repos ?? []);
  for (const org of [...githubOrgs].slice(0, 1)) {
    if (repos.size >= 4 || !budgetLeft()) break;
    const token = process.env.GITHUB_TOKEN;
    if (token) {
      const o = await get(`https://api.github.com/search/repositories?q=org:${encodeURIComponent(org)}+fork:false&sort=stars&order=desc&per_page=5`, { accept: "application/vnd.github+json", headers: { authorization: `Bearer ${token}`, "x-github-api-version": "2022-11-28" } });
      if (o.body && o.meta.status === 200) {
        try {
          const j = JSON.parse(o.body.toString("utf8")) as { items?: { full_name: string; archived?: boolean; stargazers_count?: number }[] };
          for (const r of j.items ?? []) if (!r.archived && (r.stargazers_count ?? 0) >= 50) repos.add(r.full_name);
        } catch {
          // ignore
        }
      }
    } else {
      // Unauthenticated: the organization page lists pinned / popular repositories server-side.
      const o = await get(`https://github.com/${org}`, { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024 });
      if (o.body && o.meta.status === 200) {
        const html = o.body.toString("utf8");
        const counts = new Map<string, number>();
        for (const m of html.matchAll(new RegExp(`href="/${org.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/([A-Za-z0-9_.-]+)"`, "g"))) {
          const r = m[1]!;
          if (/^(followers|following|repositories|projects|packages|people|teams|sponsoring|discussions|orgs|\.github)$/i.test(r)) continue;
          counts.set(r, (counts.get(r) ?? 0) + 1);
        }
        const pinned = [...html.matchAll(/class="[^"]*pinned-item-list-item[^"]*"[\s\S]{0,1500}?href="\/([^/"]+)\/([^/"]+)"/g)].map((m) => `${m[1]}/${m[2]}`).filter((f) => f.toLowerCase().startsWith(org.toLowerCase() + "/"));
        for (const f of pinned) repos.add(f);
        for (const [r] of [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 6)) if (repos.size < 6) repos.add(`${org}/${r}`);
      }
    }
  }
  const repoList = [...repos].slice(0, 6);
  let releasesKept = 0;
  await parallel(repoList, 3, async (full) => {
    if (releasesKept >= 4 || !budgetLeft()) return;
    for (const kind of ["releases", "tags"] as const) {
      const url = `https://github.com/${full}/${kind}.atom`;
      const o = await get(url, { timeoutMs: 12_000, accept: "application/atom+xml, application/xml;q=0.9" });
      if (!o.body || o.meta.status !== 200) {
        if (o.meta.status === 404) return;
        continue;
      }
      try {
        const f = parseFeed(o.body.toString("utf8"), url);
        if (!f.items.length) continue;
        const dates = f.items.map((i) => (i.publishedAt ? new Date(i.publishedAt).getTime() : NaN)).filter((t) => !Number.isNaN(t));
        if (dates.length && Math.max(...dates) < Date.now() - 2 * 365 * 86400e3) return rejected.push({ url, reason: "repository inactive (> 2 years)" });
        add({ url, type: "GITHUB_RELEASE", connector: "github", kind: "releases", name: `${full.split("/")[1]} ${kind}`, config: { repo: full, kind }, suggestedTier: "B", evidence: `GitHub ${kind}.atom · ${f.items.length} items`, value: kind === "releases" ? 0.8 : 0.65, itemCount: f.items.length, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs }, fingerprint: sha256(f.items.map((i) => i.key).sort().join("\n")) });
        releasesKept++;
        return;
      } catch {
        // not a feed
      }
    }
  });

  // ---- 9. Hugging Face models ------------------------------------------------------------------------------------
  for (const author of [...hfAuthors].slice(0, 2)) {
    if (!budgetLeft()) break;
    const url = `https://huggingface.co/api/models?author=${encodeURIComponent(author)}&sort=lastModified&direction=-1&limit=50`;
    const o = await get(url, { accept: "application/json" });
    if (!o.body || o.meta.status !== 200) continue;
    try {
      const j = JSON.parse(o.body.toString("utf8")) as unknown[];
      if (Array.isArray(j) && j.length) add({ url, type: "REST_API", connector: "jsonlist", kind: "models", name: "hugging face models", config: { keyField: "id", titleField: "id", dateField: "lastModified", compareFields: ["pipeline_tag", "library_name"], maxItems: 50 }, suggestedTier: "B", evidence: `Hugging Face author ${author} · ${j.length} models`, value: 0.85, itemCount: j.length, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });
    } catch {
      // ignore
    }
  }

  // ---- 10. EDGAR filings -------------------------------------------------------------------------------------------
  if (hints.cik !== undefined && budgetLeft()) {
    const cik = String(hints.cik).replace(/\D/g, "").padStart(10, "0");
    const url = `https://data.sec.gov/submissions/CIK${cik}.json`;
    const ua = process.env.WS_EDGAR_USER_AGENT ?? "WebSensor (contact@websensor.io)";
    const o = await get(url, { accept: "application/json", userAgent: ua, headers: { "user-agent": ua }, timeoutMs: 30_000, maxBytes: 30 * 1024 * 1024 });
    if (o.body && o.meta.status === 200 && /"filings"/.test(o.body.toString("utf8").slice(0, 200_000))) {
      add({ url, type: "REST_API", connector: "edgar", kind: "filings", name: "edgar filings", config: { forms: ["8-K", "10-K", "10-Q", "6-K", "20-F", "40-F", "S-1", "SC 13D", "DEF 14A"] }, suggestedTier: "B", evidence: `SEC EDGAR submissions CIK ${cik}`, value: 0.9, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });
    } else rejected.push({ url, reason: `edgar ${o.meta.status}` });
  }

  // ---- 11. OpenAPI documents ---------------------------------------------------------------------------------------
  if (opts.probeOpenApi !== false && (subHosts.some((s) => s.kind === "docs" || s.kind === "api") || pageLinks.has("docs")) && budgetLeft()) {
    const roots = [base, ...subHosts.filter((s) => s.kind === "docs" || s.kind === "api").map((s) => `https://${s.host}`)].slice(0, 3);
    let openapiFound = false;
    for (const root of roots) {
      if (openapiFound) break;
      for (const p of OPENAPI_PATHS.slice(0, 7)) {
        if (openapiFound || !budgetLeft()) break;
        const o = await get(root + p, { timeoutMs: 15_000, maxBytes: 8 * 1024 * 1024, accept: "application/json, application/yaml, */*;q=0.5" });
        if (!o.body || o.meta.status !== 200) continue;
        const text = o.body.toString("utf8").slice(0, 4000);
        if (/^\s*[{[]/.test(text) && /"(openapi|swagger)"\s*:/.test(text)) {
          add({ url: root + p, type: "JSON", connector: "openapi", kind: "api", name: "openapi", config: {}, suggestedTier: "C", evidence: `OpenAPI document (${new URL(root).host})`, value: 0.7, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });
          openapiFound = true;
        }
      }
    }
  }

  // ---- 12. explicit hint URLs (validated like everything else) --------------------------------------------------------
  for (const h of hints.urls ?? []) {
    if (!budgetLeft()) break;
    const known = h.url.replace(/\/$/, "");
    if (found.has(known)) continue;
    const o = await get(h.url, { timeoutMs: 15_000, maxBytes: 6 * 1024 * 1024 });
    if (!o.body || o.meta.status !== 200) {
      rejected.push({ url: h.url, reason: `hint ${o.meta.status || o.error?.code}` });
      continue;
    }
    const text = o.body.toString("utf8");
    const kind = (h.kind ?? "other") as CandidateKind;
    const tier = h.tier ?? (kind === "status" ? "S" : kind === "pricing" || kind === "legal" || kind === "ir" ? "C" : "B");
    const cost = { bytes: o.meta.contentLength, ms: o.meta.durationMs };
    if (h.connector) {
      add({ url: h.url, type: (h.type ?? (h.connector === "rss" ? "RSS" : h.connector === "jsonlist" || h.connector === "edgar" ? "REST_API" : h.connector === "statuspage" || h.connector === "statusjson" ? "STATUSPAGE" : h.connector === "sitemap" ? "SITEMAP" : "HTML")) as SensorType, connector: h.connector, kind, name: h.name ?? kind, config: h.config ?? {}, suggestedTier: tier, evidence: "explicit hint · GET 200", value: 0.8, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost });
      continue;
    }
    // Sniff: feed → rss; JSON → http/json; HTML → http
    if (!/^\s*<!doctype html|<html/i.test(text.slice(0, 400)) && /<(rss|feed|rdf:RDF)\b/i.test(text.slice(0, 2000))) {
      try {
        const f = parseFeed(text, o.meta.finalUrl);
        if (f.items.length) {
          add({ url: o.meta.finalUrl || h.url, type: f.kind === "atom" ? "ATOM" : "RSS", connector: "rss", kind, name: h.name ?? feedName(kind, h.url), config: {}, suggestedTier: tier, evidence: `explicit hint · feed · ${f.items.length} items`, value: 0.85, itemCount: f.items.length, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost, fingerprint: sha256(f.items.map((i) => i.key).sort().join("\n")), itemKeys: f.items.map((i) => i.key).sort() });
          continue;
        }
      } catch {
        // fallthrough
      }
    }
    if (/^\s*[{[]/.test(text.slice(0, 50))) {
      add({ url: h.url, type: "JSON", connector: "http", kind, name: h.name ?? kind, config: h.config ?? {}, suggestedTier: tier, evidence: "explicit hint · JSON 200", value: 0.65, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost });
      continue;
    }
    const c = canonicalizeHtml(text, o.meta.finalUrl || h.url);
    if (c.text.length < 300) {
      rejected.push({ url: h.url, reason: `hint thin (${c.text.length} chars)` });
      continue;
    }
    add({ url: o.meta.finalUrl || h.url, type: "HTML", connector: "http", kind, name: h.name ?? PAGE_KINDS.find((p) => p.kind === kind)?.label ?? kind, config: h.config ?? {}, suggestedTier: tier, evidence: `explicit hint · GET 200 · ${c.text.length} chars`, value: 0.7, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost, fingerprint: c.canonicalHash, title: c.title ?? undefined });
  }

  // ---- 13. web posture (no network here: validated in shadow by the dns/tls connectors) ------------------------------
  if (opts.probePosture || hints.posture) {
    const rd = registrableDomain(domain);
    add({ url: `dns://${rd}`, type: "DNS", connector: "dns", kind: "posture", name: "dns", config: {}, suggestedTier: "D", evidence: "posture", value: 0.3, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: 0, ms: 0 } });
    add({ url: `tls://${new URL(base).hostname}`, type: "TLS", connector: "tls", kind: "posture", name: "tls certificate", config: {}, suggestedTier: "D", evidence: "posture", value: 0.3, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: 0, ms: 0 } });
    if (robotsOk) add({ url: `https://${domain}/robots.txt`, type: "HTML", connector: "http", kind: "posture", name: "robots.txt", config: {}, suggestedTier: "D", evidence: robotsMentionsAi ? "robots.txt names AI crawlers" : "robots.txt", value: robotsMentionsAi ? 0.45 : 0.3, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: robots?.meta.contentLength ?? 0, ms: robots?.meta.durationMs ?? 0 } });
  }

  const candidates = [...found.values()].sort((a, b) => b.value - a.value);
  // Strip the heavy de-dup helpers before returning (they are not serializable and no longer needed).
  for (const c of candidates) {
    delete c.textShingles;
    if (c.itemKeys && c.itemKeys.length > 60) c.itemKeys = c.itemKeys.slice(0, 60);
  }
  return { domain, origin: base, canonicalDomain, homeStatus, blocked, requests, candidates, rejected, notes, durationMs: Date.now() - started };
}

// ------------------------------------------------------------------------------------------------

/** Expected publication rate from item dates: the median gap between consecutive items (robust to one bogus date). */
export function itemsPerDayOf(dates: number[]): number | null {
  const d = [...dates].filter((t) => Number.isFinite(t)).sort((a, b) => b - a).slice(0, 30);
  if (d.length < 3) return null;
  const gaps = d.slice(1).map((t, i) => (d[i]! - t) / 86400e3).filter((g) => g >= 0).sort((a, b) => a - b);
  if (!gaps.length) return null;
  const median = gaps[Math.floor(gaps.length / 2)]!;
  if (median <= 0) return 50;
  return Math.round((1 / median) * 1000) / 1000;
}

function feedKind(url: string, title?: string): CandidateKind {
  const s = `${url} ${title ?? ""}`.toLowerCase();
  if (/press|communiqu|pressemitteilung|prensa|news-release|newsrelease/.test(s)) return "press";
  if (/investor|\/ir\/|ir\.|sec-filing|filings/.test(s)) return "ir";
  if (/security|advisor|psirt|bulletin|vulnerab|cve/.test(s)) return "security";
  if (/changelog|release-notes|releasenotes|releases|whats-new|updates/.test(s)) return "changelog";
  if (/blog|engineering|research|insights|stories/.test(s)) return "blog";
  if (/news|actualit|nouvelles|aktuell|noticias|announcement/.test(s)) return "news";
  if (/jobs|careers/.test(s)) return "careers";
  return "news";
}

function feedName(kind: CandidateKind, url: string): string {
  switch (kind) {
    case "press":
      return "press releases feed";
    case "ir":
      return "investor news feed";
    case "security":
      return "security feed";
    case "changelog":
      return "changelog feed";
    case "blog":
      return /engineering/i.test(url) ? "engineering blog feed" : /research/i.test(url) ? "research feed" : "blog feed";
    case "careers":
      return "jobs feed";
    default:
      return "news feed";
  }
}

function safePath(u: string): string {
  try {
    return new URL(u).pathname.replace(/\/$/, "");
  } catch {
    return "";
  }
}

function safeAbs(h: string, base: string): string | null {
  try {
    const u = new URL(h, base);
    if (!u.protocol.startsWith("http")) return null;
    u.hash = "";
    return u.toString();
  } catch {
    return null;
  }
}

async function resolves(host: string): Promise<boolean> {
  try {
    const r = await dns.lookup(host, { all: true });
    return r.length > 0;
  } catch {
    return false;
  }
}

async function parallel<T>(items: T[], limit: number, fn: (item: T) => Promise<unknown>): Promise<void> {
  let i = 0;
  const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
    while (i < items.length) {
      const item = items[i++]!;
      try {
        await fn(item);
      } catch {
        // best effort
      }
    }
  });
  await Promise.all(workers);
}