// author: simon-pierre boucher <contact@spboucher.ai>
import { err, normalizeUrl, ok, tendrilError, type Result } from "@tendril/shared";
import { httpFetch } from "@tendril/fetcher-http";
import { extractLinksFromHtml } from "@tendril/extract";
import { getRobots } from "./robots-cache.js";
import { parseSitemap } from "./sitemap.js";

export type MapSource = "sitemap" | "robots" | "homepage" | "llms" | "index";

export interface MappedLink {
  readonly url: string;
  readonly source: MapSource;
  readonly lastModified?: string;
}

export interface MapOptions {
  readonly search?: string;
  readonly limit?: number;
  readonly includeSubdomains?: boolean;
  readonly sitemapOnly?: boolean;
  readonly ignoreSitemap?: boolean;
}

const SITEMAP_URL_CAP = 50_000;
const DEFAULT_LIMIT = 5_000;
const MAX_SITEMAP_FETCHES = 60;

function apexOf(host: string): string {
  const parts = host.split(".");
  return parts.length <= 2 ? host : parts.slice(-2).join(".");
}

function hostMatches(host: string, baseHost: string, includeSubdomains: boolean): boolean {
  if (host === baseHost) return true;
  if (!includeSubdomains) return false;
  const apex = apexOf(baseHost);
  return host === apex || host.endsWith("." + apex);
}

async function collectSitemaps(
  seeds: string[],
  baseHost: string,
  includeSubdomains: boolean,
  add: (url: string, source: MapSource, lastModified?: string) => void,
): Promise<void> {
  const queue = [...seeds];
  const visited = new Set<string>();
  let fetches = 0;
  let collected = 0;

  while (queue.length > 0 && fetches < MAX_SITEMAP_FETCHES && collected < SITEMAP_URL_CAP) {
    const sm = queue.shift();
    if (sm === undefined) break;
    const key = normalizeUrl(sm);
    const nk = key.ok ? key.value : sm;
    if (visited.has(nk)) continue;
    visited.add(nk);

    const res = await httpFetch(sm, { timeout: 10_000 });
    fetches++;
    if (!res.ok) continue;
    const parsed = parseSitemap(res.value.body);

    for (const child of parsed.childSitemaps) queue.push(child);
    for (const u of parsed.urls) {
      if (collected >= SITEMAP_URL_CAP) break;
      let host: string;
      try {
        host = new URL(u.loc).hostname.toLowerCase();
      } catch {
        continue;
      }
      if (!hostMatches(host, baseHost, includeSubdomains)) continue;
      add(u.loc, "sitemap", u.lastModified);
      collected++;
    }
  }
}

/**
 * Discover a site's URLs without rendering (§14.3): robots.txt Sitemap
 * directives, sitemap.xml (recursing through indexes, capped at 50k), homepage
 * links, and /llms.txt — merged and deduplicated on the normalized URL, with the
 * first-seen source preserved.
 */
export async function mapSite(rawUrl: string, options: MapOptions = {}): Promise<Result<{ links: MappedLink[] }>> {
  let base: URL;
  try {
    base = new URL(rawUrl);
  } catch {
    return err(tendrilError("ERR_INVALID_URL", { details: { url: rawUrl } }));
  }
  const origin = base.origin;
  const baseHost = base.hostname.toLowerCase();
  const includeSubdomains = options.includeSubdomains ?? false;

  const byKey = new Map<string, MappedLink>();
  const add = (url: string, source: MapSource, lastModified?: string): void => {
    const norm = normalizeUrl(url);
    if (!norm.ok) return;
    if (byKey.has(norm.value)) return;
    byKey.set(norm.value, lastModified !== undefined ? { url, source, lastModified } : { url, source });
  };

  if (options.ignoreSitemap !== true) {
    const robots = await getRobots(origin);
    const seeds = [...robots.sitemaps, `${origin}/sitemap.xml`];
    await collectSitemaps(seeds, baseHost, includeSubdomains, add);
  }

  if (options.sitemapOnly !== true) {
    const home = await httpFetch(rawUrl, { timeout: 10_000 });
    if (home.ok) {
      for (const link of extractLinksFromHtml(home.value.body, home.value.finalUrl)) {
        let host: string;
        try {
          host = new URL(link.url).hostname.toLowerCase();
        } catch {
          continue;
        }
        if (hostMatches(host, baseHost, includeSubdomains)) add(link.url, "homepage");
      }
    }

    const llms = await httpFetch(`${origin}/llms.txt`, { timeout: 8_000 });
    if (llms.ok && llms.value.status < 400) {
      const urlRe = /https?:\/\/[^\s)]+/g;
      let m: RegExpExecArray | null;
      while ((m = urlRe.exec(llms.value.body)) !== null) {
        try {
          if (hostMatches(new URL(m[0]).hostname.toLowerCase(), baseHost, includeSubdomains)) add(m[0], "llms");
        } catch {
          continue;
        }
      }
    }
  }

  let links = [...byKey.values()];

  const search = options.search?.trim().toLowerCase();
  if (search !== undefined && search !== "") {
    links = links
      .map((l) => ({ l, pos: l.url.toLowerCase().indexOf(search) }))
      .filter((x) => x.pos !== -1)
      .sort((a, b) => a.pos - b.pos)
      .map((x) => x.l);
  }

  const limit = options.limit ?? DEFAULT_LIMIT;
  return ok({ links: links.slice(0, limit) });
}