# Trouve-KA — parseur HTML
# Author: Simon-Pierre Boucher
# Contact: contact@spboucher.ai

"""Extraction HTML avec selectolax (rapide, tolérant).

Extrait : titre, meta description, corps principal (boilerplate retiré),
headings, liens + anchors + nofollow, canonical, langue, dates de publication,
auteur, indices structurés (JSON-LD/OpenGraph) pour le scoring Québec.
"""

import json
import re
from datetime import datetime

from dateutil import parser as dateparser
from langdetect import DetectorFactory, LangDetectException, detect
from selectolax.parser import HTMLParser

from trouveka.shared import canonicalize_url
from trouveka.types import ExtractedLink, ParsedPage

DetectorFactory.seed = 42  # détection de langue déterministe

# Éléments retirés avant extraction du corps (navigation, pub, scripts…)
_STRIP_SELECTORS = (
    "script", "style", "noscript", "template", "svg", "iframe", "form",
    "nav", "header", "footer", "aside",
    "[role=navigation]", "[role=banner]", "[role=contentinfo]", "[role=complementary]",
    "[aria-hidden=true]", ".cookie-banner", "#cookie-banner", ".cookies", "[class*=cookie-consent]",
)

_MAIN_SELECTORS = ("main", "article", "[role=main]", "#main", "#content", ".main-content", ".content")

_WS = re.compile(r"\s+")
_CTRL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")


def _clean(text: str | None) -> str:
    if not text:
        return ""
    return _WS.sub(" ", _CTRL.sub(" ", text)).strip()


def _parse_date(value: str | None) -> datetime | None:
    if not value:
        return None
    try:
        dt = dateparser.parse(value)
    except (ValueError, OverflowError, TypeError):
        return None
    if dt and dt.year >= 1990:
        return dt
    return None


def _meta(tree: HTMLParser, *names: str) -> str | None:
    for name in names:
        for attr in ("name", "property", "itemprop"):
            node = tree.css_first(f'meta[{attr}="{name}"]')
            if node:
                content = node.attributes.get("content")
                if content and content.strip():
                    return content.strip()
    return None


def _extract_jsonld_hints(tree: HTMLParser, max_blocks: int = 10) -> tuple[list[str], datetime | None]:
    """Extrait des textes utiles des blocs JSON-LD (adresses, noms d'org) + datePublished."""
    hints: list[str] = []
    published: datetime | None = None
    for node in tree.css('script[type="application/ld+json"]')[:max_blocks]:
        raw = node.text()
        if not raw or len(raw) > 100_000:
            continue
        try:
            data = json.loads(raw)
        except (json.JSONDecodeError, ValueError):
            continue
        stack = [data]
        while stack:
            item = stack.pop()
            if isinstance(item, list):
                stack.extend(item[:20])
            elif isinstance(item, dict):
                for key in ("name", "legalName", "addressLocality", "addressRegion",
                            "postalCode", "streetAddress", "telephone"):
                    val = item.get(key)
                    if isinstance(val, str) and len(val) < 200:
                        hints.append(val)
                if published is None:
                    published = _parse_date(item.get("datePublished")) or _parse_date(item.get("dateCreated"))
                stack.extend(v for v in item.values() if isinstance(v, (dict, list)))
                if len(hints) > 60:
                    break
    return hints[:60], published


_META_CHARSET_RE = re.compile(
    rb'<meta[^>]+charset=["\']?\s*([a-zA-Z0-9_-]{2,20})', re.IGNORECASE
)
_CHARSET_ALIASES = {"iso-8859-1": "cp1252", "latin-1": "cp1252", "latin1": "cp1252", "ansi": "cp1252"}


def decode_html(body: bytes, declared_charset: str | None = None) -> str:
    """Décode le HTML : charset HTTP déclaré → meta charset → UTF-8 strict → cp1252.

    Les vieux sites québécois sont souvent en ISO-8859-1/cp1252; décoder
    aveuglément en UTF-8 produit du charabia qui finirait dans l'index.
    """
    candidates: list[str] = []
    if declared_charset:
        candidates.append(declared_charset.lower())
    meta = _META_CHARSET_RE.search(body[:4096])
    if meta:
        candidates.append(meta.group(1).decode("ascii", errors="ignore").lower())
    candidates.extend(["utf-8", "cp1252"])
    for charset in candidates:
        charset = _CHARSET_ALIASES.get(charset, charset)
        try:
            return body.decode(charset)
        except (UnicodeDecodeError, LookupError):
            continue
    return body.decode("utf-8", errors="replace")


def looks_like_garbage(text: str, *, threshold: float = 0.03) -> bool:
    """True si le texte contient trop de caractères de remplacement/contrôle
    (contenu binaire ou mal décodé — ne doit jamais être indexé)."""
    if not text:
        return False
    sample = text[:20_000]
    bad = sum(1 for c in sample if c == "�" or (ord(c) < 32 and c not in "\t\n\r"))
    return bad / len(sample) > threshold


def parse_html(
    url: str, html: bytes | str, *, max_links: int = 300, charset: str | None = None
) -> ParsedPage:
    """Parse une page HTML en ParsedPage. Ne lève pas sur du HTML dégueulasse."""
    if isinstance(html, bytes):
        html = decode_html(html, charset)
    tree = HTMLParser(html)

    # Directives robots de la page
    robots_meta = (_meta(tree, "robots", "googlebot") or "").lower()
    noindex = "noindex" in robots_meta
    nofollow_page = "nofollow" in robots_meta

    # Canonical
    canonical = None
    link_canonical = tree.css_first('link[rel="canonical"]')
    if link_canonical:
        canonical = canonicalize_url(link_canonical.attributes.get("href") or "", base=url)

    # Titre / description
    title = _clean(tree.css_first("title").text() if tree.css_first("title") else None)
    og_title = _meta(tree, "og:title")
    if not title and og_title:
        title = _clean(og_title)
    description = _clean(_meta(tree, "description", "og:description") or "")

    # Langue déclarée
    declared_lang = None
    html_node = tree.css_first("html")
    if html_node:
        lang_attr = (html_node.attributes.get("lang") or "").strip().lower()
        if lang_attr:
            declared_lang = lang_attr[:2]

    # Image représentative (og:image / twitter:image) — hotlinkée, jamais crawlée
    image_url = None
    raw_image = _meta(tree, "og:image", "og:image:url", "twitter:image", "twitter:image:src")
    if raw_image:
        candidate = canonicalize_url(raw_image, base=url)
        if candidate and len(candidate) <= 500:
            image_url = candidate

    # Dates / auteur
    published = _parse_date(_meta(tree, "article:published_time", "datePublished", "date", "dc.date"))
    modified = _parse_date(_meta(tree, "article:modified_time", "dateModified"))
    author = _meta(tree, "author", "article:author")

    # Indices structurés (avant strip : JSON-LD est dans <script>)
    structured_hints, jsonld_published = _extract_jsonld_hints(tree)
    published = published or jsonld_published

    # Liens (avant strip : les liens de nav comptent pour la découverte)
    links: list[ExtractedLink] = []
    seen: set[str] = set()
    for node in tree.css("a[href]"):
        if len(links) >= max_links:
            break
        href = node.attributes.get("href") or ""
        if href.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
            continue
        normalized = canonicalize_url(href, base=url)
        if not normalized or normalized in seen:
            continue
        seen.add(normalized)
        rel = (node.attributes.get("rel") or "").lower()
        links.append(
            ExtractedLink(
                url=normalized,
                anchor=_clean(node.text())[:200],
                nofollow="nofollow" in rel or nofollow_page,
            )
        )

    # Headings
    headings = [_clean(n.text()) for n in tree.css("h1, h2, h3")[:40]]
    headings = [h for h in headings if h]

    # Corps principal : strip du boilerplate, puis zone principale si identifiable
    for selector in _STRIP_SELECTORS:
        for node in tree.css(selector):
            node.decompose()
    body_node = None
    for selector in _MAIN_SELECTORS:
        body_node = tree.css_first(selector)
        if body_node:
            break
    if body_node is None:
        body_node = tree.css_first("body") or tree.root
    body = _clean(body_node.text(separator=" ", deep=True))[:200_000]

    # Langue : déclaration HTML validée/complétée par détection statistique
    language = declared_lang if declared_lang in ("fr", "en") else None
    sample = body[:4000] or title
    if sample and len(sample) > 40:
        try:
            detected = detect(sample)
            if detected in ("fr", "en"):
                # La détection l'emporte si elle contredit une déclaration douteuse
                language = detected if language is None or detected != language else language
        except LangDetectException:
            pass

    return ParsedPage(
        url=url,
        canonical_url=canonical,
        title=title[:500],
        description=description[:1000],
        body=body,
        headings=headings,
        language=language,
        links=links,
        published_at=published,
        modified_at=modified,
        author=_clean(author)[:200] if author else None,
        image_url=image_url,
        noindex=noindex,
        nofollow_page=nofollow_page,
        structured_hints=structured_hints,
    )