spb/trouve-ka Public

1# Trouve-KA — parseur HTML2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45"""Extraction HTML avec selectolax (rapide, tolérant).67Extrait : titre, meta description, corps principal (boilerplate retiré),8headings, liens + anchors + nofollow, canonical, langue, dates de publication,9auteur, indices structurés (JSON-LD/OpenGraph) pour le scoring Québec.10"""1112import json13import re14from datetime import datetime1516from dateutil import parser as dateparser17from langdetect import DetectorFactory, LangDetectException, detect18from selectolax.parser import HTMLParser1920from trouveka.shared import canonicalize_url21from trouveka.types import ExtractedLink, ParsedPage2223DetectorFactory.seed = 42  # détection de langue déterministe2425# Éléments retirés avant extraction du corps (navigation, pub, scripts…)26_STRIP_SELECTORS = (27    "script", "style", "noscript", "template", "svg", "iframe", "form",28    "nav", "header", "footer", "aside",29    "[role=navigation]", "[role=banner]", "[role=contentinfo]", "[role=complementary]",30    "[aria-hidden=true]", ".cookie-banner", "#cookie-banner", ".cookies", "[class*=cookie-consent]",31)3233_MAIN_SELECTORS = ("main", "article", "[role=main]", "#main", "#content", ".main-content", ".content")3435_WS = re.compile(r"\s+")36_CTRL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")373839def _clean(text: str | None) -> str:40    if not text:41        return ""42    return _WS.sub(" ", _CTRL.sub(" ", text)).strip()434445def _parse_date(value: str | None) -> datetime | None:46    if not value:47        return None48    try:49        dt = dateparser.parse(value)50    except (ValueError, OverflowError, TypeError):51        return None52    if dt and dt.year >= 1990:53        return dt54    return None555657def _meta(tree: HTMLParser, *names: str) -> str | None:58    for name in names:59        for attr in ("name", "property", "itemprop"):60            node = tree.css_first(f'meta[{attr}="{name}"]')61            if node:62                content = node.attributes.get("content")63                if content and content.strip():64                    return content.strip()65    return None666768def _extract_jsonld_hints(tree: HTMLParser, max_blocks: int = 10) -> tuple[list[str], datetime | None]:69    """Extrait des textes utiles des blocs JSON-LD (adresses, noms d'org) + datePublished."""70    hints: list[str] = []71    published: datetime | None = None72    for node in tree.css('script[type="application/ld+json"]')[:max_blocks]:73        raw = node.text()74        if not raw or len(raw) > 100_000:75            continue76        try:77            data = json.loads(raw)78        except (json.JSONDecodeError, ValueError):79            continue80        stack = [data]81        while stack:82            item = stack.pop()83            if isinstance(item, list):84                stack.extend(item[:20])85            elif isinstance(item, dict):86                for key in ("name", "legalName", "addressLocality", "addressRegion",87                            "postalCode", "streetAddress", "telephone"):88                    val = item.get(key)89                    if isinstance(val, str) and len(val) < 200:90                        hints.append(val)91                if published is None:92                    published = _parse_date(item.get("datePublished")) or _parse_date(item.get("dateCreated"))93                stack.extend(v for v in item.values() if isinstance(v, (dict, list)))94                if len(hints) > 60:95                    break96    return hints[:60], published979899_META_CHARSET_RE = re.compile(100    rb'<meta[^>]+charset=["\']?\s*([a-zA-Z0-9_-]{2,20})', re.IGNORECASE101)102_CHARSET_ALIASES = {"iso-8859-1": "cp1252", "latin-1": "cp1252", "latin1": "cp1252", "ansi": "cp1252"}103104105def decode_html(body: bytes, declared_charset: str | None = None) -> str:106    """Décode le HTML : charset HTTP déclaré → meta charset → UTF-8 strict → cp1252.107108    Les vieux sites québécois sont souvent en ISO-8859-1/cp1252; décoder109    aveuglément en UTF-8 produit du charabia qui finirait dans l'index.110    """111    candidates: list[str] = []112    if declared_charset:113        candidates.append(declared_charset.lower())114    meta = _META_CHARSET_RE.search(body[:4096])115    if meta:116        candidates.append(meta.group(1).decode("ascii", errors="ignore").lower())117    candidates.extend(["utf-8", "cp1252"])118    for charset in candidates:119        charset = _CHARSET_ALIASES.get(charset, charset)120        try:121            return body.decode(charset)122        except (UnicodeDecodeError, LookupError):123            continue124    return body.decode("utf-8", errors="replace")125126127def looks_like_garbage(text: str, *, threshold: float = 0.03) -> bool:128    """True si le texte contient trop de caractères de remplacement/contrôle129    (contenu binaire ou mal décodé — ne doit jamais être indexé)."""130    if not text:131        return False132    sample = text[:20_000]133    bad = sum(1 for c in sample if c == "�" or (ord(c) < 32 and c not in "\t\n\r"))134    return bad / len(sample) > threshold135136137def parse_html(138    url: str, html: bytes | str, *, max_links: int = 300, charset: str | None = None139) -> ParsedPage:140    """Parse une page HTML en ParsedPage. Ne lève pas sur du HTML dégueulasse."""141    if isinstance(html, bytes):142        html = decode_html(html, charset)143    tree = HTMLParser(html)144145    # Directives robots de la page146    robots_meta = (_meta(tree, "robots", "googlebot") or "").lower()147    noindex = "noindex" in robots_meta148    nofollow_page = "nofollow" in robots_meta149150    # Canonical151    canonical = None152    link_canonical = tree.css_first('link[rel="canonical"]')153    if link_canonical:154        canonical = canonicalize_url(link_canonical.attributes.get("href") or "", base=url)155156    # Titre / description157    title = _clean(tree.css_first("title").text() if tree.css_first("title") else None)158    og_title = _meta(tree, "og:title")159    if not title and og_title:160        title = _clean(og_title)161    description = _clean(_meta(tree, "description", "og:description") or "")162163    # Langue déclarée164    declared_lang = None165    html_node = tree.css_first("html")166    if html_node:167        lang_attr = (html_node.attributes.get("lang") or "").strip().lower()168        if lang_attr:169            declared_lang = lang_attr[:2]170171    # Image représentative (og:image / twitter:image) — hotlinkée, jamais crawlée172    image_url = None173    raw_image = _meta(tree, "og:image", "og:image:url", "twitter:image", "twitter:image:src")174    if raw_image:175        candidate = canonicalize_url(raw_image, base=url)176        if candidate and len(candidate) <= 500:177            image_url = candidate178179    # Dates / auteur180    published = _parse_date(_meta(tree, "article:published_time", "datePublished", "date", "dc.date"))181    modified = _parse_date(_meta(tree, "article:modified_time", "dateModified"))182    author = _meta(tree, "author", "article:author")183184    # Indices structurés (avant strip : JSON-LD est dans <script>)185    structured_hints, jsonld_published = _extract_jsonld_hints(tree)186    published = published or jsonld_published187188    # Liens (avant strip : les liens de nav comptent pour la découverte)189    links: list[ExtractedLink] = []190    seen: set[str] = set()191    for node in tree.css("a[href]"):192        if len(links) >= max_links:193            break194        href = node.attributes.get("href") or ""195        if href.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):196            continue197        normalized = canonicalize_url(href, base=url)198        if not normalized or normalized in seen:199            continue200        seen.add(normalized)201        rel = (node.attributes.get("rel") or "").lower()202        links.append(203            ExtractedLink(204                url=normalized,205                anchor=_clean(node.text())[:200],206                nofollow="nofollow" in rel or nofollow_page,207            )208        )209210    # Headings211    headings = [_clean(n.text()) for n in tree.css("h1, h2, h3")[:40]]212    headings = [h for h in headings if h]213214    # Corps principal : strip du boilerplate, puis zone principale si identifiable215    for selector in _STRIP_SELECTORS:216        for node in tree.css(selector):217            node.decompose()218    body_node = None219    for selector in _MAIN_SELECTORS:220        body_node = tree.css_first(selector)221        if body_node:222            break223    if body_node is None:224        body_node = tree.css_first("body") or tree.root225    body = _clean(body_node.text(separator=" ", deep=True))[:200_000]226227    # Langue : déclaration HTML validée/complétée par détection statistique228    language = declared_lang if declared_lang in ("fr", "en") else None229    sample = body[:4000] or title230    if sample and len(sample) > 40:231        try:232            detected = detect(sample)233            if detected in ("fr", "en"):234                # La détection l'emporte si elle contredit une déclaration douteuse235                language = detected if language is None or detected != language else language236        except LangDetectException:237            pass238239    return ParsedPage(240        url=url,241        canonical_url=canonical,242        title=title[:500],243        description=description[:1000],244        body=body,245        headings=headings,246        language=language,247        links=links,248        published_at=published,249        modified_at=modified,250        author=_clean(author)[:200] if author else None,251        image_url=image_url,252        noindex=noindex,253        nofollow_page=nofollow_page,254        structured_hints=structured_hints,255    )256