# Trouve-KA — tests d'extraction HTML
# Author: Simon-Pierre Boucher
# Contact: contact@spboucher.ai
from trouveka.parser import decode_html, looks_like_garbage, parse_html
FIXTURE = """
Plomberie XYZ — Plombier à Gatineau
Plombier Gatineau
Nos services
Nous desservons Gatineau et tout l'Outaouais depuis 1998.
TarifsPartenaireCourrielJS
"""
def test_full_extraction():
page = parse_html("https://plomberiexyz.qc.ca/services/", FIXTURE)
assert page.title == "Plomberie XYZ — Plombier à Gatineau"
assert "Outaouais" in page.description
assert page.canonical_url == "https://plomberiexyz.qc.ca/services"
assert page.language == "fr"
assert "Plombier Gatineau" in page.headings
assert page.published_at and page.published_at.year == 2024
# Corps : contenu principal présent, nav/script exclus
assert "Nous desservons Gatineau" in page.body
assert "Menu qui ne doit pas" not in page.body
assert "trackingStuff" not in page.body
# JSON-LD → indices structurés
assert "Gatineau" in page.structured_hints
# Image og:image résolue et nettoyée
assert page.image_url == "https://plomberiexyz.qc.ca/images/facade.jpg"
def test_links_normalized_and_flagged():
page = parse_html("https://plomberiexyz.qc.ca/services/", FIXTURE)
urls = {link.url for link in page.links}
assert "https://plomberiexyz.qc.ca/tarifs" in urls # utm retiré, relatif résolu
nofollow = {link.url: link.nofollow for link in page.links}
assert nofollow["https://autresite.qc.ca/page"] is True
assert not any(u.startswith(("mailto:", "javascript:")) for u in urls)
def test_noindex_detected():
html = 'xcorps'
page = parse_html("https://x.qc.ca/", html)
assert page.noindex is True
assert page.nofollow_page is True
def test_garbage_html_does_not_crash():
page = parse_html("https://x.qc.ca/", b"\x00\xffPas du HTML
")
assert page.url == "https://x.qc.ca/"
def test_decode_latin1_quebec_site():
# Vieux site québécois en ISO-8859-1 : les accents doivent survivre
body = "Éducation à QuébecMinistère de l'Éducation".encode("cp1252")
assert "Éducation à Québec" in decode_html(body)
page = parse_html("https://vieux-site.qc.ca/", body)
assert page.title == "Éducation à Québec"
def test_decode_http_charset_priority():
body = "Montréal".encode("cp1252")
assert "Montréal" in decode_html(body, "iso-8859-1")
def test_binary_garbage_detected():
# Contenu binaire (ex. brotli non décodé) décodé de force → détecté comme charabia
binary = bytes(range(256)) * 40
text = binary.decode("utf-8", errors="replace")
assert looks_like_garbage(text)
assert not looks_like_garbage("Une page normale sur le Québec, avec des accents é à ç.")
assert not looks_like_garbage("")