spb/trouve-ka Public
Trouve-KA — moteur de recherche web indépendant, Québec-first. Crawler distribué, index OpenSearch, ranking bilingue, galerie d'images. En prod : www.trouve-ka.com
Python 76.8%
TypeScript 15.7%
SQL 3.9%
Shell 1.4%
CSS 1.3%
Dockerfile 0.7%
1# Trouve-KA — tests d'extraction HTML2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45from trouveka.parser import decode_html, looks_like_garbage, parse_html67FIXTURE = """<!DOCTYPE html>8<html lang="fr">9<head>10 <title>Plomberie XYZ — Plombier à Gatineau</title>11 <meta name="description" content="Service de plomberie en Outaouais depuis 1998.">12 <link rel="canonical" href="https://plomberiexyz.qc.ca/services">13 <meta property="article:published_time" content="2024-06-12T10:00:00-04:00">14 <meta property="og:image" content="/images/facade.jpg?utm_source=og">15 <script type="application/ld+json">16 {"@type": "Plumber", "name": "Plomberie XYZ",17 "address": {"addressLocality": "Gatineau", "addressRegion": "QC", "postalCode": "J8X 3X3"}}18 </script>19</head>20<body>21 <nav><a href="/menu1">Menu qui ne doit pas polluer le corps</a></nav>22 <main>23 <h1>Plombier Gatineau</h1>24 <h2>Nos services</h2>25 <p>Nous desservons Gatineau et tout l'Outaouais depuis 1998.</p>26 <a href="/tarifs?utm_source=nav">Tarifs</a>27 <a href="https://autresite.qc.ca/page" rel="nofollow">Partenaire</a>28 <a href="mailto:info@x.ca">Courriel</a>29 <a href="javascript:void(0)">JS</a>30 </main>31 <footer>123 rue Principale, Gatineau (Québec) J8X 3X3 — 819-555-1234</footer>32 <script>trackingStuff();</script>33</body>34</html>"""353637def test_full_extraction():38 page = parse_html("https://plomberiexyz.qc.ca/services/", FIXTURE)39 assert page.title == "Plomberie XYZ — Plombier à Gatineau"40 assert "Outaouais" in page.description41 assert page.canonical_url == "https://plomberiexyz.qc.ca/services"42 assert page.language == "fr"43 assert "Plombier Gatineau" in page.headings44 assert page.published_at and page.published_at.year == 202445 # Corps : contenu principal présent, nav/script exclus46 assert "Nous desservons Gatineau" in page.body47 assert "Menu qui ne doit pas" not in page.body48 assert "trackingStuff" not in page.body49 # JSON-LD → indices structurés50 assert "Gatineau" in page.structured_hints51 # Image og:image résolue et nettoyée52 assert page.image_url == "https://plomberiexyz.qc.ca/images/facade.jpg"535455def test_links_normalized_and_flagged():56 page = parse_html("https://plomberiexyz.qc.ca/services/", FIXTURE)57 urls = {link.url for link in page.links}58 assert "https://plomberiexyz.qc.ca/tarifs" in urls # utm retiré, relatif résolu59 nofollow = {link.url: link.nofollow for link in page.links}60 assert nofollow["https://autresite.qc.ca/page"] is True61 assert not any(u.startswith(("mailto:", "javascript:")) for u in urls)626364def test_noindex_detected():65 html = '<html><head><meta name="robots" content="noindex, nofollow"><title>x</title></head><body>corps</body></html>'66 page = parse_html("https://x.qc.ca/", html)67 assert page.noindex is True68 assert page.nofollow_page is True697071def test_garbage_html_does_not_crash():72 page = parse_html("https://x.qc.ca/", b"\x00\xffPas du HTML <div <<< &&& </span>")73 assert page.url == "https://x.qc.ca/"747576def test_decode_latin1_quebec_site():77 # Vieux site québécois en ISO-8859-1 : les accents doivent survivre78 body = "<html><head><meta charset=\"iso-8859-1\"><title>Éducation à Québec</title></head><body>Ministère de l'Éducation</body></html>".encode("cp1252")79 assert "Éducation à Québec" in decode_html(body)80 page = parse_html("https://vieux-site.qc.ca/", body)81 assert page.title == "Éducation à Québec"828384def test_decode_http_charset_priority():85 body = "<html><title>Montréal</title></html>".encode("cp1252")86 assert "Montréal" in decode_html(body, "iso-8859-1")878889def test_binary_garbage_detected():90 # Contenu binaire (ex. brotli non décodé) décodé de force → détecté comme charabia91 binary = bytes(range(256)) * 4092 text = binary.decode("utf-8", errors="replace")93 assert looks_like_garbage(text)94 assert not looks_like_garbage("Une page normale sur le Québec, avec des accents é à ç.")95 assert not looks_like_garbage("")96