spb/trouve-ka Public
Trouve-KA — moteur de recherche web indépendant, Québec-first. Crawler distribué, index OpenSearch, ranking bilingue, galerie d'images. En prod : www.trouve-ka.com
Python 76.8%
TypeScript 15.7%
SQL 3.9%
Shell 1.4%
CSS 1.3%
Dockerfile 0.7%
1# Trouve-KA — tests d'extraction HTML2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45from trouveka.parser import decode_html, looks_like_garbage, parse_html67FIXTURE = """<!DOCTYPE html>8<html lang="fr">9<head>10 <title>Plomberie XYZ — Plombier à Gatineau</title>11 <meta name="description" content="Service de plomberie en Outaouais depuis 1998.">12 <link rel="canonical" href="https://plomberiexyz.qc.ca/services">13 <meta property="article:published_time" content="2024-06-12T10:00:00-04:00">14 <script type="application/ld+json">15 {"@type": "Plumber", "name": "Plomberie XYZ",16 "address": {"addressLocality": "Gatineau", "addressRegion": "QC", "postalCode": "J8X 3X3"}}17 </script>18</head>19<body>20 <nav><a href="/menu1">Menu qui ne doit pas polluer le corps</a></nav>21 <main>22 <h1>Plombier Gatineau</h1>23 <h2>Nos services</h2>24 <p>Nous desservons Gatineau et tout l'Outaouais depuis 1998.</p>25 <a href="/tarifs?utm_source=nav">Tarifs</a>26 <a href="https://autresite.qc.ca/page" rel="nofollow">Partenaire</a>27 <a href="mailto:info@x.ca">Courriel</a>28 <a href="javascript:void(0)">JS</a>29 </main>30 <footer>123 rue Principale, Gatineau (Québec) J8X 3X3 — 819-555-1234</footer>31 <script>trackingStuff();</script>32</body>33</html>"""343536def test_full_extraction():37 page = parse_html("https://plomberiexyz.qc.ca/services/", FIXTURE)38 assert page.title == "Plomberie XYZ — Plombier à Gatineau"39 assert "Outaouais" in page.description40 assert page.canonical_url == "https://plomberiexyz.qc.ca/services"41 assert page.language == "fr"42 assert "Plombier Gatineau" in page.headings43 assert page.published_at and page.published_at.year == 202444 # Corps : contenu principal présent, nav/script exclus45 assert "Nous desservons Gatineau" in page.body46 assert "Menu qui ne doit pas" not in page.body47 assert "trackingStuff" not in page.body48 # JSON-LD → indices structurés49 assert "Gatineau" in page.structured_hints505152def test_links_normalized_and_flagged():53 page = parse_html("https://plomberiexyz.qc.ca/services/", FIXTURE)54 urls = {link.url for link in page.links}55 assert "https://plomberiexyz.qc.ca/tarifs" in urls # utm retiré, relatif résolu56 nofollow = {link.url: link.nofollow for link in page.links}57 assert nofollow["https://autresite.qc.ca/page"] is True58 assert not any(u.startswith(("mailto:", "javascript:")) for u in urls)596061def test_noindex_detected():62 html = '<html><head><meta name="robots" content="noindex, nofollow"><title>x</title></head><body>corps</body></html>'63 page = parse_html("https://x.qc.ca/", html)64 assert page.noindex is True65 assert page.nofollow_page is True666768def test_garbage_html_does_not_crash():69 page = parse_html("https://x.qc.ca/", b"\x00\xffPas du HTML <div <<< &&& </span>")70 assert page.url == "https://x.qc.ca/"717273def test_decode_latin1_quebec_site():74 # Vieux site québécois en ISO-8859-1 : les accents doivent survivre75 body = "<html><head><meta charset=\"iso-8859-1\"><title>Éducation à Québec</title></head><body>Ministère de l'Éducation</body></html>".encode("cp1252")76 assert "Éducation à Québec" in decode_html(body)77 page = parse_html("https://vieux-site.qc.ca/", body)78 assert page.title == "Éducation à Québec"798081def test_decode_http_charset_priority():82 body = "<html><title>Montréal</title></html>".encode("cp1252")83 assert "Montréal" in decode_html(body, "iso-8859-1")848586def test_binary_garbage_detected():87 # Contenu binaire (ex. brotli non décodé) décodé de force → détecté comme charabia88 binary = bytes(range(256)) * 4089 text = binary.decode("utf-8", errors="replace")90 assert looks_like_garbage(text)91 assert not looks_like_garbage("Une page normale sur le Québec, avec des accents é à ç.")92 assert not looks_like_garbage("")93