spb/trouve-ka Public

1# Trouve-KA — pipeline de requête et ranking2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45"""Pipeline de requête : normalisation → langue → lieux → requête OpenSearch.67BM25 (multi_match bilingue) au cœur, function_score par-dessus :8Québec, autorité, fraîcheur, localité. Poids nommés, non figés (§9).9"""1011import re12import unicodedata13from typing import Any1415from trouveka.classifier import QUEBEC_TOPONYMS1617# Poids de ranking — à calibrer avec le dataset d'évaluation (§16)18W_PAGE_QUEBEC = 1.619W_DOMAIN_QUEBEC = 1.020W_AUTHORITY = 0.821W_FRESHNESS = 0.522W_LOCALITY = 2.02324_FR_HINTS = {"le", "la", "les", "des", "une", "un", "du", "de", "et", "ou", "pour",25             "avec", "dans", "sur", "meilleur", "meilleure", "comment", "où", "quel", "quelle"}26_EN_HINTS = {"the", "a", "an", "of", "and", "or", "for", "with", "in", "on", "best", "how", "what", "where"}272829def _fold(text: str) -> str:30    return "".join(c for c in unicodedata.normalize("NFD", text) if unicodedata.category(c) != "Mn")313233def analyze_query(q: str) -> dict[str, Any]:34    """Normalisation, détection de langue heuristique, extraction de lieux québécois."""35    normalized = re.sub(r"\s+", " ", q).strip()[:200]36    tokens = [t.lower() for t in re.findall(r"[\w'-]+", normalized, re.UNICODE)]3738    fr_hits = sum(1 for t in tokens if t in _FR_HINTS)39    en_hits = sum(1 for t in tokens if t in _EN_HINTS)40    has_accents = normalized != _fold(normalized)41    language = "fr" if (fr_hits > en_hits or has_accents) else ("en" if en_hits > fr_hits else None)4243    # Lieux : tokens simples + bigrammes contre le gazetteer44    folded_gazetteer = {_fold(t): t for t in QUEBEC_TOPONYMS}45    locations: list[str] = []46    candidates = tokens + [f"{a}-{b}" for a, b in zip(tokens, tokens[1:])] + [47        f"{a} {b}" for a, b in zip(tokens, tokens[1:])48    ]49    for cand in candidates:50        folded = _fold(cand)51        if folded in folded_gazetteer:52            locations.append(folded_gazetteer[folded])53    return {"query": normalized, "language": language, "locations": sorted(set(locations))}545556def build_search_body(57    q: str,58    *,59    page: int = 1,60    limit: int = 10,61    language: str | None = None,62    category: str | None = None,63    quebec_only: bool = False,64    freshness: str | None = None,65    images_only: bool = False,66) -> tuple[dict[str, Any], dict[str, Any]]:67    """Construit le corps de requête OpenSearch. Retourne (body, analyse)."""68    analysis = analyze_query(q)69    normalized = analysis["query"]7071    # BM25 bilingue : les deux analyzers interrogés, le meilleur champ gagne72    text_query: dict[str, Any] = {73        "multi_match": {74            "query": normalized,75            "type": "most_fields",76            "fields": [77                "title^4", "title.en^4",78                "headings^2", "headings.en^2",79                "description^2", "description.en^2",80                "body", "body.en",81            ],82            "fuzziness": "AUTO",83            "prefix_length": 2,84        }85    }8687    filters: list[dict[str, Any]] = []88    if language in ("fr", "en"):89        filters.append({"term": {"language": language}})90    if category:91        filters.append({"term": {"categories": category}})92    if quebec_only:93        filters.append({94            "bool": {95                "should": [96                    {"range": {"page_quebec_score": {"gte": 0.45}}},97                    {"range": {"domain_quebec_score": {"gte": 0.6}}},98                ],99                "minimum_should_match": 1,100            }101        })102    if freshness in ("day", "week", "month", "year"):103        filters.append({"range": {"crawled_at": {"gte": f"now-1{freshness[0]}/d"}}})104    if images_only:105        # Galerie d'images : seulement les pages avec une image représentative106        filters.append({"exists": {"field": "image_url"}})107108    functions: list[dict[str, Any]] = [109        {"field_value_factor": {"field": "page_quebec_score", "factor": W_PAGE_QUEBEC, "missing": 0}},110        {"field_value_factor": {"field": "domain_quebec_score", "factor": W_DOMAIN_QUEBEC, "missing": 0}},111        {"field_value_factor": {"field": "authority_score", "factor": W_AUTHORITY, "missing": 0}},112        {113            "gauss": {"published_at": {"origin": "now", "scale": "180d", "decay": 0.6}},114            "weight": W_FRESHNESS,115        },116    ]117    # Localité : « plombier Gatineau » booste les documents avec preuve géographique explicite118    if analysis["locations"]:119        functions.append({120            "filter": {"terms": {"locations": analysis["locations"]}},121            "weight": W_LOCALITY,122        })123124    body = {125        "from": max(page - 1, 0) * limit,126        "size": limit,127        "query": {128            "function_score": {129                "query": {"bool": {"must": [text_query], "filter": filters}},130                "functions": functions,131                "score_mode": "sum",132                "boost_mode": "sum",133            }134        },135        "highlight": {136            "pre_tags": ["<em>"],137            "post_tags": ["</em>"],138            "fields": {139                "body": {"fragment_size": 180, "number_of_fragments": 2},140                "description": {"fragment_size": 180, "number_of_fragments": 1},141            },142            "encoder": "html",143        },144        "_source": [145            "url", "canonical_url", "domain", "title", "description", "language",146            "page_quebec_score", "domain_quebec_score", "categories", "published_at",147            "image_url",148        ],149        "track_total_hits": True,150    }151    return body, analysis152