spb/trouve-ka Public

1# Trouve-KA — pipeline de requête et ranking2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45"""Pipeline de requête : normalisation → langue → lieux → requête OpenSearch.67BM25 (multi_match bilingue) au cœur, function_score par-dessus :8Québec, autorité, fraîcheur, localité. Poids nommés, non figés (§9).9"""1011import re12import unicodedata13from typing import Any1415from trouveka.classifier import QUEBEC_TOPONYMS1617# Poids de ranking — à calibrer avec le dataset d'évaluation (§16)18W_PAGE_QUEBEC = 1.619W_DOMAIN_QUEBEC = 1.020W_AUTHORITY = 0.821W_FRESHNESS = 0.522W_LOCALITY = 2.02324_FR_HINTS = {"le", "la", "les", "des", "une", "un", "du", "de", "et", "ou", "pour",25             "avec", "dans", "sur", "meilleur", "meilleure", "comment", "où", "quel", "quelle"}26_EN_HINTS = {"the", "a", "an", "of", "and", "or", "for", "with", "in", "on", "best", "how", "what", "where"}272829def _fold(text: str) -> str:30    return "".join(c for c in unicodedata.normalize("NFD", text) if unicodedata.category(c) != "Mn")313233def analyze_query(q: str) -> dict[str, Any]:34    """Normalisation, détection de langue heuristique, extraction de lieux québécois."""35    normalized = re.sub(r"\s+", " ", q).strip()[:200]36    tokens = [t.lower() for t in re.findall(r"[\w'-]+", normalized, re.UNICODE)]3738    fr_hits = sum(1 for t in tokens if t in _FR_HINTS)39    en_hits = sum(1 for t in tokens if t in _EN_HINTS)40    has_accents = normalized != _fold(normalized)41    language = "fr" if (fr_hits > en_hits or has_accents) else ("en" if en_hits > fr_hits else None)4243    # Lieux : tokens simples + bigrammes contre le gazetteer44    folded_gazetteer = {_fold(t): t for t in QUEBEC_TOPONYMS}45    locations: list[str] = []46    candidates = tokens + [f"{a}-{b}" for a, b in zip(tokens, tokens[1:])] + [47        f"{a} {b}" for a, b in zip(tokens, tokens[1:])48    ]49    for cand in candidates:50        folded = _fold(cand)51        if folded in folded_gazetteer:52            locations.append(folded_gazetteer[folded])53    return {"query": normalized, "language": language, "locations": sorted(set(locations))}545556def build_search_body(57    q: str,58    *,59    page: int = 1,60    limit: int = 10,61    language: str | None = None,62    category: str | None = None,63    quebec_only: bool = False,64    freshness: str | None = None,65) -> tuple[dict[str, Any], dict[str, Any]]:66    """Construit le corps de requête OpenSearch. Retourne (body, analyse)."""67    analysis = analyze_query(q)68    normalized = analysis["query"]6970    # BM25 bilingue : les deux analyzers interrogés, le meilleur champ gagne71    text_query: dict[str, Any] = {72        "multi_match": {73            "query": normalized,74            "type": "most_fields",75            "fields": [76                "title^4", "title.en^4",77                "headings^2", "headings.en^2",78                "description^2", "description.en^2",79                "body", "body.en",80            ],81            "fuzziness": "AUTO",82            "prefix_length": 2,83        }84    }8586    filters: list[dict[str, Any]] = []87    if language in ("fr", "en"):88        filters.append({"term": {"language": language}})89    if category:90        filters.append({"term": {"categories": category}})91    if quebec_only:92        filters.append({93            "bool": {94                "should": [95                    {"range": {"page_quebec_score": {"gte": 0.45}}},96                    {"range": {"domain_quebec_score": {"gte": 0.6}}},97                ],98                "minimum_should_match": 1,99            }100        })101    if freshness in ("day", "week", "month", "year"):102        filters.append({"range": {"crawled_at": {"gte": f"now-1{freshness[0]}/d"}}})103104    functions: list[dict[str, Any]] = [105        {"field_value_factor": {"field": "page_quebec_score", "factor": W_PAGE_QUEBEC, "missing": 0}},106        {"field_value_factor": {"field": "domain_quebec_score", "factor": W_DOMAIN_QUEBEC, "missing": 0}},107        {"field_value_factor": {"field": "authority_score", "factor": W_AUTHORITY, "missing": 0}},108        {109            "gauss": {"published_at": {"origin": "now", "scale": "180d", "decay": 0.6}},110            "weight": W_FRESHNESS,111        },112    ]113    # Localité : « plombier Gatineau » booste les documents avec preuve géographique explicite114    if analysis["locations"]:115        functions.append({116            "filter": {"terms": {"locations": analysis["locations"]}},117            "weight": W_LOCALITY,118        })119120    body = {121        "from": max(page - 1, 0) * limit,122        "size": limit,123        "query": {124            "function_score": {125                "query": {"bool": {"must": [text_query], "filter": filters}},126                "functions": functions,127                "score_mode": "sum",128                "boost_mode": "sum",129            }130        },131        "highlight": {132            "pre_tags": ["<em>"],133            "post_tags": ["</em>"],134            "fields": {135                "body": {"fragment_size": 180, "number_of_fragments": 2},136                "description": {"fragment_size": 180, "number_of_fragments": 1},137            },138            "encoder": "html",139        },140        "_source": [141            "url", "canonical_url", "domain", "title", "description", "language",142            "page_quebec_score", "domain_quebec_score", "categories", "published_at",143        ],144        "track_total_hits": True,145    }146    return body, analysis147