spb/trouve-ka Public

1# Trouve-KA — construction du document d'index2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45"""Transforme une page parsée + signaux Québec en document OpenSearch (étape 1)."""67from datetime import UTC, datetime8from typing import Any910from trouveka.types import ParsedPage, QuebecSignals1112# Catégorisation grossière par domaine — raffinée en asynchrone (étape 2)13_GOV_SUFFIXES = (".gouv.qc.ca", ".gc.ca", ".quebec.ca")14_GOV_DOMAINS = {"quebec.ca", "canada.ca", "montreal.ca", "laval.ca", "gatineau.ca", "sherbrooke.ca"}15_NEWS_DOMAINS = {16    "lapresse.ca", "ledevoir.com", "journaldemontreal.com", "journaldequebec.com",17    "radio-canada.ca", "tvanouvelles.ca", "lesoleil.com", "ledroit.com",18    "latribune.ca", "lenouvelliste.ca", "lequotidien.com", "lavoixdelest.ca",19    "noovo.info", "24heures.ca", "montrealgazette.com",20}21_EDU_SUFFIXES = (".ulaval.ca", ".umontreal.ca", ".mcgill.ca", ".uqam.ca", ".usherbrooke.ca",22                 ".concordia.ca", ".polymtl.ca", ".etsmtl.ca", ".hec.ca")232425def categorize_domain(domain: str) -> list[str]:26    d = domain.lower()27    cats: list[str] = []28    if d in _GOV_DOMAINS or any(d.endswith(s) for s in _GOV_SUFFIXES) or ".gouv." in d:29        cats.append("government")30    if d in _NEWS_DOMAINS:31        cats.append("news")32    if any(d.endswith(s) or d == s.lstrip(".") for s in _EDU_SUFFIXES) or d.endswith(".edu"):33        cats.append("education")34    return cats353637def build_search_document(38    page: ParsedPage,39    signals: QuebecSignals,40    *,41    domain: str,42    domain_quebec_score: float,43    authority_score: float = 0.0,44) -> dict[str, Any]:45    return {46        "url": page.url,47        "canonical_url": page.canonical_url or page.url,48        "domain": domain,49        "title": page.title,50        "description": page.description,51        "body": page.body[:100_000],52        "headings": page.headings,53        "language": page.language,54        "page_quebec_score": signals.score,55        "domain_quebec_score": round(domain_quebec_score, 4),56        "locations": signals.locations,57        "organizations": [],   # enrichissement étape 258        "people": [],          # enrichissement étape 259        "categories": categorize_domain(domain),60        "image_url": page.image_url,61        "published_at": page.published_at.isoformat() if page.published_at else None,62        "crawled_at": datetime.now(UTC).isoformat(),63        "authority_score": round(authority_score, 4),64        "freshness_score": 0.0,  # enrichissement étape 365        "quality_score": 0.0,    # enrichissement étape 366        "spam_score": 0.0,       # enrichissement étape 367    }68