# Trouve-KA — réparation du mojibake dans l'index
# Author: Simon-Pierre Boucher
# Contact: contact@spboucher.ai

"""Répare les documents indexés avec du double encodage UTF-8→cp1252
(« é » → « é ») produit avant le fix de decode_html.

Pour chaque doc contenant des signatures : répare title/description/body/headings
via fix_mojibake (réparation strictement non destructive), réécrit le doc et
efface son embedding (recalculé par le backfill sur le texte propre).
Synchronise aussi title/description dans Postgres (table documents).

Usage : python scripts/fix-mojibake/repair.py
"""

import asyncio

from opensearchpy import AsyncOpenSearch
from opensearchpy.helpers import async_bulk

from trouveka.config import get_settings
from trouveka.database import Database
from trouveka.parser import fix_mojibake

FIELDS = ["title", "description", "body", "headings"]
QUERY = {
    "bool": {
        "should": [
            {"match_phrase": {"body": "Ã"}},
            {"match_phrase": {"title": "Ã"}},
            {"match_phrase": {"description": "Ã"}},
            {"query_string": {"query": 'description:"é" OR body:"é" OR title:"é" OR body:"â€" OR body:"è"'}},
        ],
        "minimum_should_match": 1,
    }
}


async def main() -> None:
    s = get_settings()
    client = AsyncOpenSearch(hosts=[s.search_url], timeout=120)
    db = Database(s.database_url, pool_min=1, pool_max=3)
    await db.connect()
    repaired = scanned = 0
    search_after = None
    try:
        while True:
            body = {
                "size": 200,
                "query": QUERY,
                "_source": FIELDS + ["url"],
                # Curseur stable : on balaie TOUT le stock une fois, même si
                # certains documents matchent sans être réparables.
                "sort": [{"_id": "asc"}],
            }
            if search_after is not None:
                body["search_after"] = search_after
            res = await client.search(index=s.search_index, body=body)
            hits = res["hits"]["hits"]
            if not hits:
                break
            search_after = hits[-1]["sort"]
            actions = []
            for hit in hits:
                scanned += 1
                src = hit["_source"]
                changes = {}
                for f in FIELDS:
                    val = src.get(f)
                    if isinstance(val, str):
                        fixed = fix_mojibake(val)
                        if fixed != val:
                            changes[f] = fixed
                    elif isinstance(val, list):
                        fixed_list = [fix_mojibake(v) if isinstance(v, str) else v for v in val]
                        if fixed_list != val:
                            changes[f] = fixed_list
                if not changes:
                    continue  # signatures détectées mais irréparables : recrawl s'en chargera
                repaired += 1
                actions.append({
                    "_op_type": "update",
                    "_index": s.search_index,
                    "_id": hit["_id"],
                    # embedding effacé : le backfill le recalcule sur le texte propre
                    "script": {"source": "ctx._source.remove('embedding')"},
                })
                actions.append({
                    "_op_type": "update",
                    "_index": s.search_index,
                    "_id": hit["_id"],
                    "doc": changes,
                })
                if "title" in changes or "description" in changes:
                    await db.pool.execute(
                        """
                        UPDATE documents SET
                            title = COALESCE($2, title),
                            description = COALESCE($3, description)
                        WHERE url_id = (SELECT id FROM urls WHERE url = $1 OR canonical_url = $1 LIMIT 1)
                        """,
                        src.get("url"),
                        changes.get("title"),
                        changes.get("description"),
                    )
            if actions:
                await async_bulk(client, actions, raise_on_error=False)
            print(f"→ scannés {scanned}, réparés {repaired}")
    finally:
        print(f"✓ terminé : {repaired} documents réparés sur {scanned} scannés")
        await db.close()
        await client.close()


if __name__ == "__main__":
    asyncio.run(main())