#!/usr/bin/env python3
# Trouve-KA — purge des documents mal décodés de l'index
# Author: Simon-Pierre Boucher
# Contact: contact@spboucher.ai

"""Scanne l'index, supprime les documents au contenu charabia (binaire mal décodé)
et remet leurs URLs en recrawl (le fetcher corrigé les réindexera proprement).

Usage : python -m scripts... non — exécuter depuis la racine :
  .venv/bin/python scripts/cleanup-garbage.py            # dev local
  (sur m2m32 : docker compose run --rm --entrypoint python api /app/scripts/cleanup-garbage.py)
"""

import asyncio

from trouveka.config import get_settings
from trouveka.database import Database
from trouveka.parser import looks_like_garbage
from trouveka.search_core import SearchCore


async def main() -> None:
    s = get_settings()
    db = Database(s.database_url, pool_min=1, pool_max=3)
    await db.connect()
    search = SearchCore(s.search_url, s.search_index)

    removed = 0
    scanned = 0
    # Scroll de tout l'index par pages de 200
    body = {"query": {"match_all": {}}, "size": 200, "_source": ["url", "title", "description", "body"]}
    resp = await search.client.search(index=s.search_index, body=body, scroll="2m")
    scroll_id = resp.get("_scroll_id")
    try:
        while True:
            hits = resp["hits"]["hits"]
            if not hits:
                break
            for hit in hits:
                scanned += 1
                src = hit["_source"]
                text = " ".join([src.get("title") or "", src.get("description") or "",
                                 (src.get("body") or "")[:20_000]])
                if looks_like_garbage(text, threshold=0.02):
                    await search.client.delete(index=s.search_index, id=hit["_id"], ignore=[404])
                    await db.requeue_url(src["url"])
                    await db.pool.execute(
                        "DELETE FROM documents WHERE url_id = (SELECT id FROM urls WHERE url = $1)",
                        src["url"],
                    )
                    removed += 1
                    print(f"purgé : {src['url']}")
            resp = await search.client.scroll(scroll_id=scroll_id, scroll="2m")
    finally:
        if scroll_id:
            await search.client.clear_scroll(scroll_id=scroll_id)
        await search.close()
        await db.close()
    print(f"\n{scanned} documents scannés, {removed} purgés et remis en recrawl.")


if __name__ == "__main__":
    asyncio.run(main())