spb/trouve-ka Public

1#!/usr/bin/env python32# Trouve-KA — purge des documents mal décodés de l'index3# Author: Simon-Pierre Boucher4# Contact: contact@spboucher.ai56"""Scanne l'index, supprime les documents au contenu charabia (binaire mal décodé)7et remet leurs URLs en recrawl (le fetcher corrigé les réindexera proprement).89Usage : python -m scripts... non — exécuter depuis la racine :10  .venv/bin/python scripts/cleanup-garbage.py            # dev local11  (sur m2m32 : docker compose run --rm --entrypoint python api /app/scripts/cleanup-garbage.py)12"""1314import asyncio1516from trouveka.config import get_settings17from trouveka.database import Database18from trouveka.parser import looks_like_garbage19from trouveka.search_core import SearchCore202122async def main() -> None:23    s = get_settings()24    db = Database(s.database_url, pool_min=1, pool_max=3)25    await db.connect()26    search = SearchCore(s.search_url, s.search_index)2728    removed = 029    scanned = 030    # Scroll de tout l'index par pages de 20031    body = {"query": {"match_all": {}}, "size": 200, "_source": ["url", "title", "description", "body"]}32    resp = await search.client.search(index=s.search_index, body=body, scroll="2m")33    scroll_id = resp.get("_scroll_id")34    try:35        while True:36            hits = resp["hits"]["hits"]37            if not hits:38                break39            for hit in hits:40                scanned += 141                src = hit["_source"]42                text = " ".join([src.get("title") or "", src.get("description") or "",43                                 (src.get("body") or "")[:20_000]])44                if looks_like_garbage(text, threshold=0.02):45                    await search.client.delete(index=s.search_index, id=hit["_id"], ignore=[404])46                    await db.requeue_url(src["url"])47                    await db.pool.execute(48                        "DELETE FROM documents WHERE url_id = (SELECT id FROM urls WHERE url = $1)",49                        src["url"],50                    )51                    removed += 152                    print(f"purgé : {src['url']}")53            resp = await search.client.scroll(scroll_id=scroll_id, scroll="2m")54    finally:55        if scroll_id:56            await search.client.clear_scroll(scroll_id=scroll_id)57        await search.close()58        await db.close()59    print(f"\n{scanned} documents scannés, {removed} purgés et remis en recrawl.")606162if __name__ == "__main__":63    asyncio.run(main())64