# Trouve-KA — chargeur de seeds # Author: Simon-Pierre Boucher # Contact: contact@spboucher.ai """Charge les seeds dans le frontier avec priorité maximale. Usage : python -m trouveka.crawler.seed [chemin/vers/seeds.txt] """ import asyncio import pathlib import sys from trouveka.config import get_settings from trouveka.database import Database, run_migrations from trouveka.logging import get_logger from trouveka.shared import canonicalize_url, extract_domain log = get_logger("crawler.seed") DEFAULT_SEEDS = pathlib.Path(__file__).resolve().parents[2] / "scripts" / "bootstrap-seeds" / "seeds.txt" async def load_seeds(path: pathlib.Path) -> int: settings = get_settings() applied = await run_migrations(settings.database_url) if applied: log.info("migrations appliquées", extra={"ctx": {"files": applied}}) db = Database(settings.database_url, pool_min=1, pool_max=3) await db.connect() added = 0 try: for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line or line.startswith("#"): continue url = canonicalize_url(line) domain = extract_domain(url) if url else None if not url or not domain: log.info("seed ignorée (URL invalide)", extra={"ctx": {"line": line}}) continue if await db.enqueue_url(url, domain, priority=1.0, depth=0, is_seed=True): added += 1 finally: await db.close() log.info("seeds chargées", extra={"ctx": {"added": added, "file": str(path)}}) return added def main() -> None: path = pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_SEEDS asyncio.run(load_seeds(path)) if __name__ == "__main__": main()