spb/trouve-ka Public

1# Trouve-KA — chargeur de seeds2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai45"""Charge les seeds dans le frontier avec priorité maximale.67Usage : python -m trouveka.crawler.seed [chemin/vers/seeds.txt]8"""910import asyncio11import pathlib12import sys1314from trouveka.config import get_settings15from trouveka.database import Database, run_migrations16from trouveka.logging import get_logger17from trouveka.shared import canonicalize_url, extract_domain1819log = get_logger("crawler.seed")2021DEFAULT_SEEDS = pathlib.Path(__file__).resolve().parents[2] / "scripts" / "bootstrap-seeds" / "seeds.txt"222324async def load_seeds(path: pathlib.Path) -> int:25    settings = get_settings()26    applied = await run_migrations(settings.database_url)27    if applied:28        log.info("migrations appliquées", extra={"ctx": {"files": applied}})2930    db = Database(settings.database_url, pool_min=1, pool_max=3)31    await db.connect()32    added = 033    try:34        for line in path.read_text(encoding="utf-8").splitlines():35            line = line.strip()36            if not line or line.startswith("#"):37                continue38            url = canonicalize_url(line)39            domain = extract_domain(url) if url else None40            if not url or not domain:41                log.info("seed ignorée (URL invalide)", extra={"ctx": {"line": line}})42                continue43            if await db.enqueue_url(url, domain, priority=1.0, depth=0, is_seed=True):44                added += 145    finally:46        await db.close()47    log.info("seeds chargées", extra={"ctx": {"added": added, "file": str(path)}})48    return added495051def main() -> None:52    path = pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_SEEDS53    asyncio.run(load_seeds(path))545556if __name__ == "__main__":57    main()58