# ----------------------------------------------------------------------------- # Lou-Ka — Location court terme # ingest.py : pipeline d'ingestion court terme — exécute les connecteurs # court terme et synchronise la base séparée (louka_ct.db). # ----------------------------------------------------------------------------- from __future__ import annotations import sys import time import traceback from . import db from .connectors import ST_CONNECTORS BREAKER_FAILS = 5 BREAKER_RETRY_EVERY = 5 def _consecutive_failures(con, source: str) -> int: rows = con.execute( "SELECT ok FROM st_sync_log WHERE source=? ORDER BY ts DESC LIMIT 60", (source,)).fetchall() n = 0 for r in rows: if r["ok"]: break n += 1 return n def run(sources: list[str] | None = None) -> list[dict]: """Exécute l'ingestion court terme (toutes les sources ou celles demandées).""" con = db.connect() results = [] explicit = sources is not None targets = sources or list(ST_CONNECTORS.keys()) for sid in targets: cls = ST_CONNECTORS.get(sid) if cls is None: print(f"[lou-ka/ct] connecteur inconnu : {sid}", file=sys.stderr) continue if not explicit: fails = _consecutive_failures(con, sid) if fails >= BREAKER_FAILS and fails % BREAKER_RETRY_EVERY != 0: print(f"[lou-ka/ct] {sid} sauté (disjoncteur : {fails} échecs)") db.log_failure(con, sid, f"disjoncteur: sauté ({fails} échecs)") results.append({"source": sid, "skipped": True, "fails": fails}) continue t0 = time.time() print(f"[lou-ka/ct] sync {sid} ...") try: listings = cls().fetch() finalized, dropped = [], 0 for lst in listings: try: if not (lst.url and lst.title): dropped += 1 continue finalized.append(lst.finalize()) except Exception: dropped += 1 stats = db.sync_source(con, sid, finalized) stats["seconds"] = round(time.time() - t0, 1) if dropped: stats["dropped"] = dropped print(f"[lou-ka/ct] {stats}") results.append(stats) except Exception as exc: db.log_failure(con, sid, f"{exc}") traceback.print_exc() results.append({"source": sid, "error": str(exc)}) con.close() return results def watch(interval_seconds: int = 6 * 3600) -> None: """Boucle de rafraîchissement périodique (défaut : 6 h — l'inventaire court terme bouge moins vite que les logements au mois).""" while True: run() print(f"[lou-ka/ct] prochaine synchronisation dans {interval_seconds}s") time.sleep(interval_seconds) if __name__ == "__main__": run(sys.argv[1:] or None)