# ----------------------------------------------------------------------------- # Fabri-Ka — Agrégateur de produits québécois # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # ingest.py : orchestration des synchronisations — registre -> connecteurs -> # diff BD. Parallélisé par boutique (thread pool), tolérant aux # pannes (une boutique qui casse n'affecte jamais les autres). # ----------------------------------------------------------------------------- from __future__ import annotations import concurrent.futures as cf import json import sys import threading import time from pathlib import Path from . import db from .connectors import connector_for REGISTRY_PATH = Path(__file__).resolve().parent.parent / "data" / "stores.json" _db_lock = threading.Lock() def load_registry() -> list[dict]: with open(REGISTRY_PATH) as f: return json.load(f)["stores"] def sync_store(store: dict) -> dict: """Synchronise une boutique ; retourne un résumé (jamais d'exception).""" sid = store["id"] conn = connector_for(store) if conn is None: return {"store": sid, "status": "no-connector"} try: products = conn.fetch() except Exception as exc: with _db_lock: con = db.connect() db.log_sync(con, sid, 0, 0, 0, 0, "error", str(exc)) con.commit(); con.close() return {"store": sid, "status": "error", "message": str(exc)[:200]} with _db_lock: con = db.connect() db.upsert_store(con, store) added, updated, removed = db.sync_products(con, sid, store.get("name") or sid, products) db.log_sync(con, sid, len(products), added, updated, removed, "ok") con.commit(); con.close() return {"store": sid, "status": "ok", "found": len(products), "added": added, "updated": updated, "removed": removed} def run(only: list[str] | None = None, workers: int = 8) -> None: stores = load_registry() if only: wanted = {o.lower() for o in only} stores = [s for s in stores if s["id"].lower() in wanted] # le répertoire /boutiques affiche TOUTES les boutiques du registre : # on upsert chaque fiche, même non connectable, avant la sync produits con = db.connect() for s in stores: db.upsert_store(con, s) con.commit(); con.close() stores = [s for s in stores if s.get("enabled", True) and s.get("platform") and s.get("catalog_endpoint")] print(f"[fabri-ka] sync de {len(stores)} boutiques…", flush=True) t0 = time.time() ok = err = total = 0 with cf.ThreadPoolExecutor(workers) as ex: for res in ex.map(sync_store, stores): status = res.get("status") if status == "ok": ok += 1 total += res.get("found", 0) print(f" ✓ {res['store']}: {res['found']} produits " f"(+{res['added']} ~{res['updated']} -{res['removed']})", flush=True) elif status == "error": err += 1 print(f" ✗ {res['store']}: {res.get('message','')}", flush=True) print(f"[fabri-ka] terminé en {time.time()-t0:.0f}s — {ok} ok, {err} erreurs, " f"{total} produits vus", flush=True) def watch(interval_seconds: int) -> None: while True: try: run() except Exception as exc: # noqa: BLE001 — le watcher ne meurt jamais print(f"[fabri-ka] watch: {exc}", file=sys.stderr, flush=True) time.sleep(interval_seconds)