# ----------------------------------------------------------------------------- # Fabri-Ka — Agrégateur de produits québécois # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/shopify.py : connecteur générique Shopify — catalogue complet # via l'endpoint public /products.json (paginé, 250 produits/page). # ----------------------------------------------------------------------------- from __future__ import annotations import threading import time from ..schema import Product, parse_price from .base import BaseConnector # Shopify applique une limite de débit PAR IP CLIENTE partagée entre toutes les # boutiques derrière son CDN : on sérialise donc les requêtes Shopify # globalement (toutes boutiques confondues) + retry sur 429. _GLOBAL_LOCK = threading.Lock() _MIN_INTERVAL = 0.7 _last_shopify_req = [0.0] class ShopifyConnector(BaseConnector): platform = "shopify" def _get_json(self, url: str) -> dict: """GET JSON via curl (l'empreinte TLS de python-requests déclenche le 429 de Shopify sous volume ; curl passe). Throttle global + retry.""" import json as _json import subprocess last_err = None for attempt in range(4): with _GLOBAL_LOCK: wait = _MIN_INTERVAL - (time.time() - _last_shopify_req[0]) if wait > 0: time.sleep(wait) _last_shopify_req[0] = time.time() p = subprocess.run( ["curl", "-sS", "--compressed", "--max-time", str(self.timeout), "-A", self.session.headers["User-Agent"].split(" FabriKaBot")[0], "-w", "\n%{http_code}", url], capture_output=True, text=True, errors="replace") body, _, code = p.stdout.rpartition("\n") if code == "200": try: return _json.loads(body) except _json.JSONDecodeError as exc: last_err = exc elif code == "429": time.sleep(5 * (attempt + 1)) last_err = RuntimeError("429 Too Many Requests") else: last_err = RuntimeError(f"HTTP {code or 'error'} {p.stderr[:120]}") time.sleep(2) # dernier recours : Scrapfly (anti-bot) si configuré from . import scrapfly if scrapfly.available(): try: return scrapfly.scrapfly_json(url) except Exception as exc: last_err = exc raise RuntimeError(f"{url}: {last_err}") def fetch(self) -> list[Product]: out: list[Product] = [] page = 1 while page <= self.max_pages: data = self._get_json(f"{self.base}/products.json?limit=250&page={page}") items = data.get("products", []) if not items: break for it in items: variants = it.get("variants") or [] prices = [parse_price(v.get("price")) for v in variants] prices = [p for p in prices if p] compare = [parse_price(v.get("compare_at_price")) for v in variants] compare = [c for c in compare if c] available = any(v.get("available", True) for v in variants) if variants else None out.append(Product( store_id=self.store_id, external_id=str(it["id"]), url=f"{self.base}/products/{it.get('handle','')}", title=it.get("title", ""), description=it.get("body_html", "") or "", price=min(prices) if prices else None, price_max=max(prices) if prices else None, compare_at_price=max(compare) if compare else None, images=[im.get("src", "") for im in (it.get("images") or [])], product_type=it.get("product_type", "") or "", tags=(it.get("tags") if isinstance(it.get("tags"), list) else [t.strip() for t in (it.get("tags") or "").split(",") if t.strip()]), vendor=it.get("vendor", "") or "", available=available, )) if len(items) < 250: break page += 1 return out