spb/fabri-ka

1#!/usr/bin/env python32"""Phase 2 — taxonomie métier réelle des boutiques Shopify via /collections.json.341 requête par boutique Shopify productive (curl, throttle global 0,7 s) :5la liste des collections (titres + handles + published) est la taxonomie6métier déclarée par le marchand. On la mappe vers la taxonomie Fabri-Ka7(infer_category sur les titres de collections) pour :89  - assigner des catégories boutique (registre + DB) quand il n'y en a pas ;10  - alimenter l'héritage de catégorie d'enrich_products.py (produits « autre ») ;11  - consigner la table de correspondance collection -> catégorie12    (data/collections_taxonomy.json, rejouable et auditable).1314WooCommerce : PAS de requête réseau — la Store API renvoie déjà les15catégories par produit (champ product_type), l'endpoint16/wc/store/v1/products/categories serait redondant (décision consignée).1718Cache : data/enrich_cache/collections/<dom>.json.19Usage : .venv/bin/python scripts/enrich_collections.py [--cap 700]20"""21import argparse22import concurrent.futures as cf23import json24import os25import subprocess26import sys27import threading28import time29from collections import Counter3031ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))32sys.path.insert(0, ROOT)33CACHE = os.path.join(ROOT, "data", "enrich_cache", "collections")34os.makedirs(CACHE, exist_ok=True)3536from fabrika import db as fdb  # noqa: E40237from fabrika.schema import infer_category  # noqa: E4023839UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "40      "(KHTML, like Gecko) Chrome/126 Safari/537.36")4142_LOCK = threading.Lock()43_last = [0.0]4445# titres de collections sans signal métier46NOISE = {"accueil", "home", "all", "tous", "tout", "frontpage", "nouveautés",47         "nouveautes", "new", "soldes", "sale", "promotions", "promo",48         "cadeaux", "gifts", "idées cadeaux", "best sellers", "meilleurs vendeurs"}495051def fetch_collections(dom: str, base: str) -> dict:52    cpath = os.path.join(CACHE, dom + ".json")53    if os.path.exists(cpath):54        return json.load(open(cpath))55    with _LOCK:56        wait = 0.7 - (time.time() - _last[0])57        if wait > 0:58            time.sleep(wait)59        _last[0] = time.time()60    rec = {"domain": dom, "collections": [], "checked_at": time.strftime("%Y-%m-%d")}61    for attempt in range(4):                      # retry 429 avec backoff long62        p = subprocess.run(["curl", "-sS", "--compressed", "-L", "--max-time", "25",63                            "-A", UA, "-w", "\n%{http_code}",64                            f"{base}/collections.json?limit=250"],65                           capture_output=True, text=True, errors="replace")66        body, _, code = p.stdout.rpartition("\n")67        if code == "200":68            try:69                cols = json.loads(body).get("collections", [])70                rec["collections"] = [{"handle": c.get("handle", ""),71                                       "title": c.get("title", ""),72                                       "products_count": c.get("products_count")}73                                      for c in cols]74                rec.pop("error", None)75            except Exception as exc:76                rec["error"] = str(exc)[:120]77            break78        rec["error"] = f"HTTP {code}"79        if code != "429":80            break81        time.sleep(10 * (attempt + 1))82    json.dump(rec, open(cpath, "w"), ensure_ascii=False)83    return rec848586def main():87    ap = argparse.ArgumentParser()88    ap.add_argument("--cap", type=int, default=700)89    ap.add_argument("--retry-errors", action="store_true",90                    help="purge du cache les fiches en erreur (ex. 429) avant la passe")91    args = ap.parse_args()9293    if args.retry_errors:94        import glob95        n = 096        for f in glob.glob(os.path.join(CACHE, "*.json")):97            try:98                if json.load(open(f)).get("error"):99                    os.remove(f); n += 1100            except Exception:101                pass102        print(f"[collections] {n} caches en erreur purgés")103104    con = fdb.connect()105    rows = [dict(r) for r in con.execute(106        "SELECT id, url FROM stores WHERE platform='shopify' AND product_count>0 "107        "ORDER BY product_count DESC LIMIT ?", (args.cap,))]108    con.close()109    print(f"[collections] {len(rows)} boutiques Shopify ciblées", flush=True)110111    results, done = [], 0112    with cf.ThreadPoolExecutor(4) as ex:113        for rec in ex.map(lambda s: fetch_collections(s["id"], (s["url"] or "").rstrip("/")), rows):114            results.append(rec)115            done += 1116            if done % 100 == 0:117                print(f"  {done}/{len(rows)}", flush=True)118119    # table de correspondance collection -> catégorie Fabri-Ka (consignée)120    mapping = {}121    store_cats = {}122    n_cols = 0123    for rec in results:124        cats = Counter()125        for c in rec.get("collections", []):126            title = (c.get("title") or "").strip()127            if not title or title.lower() in NOISE:128                continue129            n_cols += 1130            cat = infer_category(title)131            mapping[title] = cat132            if cat != "autre":133                cats[cat] += max(1, int(c.get("products_count") or 1))134        if cats:135            store_cats[rec["domain"]] = [k for k, _ in cats.most_common(3)]136137    json.dump({"generated": time.strftime("%Y-%m-%d"),138               "note": "titres de collections Shopify -> taxonomie Fabri-Ka (infer_category)",139               "mapping": dict(sorted(mapping.items()))},140              open(os.path.join(ROOT, "data", "collections_taxonomy.json"), "w"),141              ensure_ascii=False, indent=1)142    print(f"[collections] {n_cols} collections vues, {len(mapping)} mappées, "143          f"{len(store_cats)} boutiques avec catégories déduites")144145    # registre + DB : catégories boutique si absentes146    reg_path = os.path.join(ROOT, "data", "stores.json")147    reg = json.load(open(reg_path))148    n_reg = 0149    con = fdb.connect()150    for s in reg["stores"]:151        cats = store_cats.get(s["id"])152        if cats and not s.get("categories"):153            s["categories"] = cats154            con.execute("UPDATE stores SET categories=? WHERE id=?",155                        (json.dumps(cats, ensure_ascii=False), s["id"]))156            n_reg += 1157    con.commit(); con.close()158    json.dump(reg, open(reg_path, "w"), ensure_ascii=False, indent=1)159    print(f"[collections] catégories boutique assignées (registre+DB) : {n_reg}")160161162if __name__ == "__main__":163    main()164