spb/fabri-ka

1#!/usr/bin/env python32"""Vague 2 — re-sondage des boutiques à 0 produit (enabled=0 ou plateforme vide).34Beaucoup de boutiques ont migré de plateforme depuis la découverte initiale5(ex. WordPress vitrine -> Shopify, Wix -> WooCommerce), et les boutiques Square6Online sont maintenant connectables (connecteur square.py, vague 2). Pour7chaque cible :89  1. sondes légères d'endpoints catalogue :10       Shopify        /products.json?limit=1              (curl, throttle 0,7 s global)11       WooCommerce    /wp-json/wc/store/v1/products12       Wix Stores     /_api/v1/access-tokens (app Stores)13       Squarespace    /shop|/boutique|/store ?format=json14       Square Online  user_id/site_id du HTML + /app/store/api/v13 (total>0)15  2. sinon, détection de plateforme via le HTML d'accueil (signatures/generator)16     — plateforme notée mais boutique laissée désactivée (pas d'endpoint).1718Met à jour : data/stores.json (platform/catalog_endpoint/enabled/ecommerce),19data/verify_cache/<dom>.json, data/enriched/verified.jsonl (patch des domaines20touchés) et la table stores (upsert). Additif : ne touche jamais aux boutiques21déjà actives.2223Usage : python3 scripts/reprobe_stores.py [--cap 400] [--dry-run]24"""25import argparse26import concurrent.futures as cf27import json28import os29import subprocess30import sys31import threading32import time3334import requests3536ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))37sys.path.insert(0, ROOT)38sys.path.insert(0, os.path.join(ROOT, "scripts"))39CACHE = os.path.join(ROOT, "data", "verify_cache")40os.makedirs(CACHE, exist_ok=True)4142for line in open(os.path.join(ROOT, ".env")).read().splitlines():43    if "=" in line and not line.startswith("#"):44        k, _, v = line.partition("=")45        os.environ.setdefault(k.strip(), v.strip())4647from verify import detect_platform, HDRS  # noqa: E40248from fabrika import db as fdb  # noqa: E40249from fabrika.connectors.square import extract_ids, API_PATH as SQUARE_API  # noqa: E4025051WIX_STORES_APP = "1380b703-ce81-ff05-f115-39571d94dfcd"5253# throttle global Shopify (même règle que le connecteur : 0,7 s entre requêtes)54_SHOPIFY_LOCK = threading.Lock()55_last_shopify = [0.0]5657STEP_DELAY = 0.5   # politesse entre sondes sur un même domaine585960def curl_get(url, timeout=15):61    p = subprocess.run(["curl", "-sS", "-L", "--compressed", "--max-time", str(timeout),62                        "-A", HDRS["User-Agent"].split(" FabriKaBot")[0],63                        "-w", "\n%{http_code}", url],64                       capture_output=True, text=True, errors="replace")65    body, _, code = p.stdout.rpartition("\n")66    return (int(code) if code.isdigit() else 0), body676869def probe_shopify(base):70    with _SHOPIFY_LOCK:71        wait = 0.7 - (time.time() - _last_shopify[0])72        if wait > 0:73            time.sleep(wait)74        _last_shopify[0] = time.time()75    code, body = curl_get(f"{base}/products.json?limit=1")76    if code == 200 and body.lstrip().startswith("{") and '"products"' in body[:200]:77        return "shopify", "/products.json"78    return None798081def probe_woo(base):82    try:83        r = requests.get(f"{base}/wp-json/wc/store/v1/products?per_page=1",84                         headers=HDRS, timeout=12)85        if r.status_code == 200 and r.text.strip().startswith("["):86            return "woocommerce", "/wp-json/wc/store/v1/products"87    except Exception:88        pass89    return None909192def probe_wix(base):93    try:94        r = requests.get(f"{base}/_api/v1/access-tokens", headers=HDRS,95                         timeout=12, allow_redirects=True)96        if r.status_code == 200 and WIX_STORES_APP in (r.text or ""):97            return "wix", "/_api/wix-ecommerce-storefront-web/api"98    except Exception:99        pass100    return None101102103def probe_squarespace(base):104    for path in ("/shop", "/boutique", "/store"):105        try:106            r = requests.get(f"{base}{path}?format=json-pretty", headers=HDRS, timeout=10)107            if r.status_code == 200 and '"items"' in r.text[:5000]:108                return "squarespace", path + "?format=json"109        except Exception:110            pass111    return None112113114def probe_square(base, html):115    """html d'accueil déjà téléchargé (peut être vide -> re-fetch)."""116    if not html:117        try:118            r = requests.get(base, headers=HDRS, timeout=15, allow_redirects=True)119            html = r.text if r.status_code == 200 else ""120        except Exception:121            return None122    user, site = extract_ids(html or "")123    if not (user and site):124        return None125    try:126        r = requests.get(f"{base}{SQUARE_API}/editor/users/{user}/sites/{site}"127                         f"/products?page=1&per_page=1", headers=HDRS, timeout=15)128        if r.status_code == 200:129            data = r.json()130            total = ((data.get("meta") or {}).get("pagination") or {}).get("total", 0)131            if total and int(total) > 0:132                return "square", SQUARE_API133    except Exception:134        pass135    return None136137138def probe_store(store):139    """Retourne (store_id, platform, endpoint, detected_platform) — endpoint None si rien."""140    sid = store["id"]141    base = (store.get("url") or f"https://{sid}").rstrip("/")142    homepage = ""143    try:144        r = requests.get(base, headers=HDRS, timeout=15, allow_redirects=True)145        if r.status_code == 200:146            homepage = r.text147    except Exception:148        pass149150    hint = (store.get("platform") or "").lower()151    detected = detect_platform(homepage[:400000], {}) if homepage else None152153    # ordre des sondes : plateforme connue/détectée d'abord, puis le reste154    order = ["shopify", "woocommerce", "wix", "squarespace", "square"]155    for pref in (hint, detected):156        if pref in order:157            order.remove(pref)158            order.insert(0, pref)159160    probes = {"shopify": lambda: probe_shopify(base),161              "woocommerce": lambda: probe_woo(base),162              "wix": lambda: probe_wix(base),163              "squarespace": lambda: probe_squarespace(base),164              "square": lambda: probe_square(base, homepage)}165    for i, name in enumerate(order):166        if i:167            time.sleep(STEP_DELAY)168        res = probes[name]()169        if res:170            return sid, res[0], res[1], detected171    return sid, None, None, detected172173174def main():175    ap = argparse.ArgumentParser()176    ap.add_argument("--cap", type=int, default=400)177    ap.add_argument("--skip", type=int, default=0,178                    help="saute les N premières cibles (déjà sondées par une vague précédente)")179    ap.add_argument("--skip-stamped", action="store_true",180                    help="ignore les boutiques portant déjà un last_reprobe")181    ap.add_argument("--only-file", default=None,182                    help="restreint les cibles aux ids listés dans ce fichier (un par ligne)")183    ap.add_argument("--workers", type=int, default=8)184    ap.add_argument("--dry-run", action="store_true")185    args = ap.parse_args()186187    reg_path = os.path.join(ROOT, "data", "stores.json")188    reg = json.load(open(reg_path))189    by_id = {s["id"]: s for s in reg["stores"]}190    targets = [s for s in reg["stores"]191               if not s.get("enabled", True) or not s.get("platform")]192    # faux positifs de découverte avérés (origin_confidence 0, ex. entreprises de193    # services sans boutique) : ne jamais les réactiver par simple sonde d'endpoint194    targets = [s for s in targets if (s.get("origin_confidence") or 0) > 0]195    if args.skip_stamped:196        targets = [s for s in targets if not s.get("last_reprobe")]197    if args.only_file:198        only = {line.strip() for line in open(args.only_file) if line.strip()}199        targets = [s for s in targets if s["id"] in only]200201    # priorité : Square (nouveau connecteur), puis origin_class A, B, …202    def key(s):203        plat_rank = 0 if (s.get("platform") or "") == "square" else 1204        return (plat_rank, s.get("origin_class") or "E", s["id"])205    targets.sort(key=key)206    targets = targets[args.skip:args.skip + args.cap]207    print(f"[reprobe] {len(targets)} boutiques ciblées (skip {args.skip}, cap {args.cap})", flush=True)208209    reactivated, platform_notes = [], []210    with cf.ThreadPoolExecutor(args.workers) as ex:211        for sid, plat, ep, detected in ex.map(probe_store, targets):212            if ep:213                reactivated.append((sid, plat, ep))214                print(f"  + {sid} -> {plat} {ep}", flush=True)215            elif detected and detected != (by_id[sid].get("platform") or ""):216                platform_notes.append((sid, detected))217218    if args.dry_run:219        print(f"[dry-run] réactivables: {len(reactivated)}, plateformes corrigées: {len(platform_notes)}")220        return221222    # 1) stores.json223    stamp = time.strftime("%Y-%m-%d")224    for s in targets:                      # trace de sondage (pour les vagues suivantes)225        by_id[s["id"]]["last_reprobe"] = stamp226    touched = set()227    for sid, plat, ep in reactivated:228        s = by_id[sid]229        s["platform"], s["catalog_endpoint"] = plat, ep230        s["enabled"], s["ecommerce"] = True, True231        touched.add(sid)232    for sid, detected in platform_notes:233        by_id[sid]["platform"] = detected      # info seulement, reste désactivée234        touched.add(sid)235    json.dump(reg, open(reg_path, "w"), ensure_ascii=False, indent=1)236237    # 2) verify_cache (durabilité pour les rebuilds du registre)238    for sid in touched:239        s = by_id[sid]240        cpath = os.path.join(CACHE, sid + ".json")241        try:242            rec = json.load(open(cpath))243        except Exception:244            rec = {"domain": sid, "active": True, "final_domain": sid}245        rec["platform"] = s.get("platform") or rec.get("platform")246        rec["catalog_endpoint"] = s.get("catalog_endpoint") or rec.get("catalog_endpoint")247        rec["active"] = True248        json.dump(rec, open(cpath, "w"))249250    # 3) verified.jsonl — patch des domaines touchés251    vpath = os.path.join(ROOT, "data", "enriched", "verified.jsonl")252    if os.path.exists(vpath):253        lines = []254        for line in open(vpath):255            try:256                rec = json.loads(line)257            except Exception:258                lines.append(line.rstrip("\n"))259                continue260            if rec.get("domain") in touched or rec.get("final_domain") in touched:261                sid = rec.get("final_domain") or rec["domain"]262                if sid in by_id:263                    rec["platform"] = by_id[sid].get("platform") or rec.get("platform")264                    rec["catalog_endpoint"] = (by_id[sid].get("catalog_endpoint")265                                               or rec.get("catalog_endpoint"))266                    rec["active"] = True267            lines.append(json.dumps(rec, ensure_ascii=False))268        with open(vpath, "w") as f:269            f.write("\n".join(lines) + "\n")270271    # 4) base SQLite272    con = fdb.connect()273    for sid in touched:274        fdb.upsert_store(con, by_id[sid])275    con.commit(); con.close()276277    per_plat = {}278    for _, plat, _ in reactivated:279        per_plat[plat] = per_plat.get(plat, 0) + 1280    print(f"[reprobe] réactivées: {len(reactivated)} {json.dumps(per_plat, ensure_ascii=False)}"281          f" | plateformes corrigées (sans endpoint): {len(platform_notes)}")282    if reactivated:283        print("[reprobe] à synchroniser : python run.py sync "284              + " ".join(sid for sid, _, _ in reactivated[:50]) + (" …" if len(reactivated) > 50 else ""))285286287if __name__ == "__main__":288    main()289