spb/fabri-ka Public

1#!/usr/bin/env python32"""Re-sondage agressif pour augmenter le nombre de boutiques connectables.34Phase A — boutiques actives SANS endpoint catalogue : re-tester directement les54 endpoints (Shopify products.json via curl, WooCommerce Store API, Wix6access-tokens + app Stores, Squarespace ?format=json), même si la plateforme7détectée est « unknown/wordpress » (thèmes headless, signatures manquées).89Phase B — candidats INACTIFS (sites « morts ») : nouvelle tentative directe,10puis via Scrapfly (asp anti-bot) — beaucoup de « morts » sont en fait des murs11anti-bot qui bloquent python-requests.1213Met à jour data/verify_cache/<domain>.json ; relancer ensuite14build_registry.py puis sync.15"""16import concurrent.futures as cf17import json18import os19import re20import subprocess21import sys22import time2324import requests2526ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))27sys.path.insert(0, os.path.join(ROOT, "scripts"))28sys.path.insert(0, ROOT)29CACHE = os.path.join(ROOT, "data", "verify_cache")3031# .env32for line in open(os.path.join(ROOT, ".env")).read().splitlines():33    if "=" in line and not line.startswith("#"):34        k, _, v = line.partition("=")35        os.environ.setdefault(k.strip(), v.strip())3637from verify import detect_platform, MADE_RE, POSTAL_RE, AREA_RE, QC_WORD_RE, CART_RE, SOCIAL_RE, HDRS  # noqa: E40238from fabrika.connectors.scrapfly import scrapfly_get, available  # noqa: E4023940WIX_STORES_APP = "1380b703-ce81-ff05-f115-39571d94dfcd"414243def curl_get(url, timeout=20):44    p = subprocess.run(["curl", "-sS", "-L", "--compressed", "--max-time", str(timeout),45                        "-A", HDRS["User-Agent"].split(" FabriKaBot")[0],46                        "-w", "\n%{http_code}", url],47                       capture_output=True, text=True, errors="replace")48    body, _, code = p.stdout.rpartition("\n")49    return (int(code) if code.isdigit() else 0), body505152def probe_endpoints(domain):53    """Teste tous les endpoints catalogue ; retourne (platform, endpoint) ou (None, None)."""54    base = f"https://{domain}"55    # Shopify (via curl : la moitié des blocages TLS disparaissent)56    code, body = curl_get(f"{base}/products.json?limit=1")57    if code == 200 and body.lstrip().startswith("{") and '"products"' in body[:200]:58        return "shopify", "/products.json"59    # WooCommerce Store API60    try:61        r = requests.get(f"{base}/wp-json/wc/store/v1/products?per_page=1", headers=HDRS, timeout=15)62        if r.status_code == 200 and r.text.strip().startswith("["):63            return "woocommerce", "/wp-json/wc/store/v1/products"64    except Exception:65        pass66    # Wix Stores67    try:68        r = requests.get(f"{base}/_api/v1/access-tokens", headers=HDRS, timeout=15, allow_redirects=True)69        if r.status_code == 200 and WIX_STORES_APP in (r.text or ""):70            return "wix", "/_api/wix-ecommerce-storefront-web/api"71    except Exception:72        pass73    # Squarespace74    for path in ("/shop", "/boutique", "/store"):75        try:76            r = requests.get(f"{base}{path}?format=json-pretty", headers=HDRS, timeout=12)77            if r.status_code == 200 and '"items"' in r.text[:5000]:78                return "squarespace", path + "?format=json"79        except Exception:80            pass81    return None, None828384def phase_a():85    reg = json.load(open(os.path.join(ROOT, "data", "stores.json")))["stores"]86    targets = [s["id"] for s in reg if not s.get("catalog_endpoint")]87    print(f"[A] {len(targets)} boutiques actives sans endpoint — re-sondage direct")88    found = 08990    def work(dom):91        plat, ep = probe_endpoints(dom)92        if not ep:93            return None94        cpath = os.path.join(CACHE, dom + ".json")95        try:96            rec = json.load(open(cpath))97        except Exception:98            rec = {"domain": dom, "active": True, "final_domain": dom}99        rec["platform"] = plat100        rec["catalog_endpoint"] = ep101        json.dump(rec, open(cpath, "w"))102        return dom, plat103104    with cf.ThreadPoolExecutor(12) as ex:105        for res in ex.map(work, targets):106            if res:107                found += 1108                print(f"  + {res[0]} -> {res[1]}", flush=True)109    print(f"[A] nouveaux connectables: {found}")110111112def build_record_from_html(domain, status, final_url, html):113    text = html[:400000]114    title = re.search(r"<title[^>]*>(.*?)</title>", text, re.S | re.I)115    rec = {116        "domain": domain, "checked_at": time.strftime("%Y-%m-%d"), "status": status,117        "final_url": final_url or f"https://{domain}", "active": True,118        "final_domain": domain,119        "title": re.sub(r"\s+", " ", title.group(1)).strip()[:200] if title else "",120        "platform": detect_platform(text, {}),121        "catalog_endpoint": None, "catalog_count_hint": None,122        "has_cart": bool(CART_RE.search(text)),123        "made_in_qc_wording": bool(MADE_RE.search(text)),124        "qc_postal": (POSTAL_RE.search(text) or [None]) and (POSTAL_RE.search(text).group(0) if POSTAL_RE.search(text) else None),125        "qc_phone": AREA_RE.search(text).group(0) if AREA_RE.search(text) else None,126        "mentions_quebec": bool(QC_WORD_RE.search(text)),127        "tld_quebec": domain.endswith(".quebec") or domain.endswith(".qc.ca"),128        "socials": list(dict.fromkeys(SOCIAL_RE.findall(text)))[:4],129        "language": None, "via": "scrapfly",130    }131    return rec132133134def phase_b(limit=None):135    if not available():136        print("[B] SCRAPFLY_API_KEY manquant — phase B sautée")137        return138    cands = [json.loads(l)["domain"] for l in open(os.path.join(ROOT, "data", "enriched", "candidates.jsonl"))]139    dead = []140    for dom in cands:141        cpath = os.path.join(CACHE, dom + ".json")142        try:143            rec = json.load(open(cpath))144            if not rec.get("active"):145                dead.append(dom)146        except Exception:147            dead.append(dom)148    if limit:149        dead = dead[:limit]150    print(f"[B] {len(dead)} candidats inactifs — retentative directe puis Scrapfly")151    revived = 0152153    def work(dom):154        # 1) direct rapide (les échecs transitoires)155        try:156            r = requests.get(f"https://{dom}", headers=HDRS, timeout=12, allow_redirects=True)157            if r.status_code == 200 and len(r.text) > 2000:158                return dom, 200, r.url, r.text, "direct"159        except Exception:160            pass161        # 2) scrapfly anti-bot162        try:163            status, content = scrapfly_get(f"https://{dom}")164            if status == 200 and len(content) > 2000:165                return dom, 200, f"https://{dom}", content, "scrapfly"166        except Exception:167            pass168        return None169170    with cf.ThreadPoolExecutor(6) as ex:171        for res in ex.map(work, dead):172            if not res:173                continue174            dom, status, final_url, html, via = res175            rec = build_record_from_html(dom, status, final_url, html)176            rec["via"] = via177            # sonde les endpoints catalogue dans la foulée178            plat, ep = probe_endpoints(dom)179            if ep:180                rec["platform"], rec["catalog_endpoint"] = plat, ep181            json.dump(rec, open(os.path.join(CACHE, dom + ".json"), "w"))182            revived += 1183            print(f"  ✚ {dom} (via {via}, plat={rec['platform'] or '-'}, ep={rec['catalog_endpoint'] or '-'})", flush=True)184    print(f"[B] ressuscités: {revived}/{len(dead)}")185186187if __name__ == "__main__":188    what = sys.argv[1] if len(sys.argv) > 1 else "ab"189    if "a" in what:190        phase_a()191    if "b" in what:192        phase_b()193