spb/fabri-ka

1#!/usr/bin/env python32"""Détecte les boutiques récoltables par le connecteur générique.34Pour chaque boutique active SANS endpoint catalogue : teste si son sitemap5expose des pages produit dont le balisage (JSON-LD/microdata/og) est6extractible. Si oui → marque catalog_endpoint='__generic__' dans le cache7de vérification (repris par build_registry). Scrapfly en secours.8"""9import concurrent.futures as cf10import json11import os12import sys1314import requests1516ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))17sys.path.insert(0, ROOT)18CACHE = os.path.join(ROOT, "data", "verify_cache")1920for line in open(os.path.join(ROOT, ".env")).read().splitlines():21    if "=" in line and not line.startswith("#"):22        k, _, v = line.partition("=")23        os.environ.setdefault(k.strip(), v.strip())2425from fabrika.connectors.generic import GenericConnector, extract_product  # noqa: E4022627UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/126 Safari/537.36"282930def test_domain(dom):31    sess = requests.Session()32    sess.headers["User-Agent"] = UA33    conn = GenericConnector({"id": dom, "url": f"https://{dom}"})34    conn.session = sess35    conn.use_scrapfly = False      # détection rapide, sans anti-bot36    conn.timeout = 1237    try:38        urls = conn._sitemap_products()39    except Exception:40        return dom, None, 041    if len(urls) < 3:42        return dom, None, len(urls)43    hits = 044    for u in urls[:6]:45        try:46            html = conn._fetch_html(u)47            if html and extract_product(u, html):48                hits += 149        except Exception:50            pass51        if hits >= 3:52            break53    ok = hits >= 354    return dom, ok, len(urls)555657def main():58    reg = json.load(open(os.path.join(ROOT, "data", "stores.json")))["stores"]59    # candidats : actives, pas déjà connectables, plateforme compatible rendu serveur60    server_plats = {"", "prestashop", "magento", "bigcommerce", "lightspeed",61                    "snipcart", "wordpress", "woocommerce", "squarespace"}62    targets = [s["id"] for s in reg63               if not s.get("catalog_endpoint") and (s.get("platform") or "") in server_plats]64    print(f"test générique sur {len(targets)} boutiques")65    found = 066    done = 067    with cf.ThreadPoolExecutor(10) as ex:68        for dom, ok, n in ex.map(test_domain, targets):69            done += 170            if ok:71                found += 172                cpath = os.path.join(CACHE, dom + ".json")73                try:74                    rec = json.load(open(cpath))75                except Exception:76                    rec = {"domain": dom, "active": True, "final_domain": dom}77                if not rec.get("platform"):78                    rec["platform"] = "generic"79                rec["catalog_endpoint"] = "__generic__"80                json.dump(rec, open(cpath, "w"))81                print(f"  ✓ {dom} ({n} pages produit)", flush=True)82            if done % 200 == 0:83                print(f"  … {done}/{len(targets)} — {found} récoltables", flush=True)84    print(f"nouvelles boutiques génériques: {found}/{len(targets)}")858687if __name__ == "__main__":88    main()89