spb/fabri-ka

1#!/usr/bin/env python32"""Phase 2 — coordonnées boutiques : courriel, téléphone, réseaux sociaux.34Pour chaque boutique productive sans courriel :5  1. page d'accueil : mailto:, JSON-LD Organization/LocalBusiness6     (email, telephone, sameAs), liens Instagram/Facebook du HTML ;7  2. sinon une page contact usuelle (/contact, /pages/contact,8     /nous-joindre, /contactez-nous, /pages/nous-joindre, /a-propos).910Max 2 requêtes réseau par boutique, throttle 0,5 s par worker (Shopify :11verrou global 0,7 s comme le connecteur). Cache disque12data/enrich_cache/contacts/<dom>.json (échecs inclus).1314Écrit : stores.email, stores.phone (si vide) ; réseaux fusionnés dans le15registre (socials) sans doublon.1617Usage : .venv/bin/python scripts/enrich_contacts.py [--cap 400] [--workers 6]18"""19import argparse20import concurrent.futures as cf21import html as _html22import json23import os24import re25import sys26import threading27import time2829import requests3031ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))32sys.path.insert(0, ROOT)33CACHE = os.path.join(ROOT, "data", "enrich_cache", "contacts")34os.makedirs(CACHE, exist_ok=True)3536from fabrika import db as fdb  # noqa: E4023738HDRS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "39                      "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",40        "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}4142CONTACT_PATHS = ["/pages/contact", "/contact", "/nous-joindre", "/contactez-nous",43                 "/pages/nous-joindre", "/pages/contactez-nous", "/contact-us"]44EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")45BAD_EMAIL = re.compile(r"(sentry|example|wixpress|\.png|\.jpe?g|\.gif|\.webp|\.svg"46                       r"|@2x|@3x|schema\.org|sentry-next|\.js$|\.css$|no-?reply)", re.I)47MAILTO_RE = re.compile(r'mailto:([^"\'?>\s]+)', re.I)48PHONE_RE = re.compile(r"(?:\+1[ .-]?)?\(?\b([2-9]\d{2})\)?[ .-]?(\d{3})[ .-]?(\d{4})\b")49SOCIAL_RE = re.compile(r'https?://(?:www\.)?(?:instagram\.com|facebook\.com)/[A-Za-z0-9_.\-/%]+', re.I)50LD_RE = re.compile(r'<script[^>]+application/ld\+json[^>]*>(.*?)</script>', re.I | re.S)5152_SHOPIFY_LOCK = threading.Lock()53_last_shopify = [0.0]545556def fetch(url, shopify=False, timeout=15):57    if shopify:58        with _SHOPIFY_LOCK:59            wait = 0.7 - (time.time() - _last_shopify[0])60            if wait > 0:61                time.sleep(wait)62            _last_shopify[0] = time.time()63    try:64        r = requests.get(url, headers=HDRS, timeout=timeout, allow_redirects=True)65        if r.status_code == 200 and len(r.text) > 200:66            return r.text67    except Exception:68        pass69    return None707172def harvest(html_text: str, rec: dict) -> None:73    """Extrait courriel / téléphone / réseaux d'un HTML (JSON-LD d'abord)."""74    for m in LD_RE.finditer(html_text):75        try:76            data = json.loads(m.group(1).strip())77        except Exception:78            continue79        stack = data if isinstance(data, list) else [data]80        while stack:81            node = stack.pop()82            if isinstance(node, list):83                stack.extend(node)84                continue85            if not isinstance(node, dict):86                continue87            stack.extend(v for v in node.values() if isinstance(v, (dict, list)))88            if not rec.get("email") and isinstance(node.get("email"), str):89                e = node["email"].replace("mailto:", "").strip()90                if EMAIL_RE.fullmatch(e) and not BAD_EMAIL.search(e):91                    rec["email"] = e92            if not rec.get("phone") and isinstance(node.get("telephone"), str):93                rec["phone"] = node["telephone"].strip()[:30]94            for u in (node.get("sameAs") or []) if isinstance(node.get("sameAs"), list) else []:95                if isinstance(u, str) and SOCIAL_RE.match(u):96                    rec.setdefault("socials", []).append(u.rstrip("/"))97    if not rec.get("email"):98        m = MAILTO_RE.search(html_text)99        if m:100            e = _html.unescape(m.group(1)).strip()101            if EMAIL_RE.fullmatch(e) and not BAD_EMAIL.search(e):102                rec["email"] = e103    if not rec.get("email"):104        for e in EMAIL_RE.findall(html_text[:200000]):105            if not BAD_EMAIL.search(e):106                rec["email"] = e107                break108    if not rec.get("phone"):109        m = PHONE_RE.search(re.sub(r"<[^>]+>", " ", html_text[:150000]))110        if m:111            rec["phone"] = f"{m.group(1)} {m.group(2)}-{m.group(3)}"112    for u in SOCIAL_RE.findall(html_text[:200000]):113        if "/sharer" in u or "/share?" in u or "/plugins" in u:114            continue115        rec.setdefault("socials", []).append(u.rstrip("/"))116117118def work(store):119    dom = store["id"]120    cpath = os.path.join(CACHE, dom + ".json")121    if os.path.exists(cpath):122        return json.load(open(cpath))123    base = (store["url"] or f"https://{dom}").rstrip("/")124    sh = store["platform"] == "shopify"125    rec = {"domain": dom, "email": None, "phone": None, "socials": [],126           "checked_at": time.strftime("%Y-%m-%d"), "requests": 0}127    html_text = fetch(base + "/", shopify=sh)128    rec["requests"] += 1129    if html_text:130        harvest(html_text, rec)131        if not rec["email"]:132            # une seule page contact : la première trouvée dans le HTML d'accueil133            m = re.search(r'href="([^"]*(?:contact|nous-joindre|joindre)[^"]*)"',134                          html_text, re.I)135            path = None136            if m:137                href = _html.unescape(m.group(1))138                if href.startswith("/"):139                    path = href140                elif dom in href:141                    path = "/" + href.split(dom, 1)[1].lstrip("/")142            if not path:143                path = CONTACT_PATHS[0] if not sh else "/pages/contact"144            page = fetch(base + path, shopify=sh)145            rec["requests"] += 1146            if page:147                harvest(page, rec)148    rec["socials"] = sorted(set(rec["socials"]))[:6]149    json.dump(rec, open(cpath, "w"), ensure_ascii=False)150    return rec151152153def main():154    ap = argparse.ArgumentParser()155    ap.add_argument("--cap", type=int, default=400)156    ap.add_argument("--workers", type=int, default=6)157    ap.add_argument("--exclude-platform", default=None,158                    help="saute une plateforme (ex. shopify pendant un cycle de sync)")159    args = ap.parse_args()160161    con = fdb.connect()162    extra = "AND platform<>? " if args.exclude_platform else ""163    params = ([args.exclude_platform] if args.exclude_platform else []) + [args.cap]164    rows = [dict(r) for r in con.execute(165        "SELECT id, url, platform FROM stores WHERE product_count > 0 "166        f"AND (email IS NULL OR email='') {extra}ORDER BY product_count DESC LIMIT ?",167        params)]168    con.close()169    print(f"[contacts] {len(rows)} boutiques ciblées (cap {args.cap})", flush=True)170171    results, done, nreq = [], 0, 0172    with cf.ThreadPoolExecutor(args.workers) as ex:173        for rec in ex.map(work, rows):174            results.append(rec)175            nreq += rec.get("requests", 0)176            done += 1177            if done % 50 == 0:178                print(f"  {done}/{len(rows)}", flush=True)179180    con = fdb.connect()181    n_email = n_phone = 0182    for rec in results:183        if rec.get("email"):184            con.execute("UPDATE stores SET email=? WHERE id=?", (rec["email"], rec["domain"]))185            n_email += 1186        if rec.get("phone"):187            con.execute("UPDATE stores SET phone=CASE WHEN COALESCE(phone,'')='' "188                        "THEN ? ELSE phone END WHERE id=?", (rec["phone"], rec["domain"]))189            n_phone += 1190    con.commit(); con.close()191    print(f"[contacts] courriels : {n_email}/{len(results)} | téléphones : {n_phone} "192          f"| requêtes réseau : {nreq}")193194195if __name__ == "__main__":196    main()197