spb/fabri-ka

1#!/usr/bin/env python32"""Vague 2 — délais et politiques de livraison des boutiques productives.34Pour chaque boutique avec produits (product_count > 0) et sans shipping_info :5  1. essaie les pages politiques usuelles selon la plateforme6     (Shopify : /policies/shipping-policy ; sinon /politique-de-livraison,7      /livraison, /pages/livraison, /shipping, /expedition, …) ;8  2. à défaut, cherche un lien « livraison / shipping / expédition » sur9     la page d'accueil et le suit (1 requête de plus) ;10  3. extrait un résumé texte (≤ 400 caractères) centré sur les mentions de11     délais/tarifs (jours, $, gratuit, Postes Canada…).1213Écrit stores.shipping_info (colonne additive). Cache disque14data/enrich_cache/shipping/<domain>.json (y compris les échecs, pour ne pas15re-marteler les mêmes sites). Throttle poli : requêtes séquentielles par16domaine (0,4 s), 8 domaines en parallèle.1718Usage : python3 scripts/enrich_shipping.py [--cap 300]19"""20import argparse21import concurrent.futures as cf22import html as _html23import json24import os25import re26import sys27import threading28import time29from urllib.parse import urljoin3031import requests3233ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))34sys.path.insert(0, ROOT)35CACHE = os.path.join(ROOT, "data", "enrich_cache", "shipping")36os.makedirs(CACHE, exist_ok=True)3738for line in open(os.path.join(ROOT, ".env")).read().splitlines():39    if "=" in line and not line.startswith("#"):40        k, _, v = line.partition("=")41        os.environ.setdefault(k.strip(), v.strip())4243from fabrika import db as fdb  # noqa: E4024445HDRS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "46                      "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",47        "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}4849PATHS_SHOPIFY = ["/policies/shipping-policy"]50PATHS_COMMON = ["/politique-de-livraison", "/pages/livraison", "/livraison",51                "/pages/politique-de-livraison", "/expedition", "/pages/expedition",52                "/shipping", "/pages/shipping", "/shipping-policy",53                "/politique-dexpedition", "/pages/shipping-policy"]54LINK_RE = re.compile(55    r'href="([^"]*(?:livraison|exp[ée]dition|shipping|delivery)[^"]*)"', re.I)56KW_RE = re.compile(r"(livraison|exp[ée]dition|shipping|delivery|d[ée]lai)", re.I)57DENSITY_RE = re.compile(r"(\d|\$|gratuit|free|jours?|days?|ouvrable|business|"58                        r"postes canada|canada post|purolator|ramassage|cueillette|pickup)", re.I)59STEP = 0.46061# throttle global Shopify : le CDN limite par IP cliente, toutes boutiques62# confondues (même règle que connectors/shopify.py : 0,7 s entre requêtes)63_SHOPIFY_LOCK = threading.Lock()64_last_shopify = [0.0]656667def _text(html):68    html = re.sub(r"(?is)<(script|style|noscript|svg|header|nav|footer)[^>]*>.*?</\1>", " ", html)69    html = re.sub(r"<[^>]+>", " ", html)70    return _html.unescape(re.sub(r"\s+", " ", html)).strip()717273# bruit fréquent : sélecteurs de devises/pays des thèmes Shopify («(USD $)» ×30),74# menus. Une fenêtre qui en contient est rejetée.75NOISE_RE = re.compile(r"\((?:USD|EUR|GBP|CAD)\s*\$?\s*\)|currency|log in|wishlist", re.I)767778def summarize(html):79    """Fenêtre ≤400 c. autour de la mention livraison la plus « dense »."""80    text = _text(html)81    if len(text) < 40:82        return None83    best, best_score = None, -184    for m in KW_RE.finditer(text):85        start = max(0, m.start() - 60)86        if start:                       # démarre à une frontière de mot87            sp = text.find(" ", start)88            if 0 <= sp < m.start():89                start = sp + 190        window = text[start:start + 460]91        if len(NOISE_RE.findall(window)) >= 2:92            continue93        score = len(DENSITY_RE.findall(window))94        if score > best_score:95            best, best_score = window, score96    if best is None or best_score < 1:97        return None98    # coupe proprement au dernier espace99    out = best[:400]100    if len(best) > 400:101        out = out.rsplit(" ", 1)[0]102    return out.strip() or None103104105def fetch(url, timeout=15, shopify=False):106    for attempt in range(2):107        if shopify:108            with _SHOPIFY_LOCK:109                wait = 0.7 - (time.time() - _last_shopify[0])110                if wait > 0:111                    time.sleep(wait)112                _last_shopify[0] = time.time()113        try:114            r = requests.get(url, headers=HDRS, timeout=timeout, allow_redirects=True)115            if r.status_code == 200 and len(r.text) > 300:116                return r.text, str(r.url)117            if r.status_code not in (429, 403):118                return None, None119        except Exception:120            pass121        time.sleep(3)122    return None, None123124125def work(store):126    dom, base, platform = store["id"], (store["url"] or f"https://{store['id']}").rstrip("/"), store["platform"]127    cpath = os.path.join(CACHE, dom + ".json")128    if os.path.exists(cpath):129        return json.load(open(cpath))130    rec = {"domain": dom, "shipping_info": None, "source_url": None,131           "checked_at": time.strftime("%Y-%m-%d")}132    sh = platform == "shopify"133    paths = (PATHS_SHOPIFY + PATHS_COMMON) if sh else PATHS_COMMON134    for p in paths:135        html, final = fetch(base + p, timeout=12, shopify=sh)136        if not sh:137            time.sleep(STEP)138        if html:139            info = summarize(html)140            if info:141                rec["shipping_info"], rec["source_url"] = info, final142                break143    if not rec["shipping_info"]:144        # repli : lien « livraison » sur la page d'accueil145        html, _ = fetch(base + "/", timeout=15, shopify=sh)146        if not sh:147            time.sleep(STEP)148        if html:149            m = LINK_RE.search(html)150            if m:151                href = urljoin(base + "/", _html.unescape(m.group(1)))152                if href.startswith("http") and dom in href:153                    page, final = fetch(href, timeout=12)154                    if page:155                        info = summarize(page)156                        if info:157                            rec["shipping_info"], rec["source_url"] = info, final158    json.dump(rec, open(cpath, "w"), ensure_ascii=False)159    return rec160161162def main():163    ap = argparse.ArgumentParser()164    ap.add_argument("--cap", type=int, default=300)165    ap.add_argument("--workers", type=int, default=8)166    args = ap.parse_args()167168    con = fdb.connect()169    try:170        con.execute("ALTER TABLE stores ADD COLUMN shipping_info TEXT")171        con.commit()172    except Exception:173        pass174    rows = [dict(r) for r in con.execute(175        "SELECT id, url, platform FROM stores WHERE product_count > 0 "176        "AND (shipping_info IS NULL OR shipping_info = '') "177        "ORDER BY product_count DESC LIMIT ?", (args.cap,))]178    con.close()179    print(f"[shipping] {len(rows)} boutiques ciblées (cap {args.cap})", flush=True)180181    results, done = [], 0182    with cf.ThreadPoolExecutor(args.workers) as ex:183        for rec in ex.map(work, rows):184            results.append(rec)185            done += 1186            if done % 50 == 0:187                print(f"  {done}/{len(rows)}", flush=True)188189    con = fdb.connect()190    n = 0191    for rec in results:192        if rec.get("shipping_info"):193            con.execute("UPDATE stores SET shipping_info=? WHERE id=?",194                        (rec["shipping_info"], rec["domain"]))195            n += 1196    con.commit(); con.close()197    print(f"[shipping] shipping_info rempli : {n}/{len(results)}")198199200if __name__ == "__main__":201    main()202