#!/usr/bin/env python3
"""Vague 2 — délais et politiques de livraison des boutiques productives.

Pour chaque boutique avec produits (product_count > 0) et sans shipping_info :
  1. essaie les pages politiques usuelles selon la plateforme
     (Shopify : /policies/shipping-policy ; sinon /politique-de-livraison,
      /livraison, /pages/livraison, /shipping, /expedition, …) ;
  2. à défaut, cherche un lien « livraison / shipping / expédition » sur
     la page d'accueil et le suit (1 requête de plus) ;
  3. extrait un résumé texte (≤ 400 caractères) centré sur les mentions de
     délais/tarifs (jours, $, gratuit, Postes Canada…).

Écrit stores.shipping_info (colonne additive). Cache disque
data/enrich_cache/shipping/<domain>.json (y compris les échecs, pour ne pas
re-marteler les mêmes sites). Throttle poli : requêtes séquentielles par
domaine (0,4 s), 8 domaines en parallèle.

Usage : python3 scripts/enrich_shipping.py [--cap 300]
"""
import argparse
import concurrent.futures as cf
import html as _html
import json
import os
import re
import sys
import threading
import time
from urllib.parse import urljoin

import requests

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
CACHE = os.path.join(ROOT, "data", "enrich_cache", "shipping")
os.makedirs(CACHE, exist_ok=True)

for line in open(os.path.join(ROOT, ".env")).read().splitlines():
    if "=" in line and not line.startswith("#"):
        k, _, v = line.partition("=")
        os.environ.setdefault(k.strip(), v.strip())

from fabrika import db as fdb  # noqa: E402

HDRS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                      "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
        "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}

PATHS_SHOPIFY = ["/policies/shipping-policy"]
PATHS_COMMON = ["/politique-de-livraison", "/pages/livraison", "/livraison",
                "/pages/politique-de-livraison", "/expedition", "/pages/expedition",
                "/shipping", "/pages/shipping", "/shipping-policy",
                "/politique-dexpedition", "/pages/shipping-policy"]
LINK_RE = re.compile(
    r'href="([^"]*(?:livraison|exp[ée]dition|shipping|delivery)[^"]*)"', re.I)
KW_RE = re.compile(r"(livraison|exp[ée]dition|shipping|delivery|d[ée]lai)", re.I)
DENSITY_RE = re.compile(r"(\d|\$|gratuit|free|jours?|days?|ouvrable|business|"
                        r"postes canada|canada post|purolator|ramassage|cueillette|pickup)", re.I)
STEP = 0.4

# throttle global Shopify : le CDN limite par IP cliente, toutes boutiques
# confondues (même règle que connectors/shopify.py : 0,7 s entre requêtes)
_SHOPIFY_LOCK = threading.Lock()
_last_shopify = [0.0]


def _text(html):
    html = re.sub(r"(?is)<(script|style|noscript|svg|header|nav|footer)[^>]*>.*?</\1>", " ", html)
    html = re.sub(r"<[^>]+>", " ", html)
    return _html.unescape(re.sub(r"\s+", " ", html)).strip()


# bruit fréquent : sélecteurs de devises/pays des thèmes Shopify («(USD $)» ×30),
# menus. Une fenêtre qui en contient est rejetée.
NOISE_RE = re.compile(r"\((?:USD|EUR|GBP|CAD)\s*\$?\s*\)|currency|log in|wishlist", re.I)


def summarize(html):
    """Fenêtre ≤400 c. autour de la mention livraison la plus « dense »."""
    text = _text(html)
    if len(text) < 40:
        return None
    best, best_score = None, -1
    for m in KW_RE.finditer(text):
        start = max(0, m.start() - 60)
        if start:                       # démarre à une frontière de mot
            sp = text.find(" ", start)
            if 0 <= sp < m.start():
                start = sp + 1
        window = text[start:start + 460]
        if len(NOISE_RE.findall(window)) >= 2:
            continue
        score = len(DENSITY_RE.findall(window))
        if score > best_score:
            best, best_score = window, score
    if best is None or best_score < 1:
        return None
    # coupe proprement au dernier espace
    out = best[:400]
    if len(best) > 400:
        out = out.rsplit(" ", 1)[0]
    return out.strip() or None


def fetch(url, timeout=15, shopify=False):
    for attempt in range(2):
        if shopify:
            with _SHOPIFY_LOCK:
                wait = 0.7 - (time.time() - _last_shopify[0])
                if wait > 0:
                    time.sleep(wait)
                _last_shopify[0] = time.time()
        try:
            r = requests.get(url, headers=HDRS, timeout=timeout, allow_redirects=True)
            if r.status_code == 200 and len(r.text) > 300:
                return r.text, str(r.url)
            if r.status_code not in (429, 403):
                return None, None
        except Exception:
            pass
        time.sleep(3)
    return None, None


def work(store):
    dom, base, platform = store["id"], (store["url"] or f"https://{store['id']}").rstrip("/"), store["platform"]
    cpath = os.path.join(CACHE, dom + ".json")
    if os.path.exists(cpath):
        return json.load(open(cpath))
    rec = {"domain": dom, "shipping_info": None, "source_url": None,
           "checked_at": time.strftime("%Y-%m-%d")}
    sh = platform == "shopify"
    paths = (PATHS_SHOPIFY + PATHS_COMMON) if sh else PATHS_COMMON
    for p in paths:
        html, final = fetch(base + p, timeout=12, shopify=sh)
        if not sh:
            time.sleep(STEP)
        if html:
            info = summarize(html)
            if info:
                rec["shipping_info"], rec["source_url"] = info, final
                break
    if not rec["shipping_info"]:
        # repli : lien « livraison » sur la page d'accueil
        html, _ = fetch(base + "/", timeout=15, shopify=sh)
        if not sh:
            time.sleep(STEP)
        if html:
            m = LINK_RE.search(html)
            if m:
                href = urljoin(base + "/", _html.unescape(m.group(1)))
                if href.startswith("http") and dom in href:
                    page, final = fetch(href, timeout=12)
                    if page:
                        info = summarize(page)
                        if info:
                            rec["shipping_info"], rec["source_url"] = info, final
    json.dump(rec, open(cpath, "w"), ensure_ascii=False)
    return rec


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--cap", type=int, default=300)
    ap.add_argument("--workers", type=int, default=8)
    args = ap.parse_args()

    con = fdb.connect()
    try:
        con.execute("ALTER TABLE stores ADD COLUMN shipping_info TEXT")
        con.commit()
    except Exception:
        pass
    rows = [dict(r) for r in con.execute(
        "SELECT id, url, platform FROM stores WHERE product_count > 0 "
        "AND (shipping_info IS NULL OR shipping_info = '') "
        "ORDER BY product_count DESC LIMIT ?", (args.cap,))]
    con.close()
    print(f"[shipping] {len(rows)} boutiques ciblées (cap {args.cap})", flush=True)

    results, done = [], 0
    with cf.ThreadPoolExecutor(args.workers) as ex:
        for rec in ex.map(work, rows):
            results.append(rec)
            done += 1
            if done % 50 == 0:
                print(f"  {done}/{len(rows)}", flush=True)

    con = fdb.connect()
    n = 0
    for rec in results:
        if rec.get("shipping_info"):
            con.execute("UPDATE stores SET shipping_info=? WHERE id=?",
                        (rec["shipping_info"], rec["domain"]))
            n += 1
    con.commit(); con.close()
    print(f"[shipping] shipping_info rempli : {n}/{len(results)}")


if __name__ == "__main__":
    main()