#!/usr/bin/env python3
"""Verification pipeline: probe each candidate domain.

For each domain from data/enriched/candidates.jsonl:
  - fetch homepage (https, with UA), record status + final URL
  - detect ecommerce platform + whether its public catalog endpoint responds
    (Shopify /products.json, WooCommerce /wp-json/wc/store/v1/products,
     Squarespace ?format=json)
  - detect cart/checkout signals
  - detect Quebec-location signals (postal codes GHJ, area codes, mentions)
  - detect language, social links, "made in Quebec" wording
Results cached per-domain in data/verify_cache/. Output:
data/enriched/verified.jsonl
"""
import concurrent.futures as cf
import json
import os
import re
import sys
import time
from urllib.parse import urlparse

import requests

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CACHE = os.path.join(ROOT, "data", "verify_cache")
OUT = os.path.join(ROOT, "data", "enriched")
os.makedirs(CACHE, exist_ok=True)

HDRS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 FabriKaBot/1.0 (+contact@spboucher.ai)",
        "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}

POSTAL_RE = re.compile(r"\b[GHJ]\d[A-Z]\s?\d[A-Z]\d\b")
AREA_RE = re.compile(r"\(?\b(418|514|450|819|579|581|438|873|367)\)?[\s.\-]?\d{3}[\s.\-]?\d{4}\b")
MADE_RE = re.compile(r"(?i)(fabriqu[ée]s?\s+(au|ici\s+au)\s+qu[ée]bec|fait(es)?\s+(au|ici\s+au)\s+qu[ée]bec|"
                     r"made\s+in\s+qu[ée]bec|con[çc]u\s+(et\s+fabriqu[ée]?\s+)?au\s+qu[ée]bec|"
                     r"produit\s+du\s+qu[ée]bec|fait\s+[àa]\s+la\s+main\s+au\s+qu[ée]bec|"
                     r"artisanal|fait\s+ici|100\s?%\s?qu[ée]b[ée]cois)")
QC_WORD_RE = re.compile(r"(?i)\bqu[ée]bec\b|\bqc\b")
CART_RE = re.compile(r"(?i)(add[\s_\-]?to[\s_\-]?cart|ajouter\s+au\s+panier|/cart\b|/panier\b|"
                     r"checkout|caisse|mon\s+panier|shopping[\s_\-]?cart|data-product-form)")

SOCIAL_RE = re.compile(r'https?://(?:www\.)?(facebook\.com|instagram\.com)/[A-Za-z0-9_.\-/%]+')


def detect_platform(html, headers):
    h = html.lower()
    hdr = " ".join(f"{k}:{v}" for k, v in headers.items()).lower()
    if "cdn.shopify" in h or "shopify.theme" in h or "x-shopid" in hdr or "myshopify.com" in h:
        return "shopify"
    if "woocommerce" in h:
        return "woocommerce"
    if "wixstatic.com" in h or "wix.com" in h and "wixsite" in h:
        return "wix"
    if "squarespace" in h:
        return "squarespace"
    if "prestashop" in h:
        return "prestashop"
    if "bigcommerce" in h:
        return "bigcommerce"
    if "webshopapp" in h or "lightspeed" in h and "cart" in h:
        return "lightspeed"
    if "ecwid" in h:
        return "ecwid"
    if "snipcart" in h:
        return "snipcart"
    if "sumup" in h and "store" in h:
        return "sumup"
    if "square.site" in h or "squareup.com" in h and "cart" in h:
        return "square"
    if "magento" in h or "mage/cookies" in h:
        return "magento"
    if "wp-content" in h:
        return "wordpress"
    return ""


def probe_catalog(domain, platform, sess):
    """Check whether the public catalog endpoint responds; return (endpoint, count_hint)."""
    try:
        if platform == "shopify":
            r = sess.get(f"https://{domain}/products.json?limit=1", headers=HDRS, timeout=15)
            if r.status_code == 200 and "products" in r.text[:200]:
                return "/products.json", None
        elif platform in ("woocommerce", "wordpress"):
            r = sess.get(f"https://{domain}/wp-json/wc/store/v1/products?per_page=1", headers=HDRS, timeout=15)
            if r.status_code == 200 and r.text.strip().startswith("["):
                total = r.headers.get("X-WP-Total")
                return "/wp-json/wc/store/v1/products", int(total) if total else None
        elif platform == "squarespace":
            for path in ("/shop", "/boutique", "/store"):
                r = sess.get(f"https://{domain}{path}?format=json-pretty", headers=HDRS, timeout=15)
                if r.status_code == 200 and '"items"' in r.text[:5000]:
                    return path + "?format=json", None
    except Exception:
        pass
    return None, None


def verify_domain(domain):
    cpath = os.path.join(CACHE, domain + ".json")
    if os.path.exists(cpath):
        return json.load(open(cpath))
    rec = {"domain": domain, "checked_at": time.strftime("%Y-%m-%d")}
    sess = requests.Session()
    html, final_url, status = "", "", None
    for scheme_host in (f"https://{domain}", f"https://www.{domain}", f"http://{domain}"):
        try:
            r = sess.get(scheme_host, headers=HDRS, timeout=20, allow_redirects=True)
            status, final_url, html = r.status_code, r.url, r.text
            resp_headers = dict(r.headers)
            if status == 200:
                break
        except Exception as e:
            rec.setdefault("errors", []).append(str(e)[:120])
    rec["status"] = status
    rec["final_url"] = final_url
    if not html or status != 200:
        rec["active"] = False
        json.dump(rec, open(cpath, "w"))
        return rec

    rec["active"] = True
    final_dom = urlparse(final_url).netloc.lower().removeprefix("www.")
    rec["final_domain"] = final_dom
    text = html[:400000]
    title_m = re.search(r"<title[^>]*>(.*?)</title>", text, re.S | re.I)
    rec["title"] = re.sub(r"\s+", " ", title_m.group(1)).strip()[:200] if title_m else ""
    platform = detect_platform(text, resp_headers)
    rec["platform"] = platform
    endpoint, count = probe_catalog(final_dom, platform, sess)
    rec["catalog_endpoint"] = endpoint
    rec["catalog_count_hint"] = count
    rec["has_cart"] = bool(CART_RE.search(text))
    rec["made_in_qc_wording"] = bool(MADE_RE.search(text))
    postal = POSTAL_RE.search(text)
    rec["qc_postal"] = postal.group(0) if postal else None
    area = AREA_RE.search(text)
    rec["qc_phone"] = area.group(0) if area else None
    rec["mentions_quebec"] = bool(QC_WORD_RE.search(text))
    rec["tld_quebec"] = final_dom.endswith(".quebec") or final_dom.endswith(".qc.ca")
    rec["socials"] = list(dict.fromkeys(SOCIAL_RE.findall(text)))[:4]
    fr_hits = len(re.findall(r"(?i)\b(nous|votre|panier|boutique|livraison|accueil|produits)\b", text))
    en_hits = len(re.findall(r"(?i)\b(shop|cart|shipping|home|about us|products)\b", text))
    rec["language"] = ("fr" if fr_hits >= en_hits else "en") if (fr_hits + en_hits) > 3 else None
    json.dump(rec, open(cpath, "w"))
    return rec


def main():
    src = sys.argv[1] if len(sys.argv) > 1 else os.path.join(OUT, "candidates.jsonl")
    domains = []
    with open(src) as f:
        for line in f:
            domains.append(json.loads(line)["domain"])
    domains = list(dict.fromkeys(domains))
    print(f"verifying {len(domains)} domains")
    results = []
    with cf.ThreadPoolExecutor(24) as ex:
        for i, rec in enumerate(ex.map(verify_domain, domains)):
            results.append(rec)
            if (i + 1) % 100 == 0:
                print(f"  {i+1}/{len(domains)}", flush=True)
    out = os.path.join(OUT, "verified.jsonl")
    with open(out, "w") as f:
        for r in results:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    active = sum(1 for r in results if r.get("active"))
    plat = {}
    for r in results:
        if r.get("active"):
            plat[r.get("platform") or "unknown"] = plat.get(r.get("platform") or "unknown", 0) + 1
    connectable = sum(1 for r in results if r.get("catalog_endpoint"))
    print(f"active: {active}/{len(results)}  connectable-catalog: {connectable}")
    print("platforms:", json.dumps(plat, indent=1))


if __name__ == "__main__":
    main()