#!/usr/bin/env python3
"""Vague 3 — découverte de nouvelles boutiques via 4 annuaires québécois frais.

Annuaires moissonnés (aucune clé API) :
  lespagesvertes         Les Pages Vertes — répertoire de l'économie verte QC
                         (fiches /entreprise/<slug>/ via business-sitemap*.xml)
  mauriciemiam           MIAM Mauricie — identifiant régional agroalimentaire
                         (liens externes de /repertoire-des-membres/)
  gardemangerduquebec    La Montérégie, le Garde-Manger du Québec — membres
                         « Producteurs et transformateurs » (fiches /membres-complices/)
  gardemangerduquebec_det  idem, mais détaillants/marchés/restaurants (classe C)
  terroiretsaveurs       Terroir et Saveurs (AATGQ) — liste « producteurs et
                         artisans pour acheter local » (liens externes directs)

Écartés après reconnaissance : Etsy (HTTP 403 anti-bot), Ma Zone Québec
(coquille Drupal vide côté serveur), Goûtez Lanaudière (liste membres dans un
CRM Eudonet inaccessible), Signé Local / CMAQ / économusées / Créateurs de
saveurs (déjà couverts : search_sweep_vendors, search_sweep_cmaq,
artisansaloeuvre, createursdesaveurs).

Usage :
  python3 scripts/wave3_discovery.py harvest     # -> data/raw/<source>.jsonl
  python3 scripts/wave3_discovery.py integrate   # aggregate + verify + ajout
                                                 #    additif au registre + DB

`integrate` est strictement additif : il n'ajoute que des domaines absents du
registre, ne touche à aucune boutique existante, et applique les mêmes
garde-fous que build_registry (preuve de localisation QC, classes A–E avec
évidence, dédup par domaine final).
"""
import argparse
import concurrent.futures as cf
import html as htmllib
import json
import os
import re
import sys
import time
from collections import Counter
from urllib.parse import unquote, urlparse, parse_qs

import requests

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RAW = os.path.join(ROOT, "data", "raw")
ENR = os.path.join(ROOT, "data", "enriched")
sys.path.insert(0, ROOT)
sys.path.insert(0, os.path.join(ROOT, "scripts"))

for line in open(os.path.join(ROOT, ".env")).read().splitlines():
    if "=" in line and not line.startswith("#"):
        k, _, v = line.partition("=")
        os.environ.setdefault(k.strip(), v.strip())

from verify import HDRS, POSTAL_RE, AREA_RE, verify_domain  # noqa: E402
from aggregate import BLOCK, norm_domain                    # noqa: E402

A_RE = re.compile(r'<a\s[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>', re.S | re.I)
H1_RE = re.compile(r"<h1[^>]*>(.*?)</h1>", re.S | re.I)
TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.S | re.I)
TAG_RE = re.compile(r"<[^>]+>")
SOCIAL_HOSTS = ("facebook.com", "instagram.com", "linkedin.com", "youtube.com",
                "tiktok.com", "pinterest.", "x.com", "twitter.com")

SESS = requests.Session()


def get(url, timeout=20):
    try:
        r = SESS.get(url, headers=HDRS, timeout=timeout, allow_redirects=True)
        if r.status_code == 200:
            return r.text
    except Exception:
        pass
    return ""


def strip_tags(s):
    return re.sub(r"\s+", " ", TAG_RE.sub(" ", htmllib.unescape(s or ""))).strip()


def unwrap_safelink(url):
    """Déballe les liens Outlook SafeLinks (vus sur MIAM)."""
    if "safelinks.protection.outlook.com" in url:
        q = parse_qs(urlparse(url).query)
        if q.get("url"):
            return unquote(q["url"][0])
    return url


def ext_links(html, own_domain):
    out = []
    for url, label in A_RE.findall(html):
        url = unwrap_safelink(htmllib.unescape(url))
        host = (urlparse(url).netloc or "").lower()
        if not host or own_domain in host:
            continue
        if any(s in host for s in SOCIAL_HOSTS):
            continue
        out.append((url, strip_tags(label)))
    return out


def fiche_record(url, html, own_domain):
    text = strip_tags(html[:200000])
    h1 = H1_RE.search(html)
    title = TITLE_RE.search(html)
    postal = POSTAL_RE.search(text)
    phone = AREA_RE.search(text)
    return {
        "url": url,
        "title": strip_tags(title.group(1))[:120] if title else "",
        "h1": strip_tags(h1.group(1))[:120] if h1 else "",
        "websites": [u for u, _ in ext_links(html, own_domain)],
        "socials": [],
        "postal_prefix": postal.group(0)[:3] if postal else None,
        "phone": phone.group(0) if phone else None,
        "regions_mentioned": [],
        "text_sample": text[:600],
    }


def write_jsonl(name, records):
    path = os.path.join(RAW, name + ".jsonl")
    with open(path, "w") as f:
        for r in records:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    print(f"[harvest] {name}: {len(records)} fiches/entrées -> {path}", flush=True)


def drop_frequent_domains(records, threshold=0.05):
    """Retire des fiches les domaines « partenaires » présents partout."""
    n = max(len(records), 1)
    freq = Counter()
    for r in records:
        for dom in {norm_domain(w) for w in r["websites"]} - {None}:
            freq[dom] += 1
    noisy = {d for d, c in freq.items() if c / n > threshold and c > 3}
    if noisy:
        print(f"[harvest] domaines partenaires ignorés (>5% des fiches): {sorted(noisy)}")
    for r in records:
        r["websites"] = [w for w in r["websites"] if norm_domain(w) not in noisy]
    return records


def crawl_fiches(urls, own_domain, workers=3, delay=0.4, label=""):
    records, done = [], 0

    def one(u):
        time.sleep(delay)
        html = get(u)
        return fiche_record(u, html, own_domain) if html else None

    with cf.ThreadPoolExecutor(workers) as ex:
        for rec in ex.map(one, urls):
            done += 1
            if rec and rec["websites"]:
                records.append(rec)
            if done % 100 == 0:
                print(f"  [{label}] {done}/{len(urls)}", flush=True)
    return records


# ---------------------------------------------------------------- harvesters

def harvest_lespagesvertes():
    urls = []
    for sm in ("business-sitemap.xml", "business-sitemap2.xml"):
        xml = get(f"https://lespagesvertes.ca/{sm}")
        urls += re.findall(r"<loc>(https://lespagesvertes\.ca/entreprise/[^<]+)</loc>", xml)
    urls = list(dict.fromkeys(urls))
    print(f"[harvest] lespagesvertes: {len(urls)} fiches à visiter", flush=True)
    records = crawl_fiches(urls, "lespagesvertes.ca", workers=5, delay=0.25, label="lpv")
    write_jsonl("lespagesvertes", drop_frequent_domains(records))


GMQ_CATS = {
    "producteur": "producteurs",
    "transformateur": "producteurs",
    "épicerie": "detaillants",
    "epicerie": "detaillants",
    "boutique gourmande": "detaillants",
    "marché public": "detaillants",
    "marche public": "detaillants",
    "restaurant": "detaillants",
    "traiteur": "detaillants",
    "aubergiste": "detaillants",
}


def harvest_gardemanger():
    page = get("https://gardemangerduquebec.ca/repertoire-des-membres/", timeout=40)
    fiches = sorted(set(re.findall(
        r'href="(https://gardemangerduquebec\.ca/membres-complices/[^"#]+)"', page)))
    print(f"[harvest] gardemangerduquebec: {len(fiches)} fiches à visiter", flush=True)
    prod, det = [], []

    def classify_cat(html):
        # classe body Avada : portfolio_category-<slug> = catégorie réelle de la fiche
        if "portfolio_category-producteurs-et-transformateurs" in html:
            return "producteurs"
        return "detaillants"

    done = 0

    def one(u):
        time.sleep(0.4)
        html = get(u)
        if not html:
            return None
        return classify_cat(html), fiche_record(u, html, "gardemangerduquebec.ca")

    with cf.ThreadPoolExecutor(3) as ex:
        for res in ex.map(one, fiches):
            done += 1
            if done % 50 == 0:
                print(f"  [gmq] {done}/{len(fiches)}", flush=True)
            if not res:
                continue
            cat, rec = res
            rec["regions_mentioned"] = ["Montérégie"]
            if not rec["websites"]:
                continue
            (prod if cat == "producteurs" else det).append(rec)
    write_jsonl("gardemangerduquebec", drop_frequent_domains(prod))
    write_jsonl("gardemangerduquebec_det", drop_frequent_domains(det))


def harvest_single_page(source, page_url, own_domain, region_hint, evidence):
    page = get(page_url, timeout=40)
    records, seen = [], set()
    for url, label in ext_links(page, own_domain):
        dom = norm_domain(url)
        if not dom or dom in seen or BLOCK.search(dom) or BLOCK.search(url):
            continue
        seen.add(dom)
        name = label if 2 < len(label) < 80 else ""
        records.append({
            "name": name,
            "domain": dom,
            "url": url,
            "region_hint": region_hint,
            "category_hint": "",
            "evidence": evidence,
            "query": page_url,
        })
    write_jsonl(source, records)


def harvest():
    os.makedirs(RAW, exist_ok=True)
    harvest_single_page(
        "mauriciemiam", "https://mauriciemiam.ca/repertoire-des-membres/",
        "mauriciemiam.ca", "Mauricie",
        "Membre MIAM Mauricie (identifiant régional agroalimentaire)")
    harvest_single_page(
        "terroiretsaveurs",
        "https://terroiretsaveurs.com/repertoire-terroir-saveurs-liste-des-producteurs-et-artisans-pour-acheter-local/",
        "terroiretsaveurs.com", "",
        "Producteur/artisan du répertoire Terroir et Saveurs (AATGQ)")
    harvest_gardemanger()
    harvest_lespagesvertes()


# ---------------------------------------------------------------- integrate

WAVE3_SOURCES = ["lespagesvertes", "mauriciemiam", "gardemangerduquebec",
                 "gardemangerduquebec_det", "terroiretsaveurs"]


def integrate():
    import subprocess
    from datetime import date
    import build_registry as br
    from fabrika import db as fdb

    for s in WAVE3_SOURCES:
        assert s in br.SOURCE_PRIORS, f"prior manquant dans build_registry: {s}"

    # 1) ré-agrège tous les raw (inclut la vague 3) -> candidates.jsonl canonique
    subprocess.run([sys.executable, os.path.join(ROOT, "scripts", "aggregate.py")],
                   check=True)

    cands = {}
    with open(os.path.join(ENR, "candidates.jsonl")) as f:
        for line in f:
            c = json.loads(line)
            cands[c["domain"]] = c

    verified_path = os.path.join(ENR, "verified.jsonl")
    verified = {}
    with open(verified_path) as f:
        for line in f:
            v = json.loads(line)
            verified[v["domain"]] = v

    reg_path = os.path.join(ROOT, "data", "stores.json")
    reg = json.load(open(reg_path))
    existing_ids = {s["id"] for s in reg["stores"]}

    # 2) domaines candidats jamais vérifiés (donc nouveaux pour le pipeline)
    new_domains = sorted(d for d in cands
                         if d not in verified and d not in existing_ids)
    print(f"[integrate] {len(new_domains)} nouveaux domaines à vérifier", flush=True)

    new_recs = []
    with cf.ThreadPoolExecutor(16) as ex:
        for i, rec in enumerate(ex.map(verify_domain, new_domains)):
            new_recs.append(rec)
            if (i + 1) % 100 == 0:
                print(f"  verify {i+1}/{len(new_domains)}", flush=True)
    with open(verified_path, "a") as f:
        for r in new_recs:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
            verified[r["domain"]] = r

    # 3) construit les entrées registre (mêmes règles que build_registry)
    added, skipped_dup, skipped_qc, skipped_dead = [], 0, 0, 0
    for dom in new_domains:
        cand, ver = cands[dom], verified.get(dom, {})
        if not ver.get("active"):
            skipped_dead += 1
            continue
        final_dom = ver.get("final_domain") or dom
        if final_dom in existing_ids:
            skipped_dup += 1
            continue
        qc_signal = any(ver.get(k) for k in ("qc_postal", "qc_phone", "tld_quebec",
                                             "mentions_quebec", "made_in_qc_wording"))
        qc_source = any(s in br.QC_ONLY_SOURCES for s in cand.get("sources", []))
        if not qc_signal and not qc_source:
            skipped_qc += 1
            continue
        cls, conf, ev = br.classify(cand, ver)
        default_cat = next((br.SOURCE_PRIORS[s][3] for s in br.PRIORITY
                            if s in cand.get("sources", []) and br.SOURCE_PRIORS[s][3]), None)
        platform = ver.get("platform") or ""
        catalog_endpoint = ver.get("catalog_endpoint") or ""
        if platform == "wix" and not catalog_endpoint:
            catalog_endpoint = "/_api/wix-ecommerce-storefront-web/api"
        fu = urlparse(ver.get("final_url") or f"https://{final_dom}")
        store = {
            "id": final_dom,
            "name": br.clean_name(cand, ver),
            "url": f"{fu.scheme}://{fu.netloc}",
            "platform": platform,
            "catalog_endpoint": catalog_endpoint,
            "city": "",
            "region": br.pick_region(cand) or br.region_from_postal(cand, ver),
            "postal_prefix": cand.get("postal_prefix") or (ver.get("qc_postal") or "")[:3] or None,
            "phone": cand.get("phone") or ver.get("qc_phone"),
            "origin_class": cls,
            "origin_confidence": conf,
            "origin_evidence": ev,
            "categories": [default_cat] if default_cat else [],
            "socials": (cand.get("socials") or [])[:4] or ver.get("socials", []),
            "discovery_sources": cand.get("sources", []),
            "discovery_source_urls": cand.get("source_pages", [])[:5],
            "language": ver.get("language"),
            "ecommerce": bool(ver.get("has_cart") or catalog_endpoint),
            "verification_date": ver.get("checked_at") or str(date.today()),
            "status": "verified" if (conf >= 0.6 and ver.get("mentions_quebec")) else "probable",
            "enabled": bool(catalog_endpoint),
        }
        existing_ids.add(final_dom)
        added.append(store)

    reg["stores"].extend(added)
    reg["count"] = len(reg["stores"])
    reg["generated"] = str(date.today())
    json.dump(reg, open(reg_path, "w"), ensure_ascii=False, indent=1)

    con = fdb.connect()
    for s in added:
        fdb.upsert_store(con, s)
    con.commit(); con.close()

    enabled = [s["id"] for s in added if s["enabled"]]
    per_plat = Counter(s["platform"] or "(aucune)" for s in added)
    per_src = Counter(src for s in added for src in s["discovery_sources"])
    per_cls = Counter(s["origin_class"] for s in added)
    print(f"[integrate] boutiques ajoutées: {len(added)} | connectables (enabled): {len(enabled)}")
    print(f"[integrate] écartées — mortes/injoignables: {skipped_dead}, "
          f"dédup domaine final: {skipped_dup}, sans preuve QC: {skipped_qc}")
    print("[integrate] par plateforme:", dict(per_plat.most_common()))
    print("[integrate] par source:", dict(per_src.most_common()))
    print("[integrate] par classe:", dict(per_cls))
    with open(os.path.join(ROOT, "data", "wave3_new_enabled.txt"), "w") as f:
        f.write("\n".join(enabled) + "\n")
    if enabled:
        print("[integrate] à synchroniser: python run.py sync $(cat data/wave3_new_enabled.txt)")


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("cmd", choices=["harvest", "integrate"])
    args = ap.parse_args()
    harvest() if args.cmd == "harvest" else integrate()