#!/usr/bin/env python3
"""Vague 4 (Phase 3 — expansion) : nouvelles sources de fabricants.

Sources moissonnées (fiche d'évaluation datée du 2026-08-19 dans
docs/CONFORMITE.md ; chaque source a été sondée AVANT connexion) :

  cartv_bio           Répertoire public des entreprises certifiées biologiques
                      du Québec (CARTV / SIPAB, produitsbioquebec.info).
                      Formulaire Struts public « recherche par type
                      d'opération » ; 7 types × toutes les régions.
                      Fiches complètes : adresse, municipalité, région, CP,
                      tél, site web, certificateur, date de certification.
                      -> découverte + champ `certifications` structuré.
  ctaq                Répertoire des membres du Conseil de la transformation
                      alimentaire du Québec (conseiltaq.com,
                      /ajax-search-organisation). On ne garde PAS les
                      « Associés » (fournisseurs de services) — seulement les
                      transformateurs. Fiche /organisation/<id> = site web.
  vendors_collectifs  Champ `vendor` des boutiques collectives déjà
                      connectées (paperole, galerieiris, wachiya — patron
                      Signé Local prouvé) -> résolution prudente du site
                      officiel par candidats de domaine dérivés du nom,
                      validés par correspondance du titre de page.
  cibim               Boulangeries artisanales membres de la CIBIM
                      (cibim.org/membres) — répertoire recommandé par l'UPA.
  canardduquebec      Producteurs — Éleveurs de canards et d'oies du Québec.
  lebongoutfraisdesiles  Producteurs/transformateurs des Îles-de-la-Madeleine
                      (région sous-couverte).
  acheterquebecois_mtl   Catégories « Montréal / Fabriqué à Montréal »
                      d'acheterquebecois.ca (Montréal sous-représentée ;
                      PME MTL n'a aucun répertoire public « Fabriqué à
                      Montréal » — constaté 2026-08-19).

Écartées (raison consignée dans docs/CONFORMITE.md) : REQ open data
(WAF Cloudflare + licence CC-BY-NC-SA non commerciale), UPA Mangeons local
(app web décommissionnée — redirections vers upa.qc.ca/citoyen), Salon des
métiers d'art (salondesmetiersdart.com = coquille ; liste des exposants déjà
couverte par le répertoire CMAQ), Goûtez Lanaudière (CRM Eudonet toujours en
404, re-testé), PME MTL (aucun répertoire).

Usage :
  python3 scripts/wave4_discovery.py harvest [source ...]
  python3 scripts/wave4_discovery.py integrate   # additif, sérialisé
                                                 # (vérifier qu'aucune passe
                                                 # n'écrit stores.json)
"""
import argparse
import concurrent.futures as cf
import html as htmllib
import json
import os
import re
import subprocess
import sys
import time
import unicodedata
from collections import Counter
from urllib.parse import urlparse

import requests

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RAW = os.path.join(ROOT, "data", "raw")
ENR = os.path.join(ROOT, "data", "enriched")
sys.path.insert(0, ROOT)
sys.path.insert(0, os.path.join(ROOT, "scripts"))

for line in open(os.path.join(ROOT, ".env")).read().splitlines():
    if "=" in line and not line.startswith("#"):
        k, _, v = line.partition("=")
        os.environ.setdefault(k.strip(), v.strip())

from verify import HDRS, POSTAL_RE, AREA_RE, verify_domain  # noqa: E402
from aggregate import BLOCK, norm_domain                    # noqa: E402

TAG_RE = re.compile(r"<[^>]+>")
SESS = requests.Session()


def strip_tags(s):
    return re.sub(r"\s+", " ", TAG_RE.sub(" ", htmllib.unescape(s or ""))).strip()


def get(url, timeout=25):
    try:
        r = SESS.get(url, headers=HDRS, timeout=timeout, allow_redirects=True)
        if r.status_code == 200:
            return r.text
    except Exception:
        pass
    return ""


def write_jsonl(name, records):
    os.makedirs(RAW, exist_ok=True)
    path = os.path.join(RAW, name + ".jsonl")
    with open(path, "w") as f:
        for r in records:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    print(f"[harvest] {name}: {len(records)} entrées -> {path}", flush=True)


# ------------------------------------------------------------------ CARTV bio
# SIPAB (produitsbioquebec.info) : danse Struts en 4 temps, par type
# d'opération. Réponses en ISO-8859-1.
SIPAB_BASE = "http://www.produitsbioquebec.info"
SIPAB_ACTION = SIPAB_BASE + "/produitsbioquebec/DispatcherInterrogationGrandPublicFr.do"
SIPAB_TYPES = {
    "10": "Préparation alimentaire et transformation",
    "11": "Boissons alcoolisées",
    "20": "Production acéricole",
    "30": "Production animale",
    "40": "Production végétale",
    "50": "Récoltes sauvages et PFNL",
    "90": "Conditionnement (emballage et étiquetage)",
}
FIELD_RES = {
    "address": re.compile(r"Adresse:\s*([^<]+)"),
    "city": re.compile(r"Municipalité:\s*([^<]+)"),
    "region": re.compile(r"Région:\s*([^<]+)"),
    "postal": re.compile(r"Code postal:\s*([^<]+)"),
    "phone": re.compile(r"Tél\.:\s*([^<]+)"),
    "cert_date": re.compile(r"Date de certification:\s*([0-9-]+)"),
    "certifier": re.compile(r"Produits certifiés par:\s*([^<]+)"),
}
WEB_RE = re.compile(r'Site web:.*?<a href="([^"]+)"', re.S)
NAME_RE = re.compile(r'<font class="text1Bleu">([^<]+)</font>')


def sipab_search(code, label):
    """Une recherche complète pour un type d'opération -> liste de fiches."""
    sess = requests.Session()
    sess.get(SIPAB_BASE + "/interroGrandPublicFr.do", headers=HDRS, timeout=40)
    common = {"rechercheParProduitOuMunicipalite": "4", "langue": "Fr"}
    for action, extra in (
            ("rechercheParTypeOperation", {}),
            ("initInterrogationGrandPublicTypeOperationForm", {"codeTypesOperation": code}),
            ("rechercherDonneesProduitsCertifiesTypeOperationGrandPublic",
             {"codeTypesOperation": code, "codeRegionForChoixMultiple": "-1"})):
        data = dict(common, actionDemandee=action, **extra)
        r = sess.post(SIPAB_ACTION, data=data, headers=HDRS, timeout=180)
        time.sleep(1.0)
    # le serveur SIPAB sert de l'UTF-8 sans le déclarer (constaté : décodage
    # latin-1 = mojibake sur les champs accentués)
    html = r.content.decode("utf-8", "replace")
    out = []
    # chaque entreprise = une cellule class="celluleEntreprise"
    for cell in re.findall(r'<td class="celluleEntreprise">(.*?)</td>', html, re.S):
        nm = NAME_RE.search(cell)
        if not nm:
            continue
        rec = {"name": strip_tags(nm.group(1))[:100], "operation": label}
        for key, rx in FIELD_RES.items():
            m = rx.search(cell)
            rec[key] = strip_tags(m.group(1)).strip("\xa0 ") if m else ""
        # noms de régions SIPAB : « Saguenay--Lac-Saint-Jean » -> tiret cadratin
        # canonique du registre
        rec["region"] = rec.get("region", "").replace("--", "–")
        w = WEB_RE.search(cell)
        rec["website"] = htmllib.unescape(w.group(1)).strip() if w else ""
        out.append(rec)
    print(f"  [cartv] {label}: {len(out)} entreprises", flush=True)
    return out


def harvest_cartv_bio():
    """7 recherches (une par type d'opération), toutes régions."""
    seen, records = {}, []
    for code, label in SIPAB_TYPES.items():
        for rec in sipab_search(code, label):
            key = (rec["name"].lower(), rec.get("city", "").lower())
            if key in seen:      # même entreprise, autre type d'opération
                if label not in seen[key]["operations"]:
                    seen[key]["operations"].append(label)
                continue
            postal = (rec.pop("postal") or "").replace(" ", "")
            fiche = {
                "url": SIPAB_BASE + "/interroGrandPublicFr.do",
                "title": rec["name"],
                "h1": rec["name"],
                "websites": [rec["website"]] if rec["website"] else [],
                "socials": [],
                "postal_prefix": postal[:3] if len(postal) >= 6 else None,
                "phone": rec.get("phone") or None,
                "regions_mentioned": [rec["region"]] if rec.get("region") else [],
                "text_sample": f"{rec.get('address','')} {rec.get('city','')}",
                # champs propres à la certification (exploités par
                # scripts/backfill_certifications.py)
                "city": rec.get("city", ""),
                "certifier": rec.get("certifier", ""),
                "cert_date": rec.get("cert_date", ""),
                "operations": [label],
            }
            seen[key] = fiche
            records.append(fiche)
    write_jsonl("cartv_bio", records)


# ----------------------------------------------------------------------- CTAQ
CTAQ_LIST = "https://conseiltaq.com/ajax-search-organisation?page={p}&type=&per_page=50&lang=fr"
CTAQ_CARD = re.compile(
    r'member-item-list js-block-link.*?href="(https://conseiltaq\.com/organisation/\d+)">([^<]+)</a>'
    r'.*?<div class="desc[^"]*">\s*([^<]*).*?(?:<div class="sector[^"]*">\s*([^<]*))?</div>', re.S)


def harvest_ctaq():
    fiches = {}
    for p in range(1, 30):
        html = get(CTAQ_LIST.format(p=p), timeout=40)
        if not html:
            break
        seg = html[html.find("Tous les membres"):]
        cards = CTAQ_CARD.findall(seg)
        if not cards:
            break
        for url, name, kind, sector in cards:
            kind = htmllib.unescape(kind.strip())
            # on ne garde que les FABRICANTS : transformateurs alimentaires et
            # fabricants d'ingrédients ; les « Associés », « Fournisseur
            # produits et services », « Affilié » etc. sont des fournisseurs
            # de l'industrie, pas des fabricants de produits québécois
            if kind not in ("Transformateur", "Fournisseur d'ingrédients"):
                continue
            fiches.setdefault(url, {"name": strip_tags(name), "kind": kind,
                                    "sector": strip_tags(sector or "")})
        time.sleep(0.5)
    print(f"[ctaq] {len(fiches)} fiches membres (hors Associés) à visiter", flush=True)
    records, done = [], 0
    for url, meta in fiches.items():
        done += 1
        html = get(url, timeout=30)
        time.sleep(0.4)
        if done % 50 == 0:
            print(f"  [ctaq] {done}/{len(fiches)}", flush=True)
        if not html:
            continue
        body = html[html.find("</header>"):] if "</header>" in html else html
        webs = []
        for u in re.findall(r'href="(https?://[^"]+)"', body):
            host = (urlparse(u).netloc or "").lower()
            if not host or "conseiltaq" in host:
                continue
            if any(s in host for s in ("facebook", "instagram", "linkedin", "youtube",
                                       "twitter", "google", "w3.org", "jsdelivr",
                                       "fonts.", "gstatic", "recaptcha")):
                continue
            webs.append(u)
        text = strip_tags(body[:150000])
        postal = POSTAL_RE.search(text)
        phone = AREA_RE.search(text)
        records.append({
            "url": url,
            "title": meta["name"],
            "h1": meta["name"],
            "websites": list(dict.fromkeys(webs))[:2],
            "socials": [],
            "postal_prefix": postal.group(0)[:3] if postal else None,
            "phone": phone.group(0) if phone else None,
            "regions_mentioned": [],
            "text_sample": (meta["kind"] + " — " + meta["sector"])[:200],
            "category_hint": meta["sector"],
        })
    records = [r for r in records if r["websites"]]
    write_jsonl("ctaq", records)


# ------------------------------------------------- vendors des collectifs
COLLECTIFS = ["paperole.com", "galerieiris.com", "wachiya.com"]
STOP_TOKENS = {"inc", "enr", "ltee", "les", "the", "and", "et", "de", "du",
               "la", "le", "des", "by", "par", "studio", "atelier", "co"}


def _slug(name):
    s = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode()
    return re.sub(r"[^a-z0-9]", "", s.lower())


def _norm_name(name):
    s = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode().lower()
    return " ".join(t for t in re.findall(r"[a-z0-9]+", s) if t not in STOP_TOKENS)


def _curl(url, timeout=20):
    """GET via curl (mécanisme du projet pour éviter le 429 TLS de requests)."""
    try:
        r = subprocess.run(["curl", "-sL", "-A", HDRS["User-Agent"], "--max-time",
                            str(timeout), "--max-filesize", "400000", url],
                           capture_output=True, timeout=timeout + 10)
        return r.stdout.decode("utf-8", "replace")
    except Exception:
        return ""


def harvest_vendors_collectifs(cap=400):
    from fabrika import db as fdb
    con = fdb.connect()
    rows = con.execute(
        "SELECT vendor, COUNT(*) AS n, GROUP_CONCAT(DISTINCT store_id) AS sids "
        "FROM products WHERE active=1 AND vendor<>'' AND store_id IN (%s) "
        "GROUP BY vendor ORDER BY n DESC" % ",".join("?" * len(COLLECTIFS)),
        COLLECTIFS).fetchall()
    known_names = {_norm_name(r[0]) for r in
                   con.execute("SELECT name FROM stores") if r[0]}
    known_doms = {r[0] for r in con.execute("SELECT id FROM stores")}
    con.close()
    # candidats déjà connus du pipeline (peu importe le statut)
    cand_path = os.path.join(ENR, "candidates.jsonl")
    if os.path.exists(cand_path):
        with open(cand_path) as f:
            known_doms |= {json.loads(l)["domain"] for l in f}

    records, probes = [], 0
    for vendor, n, sids in rows:
        v = vendor.strip()
        if len(v) < 3 or len(v) > 60 or _norm_name(v) in known_names:
            continue
        slug = _slug(v)
        if not (4 <= len(slug) <= 30):
            continue
        if probes >= cap:
            break
        found = None
        for dom in (slug + ".com", slug + ".ca"):
            if dom in known_doms or BLOCK.search(dom):
                continue
            probes += 1
            html = _curl(f"https://{dom}")
            time.sleep(0.3)
            if not html:
                continue
            title_m = re.search(r"<title[^>]*>(.*?)</title>", html, re.S | re.I)
            title = _norm_name(strip_tags(title_m.group(1))[:120]) if title_m else ""
            # garde-fou anti-faux-positif : le titre de la page doit
            # recouper le nom de la marque
            vt = set(_norm_name(v).split())
            if vt and title and len(vt & set(title.split())) / len(vt) >= 0.6:
                found = dom
                break
        if found:
            records.append({
                "name": v,
                "domain": found,
                "url": f"https://{found}",
                "region_hint": "",
                "category_hint": "",
                "evidence": f"Marque vendue par les collectifs d'artisans {sids} "
                            f"({n} produits) — site officiel résolu et validé par titre",
                "query": f"vendor:{v}",
            })
    print(f"[vendors] {probes} sondes, {len(records)} sites résolus", flush=True)
    write_jsonl("vendors_collectifs", records)


# --------------------------------------------- petits répertoires sectoriels
def harvest_single_page(source, page_url, own_domain, region_hint, evidence):
    html = get(page_url, timeout=40)
    records, seen = [], set()
    for url, label in re.findall(r'<a\s[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>',
                                 html, re.S | re.I):
        url = htmllib.unescape(url)
        host = (urlparse(url).netloc or "").lower()
        if not host or own_domain in host:
            continue
        if any(s in host for s in ("facebook", "instagram", "linkedin", "youtube",
                                   "tiktok", "pinterest", "twitter", "x.com")):
            continue
        dom = norm_domain(url)
        if not dom or dom in seen or BLOCK.search(dom) or BLOCK.search(url):
            continue
        seen.add(dom)
        label = strip_tags(label)
        records.append({
            "name": label if 2 < len(label) < 80 else "",
            "domain": dom, "url": url, "region_hint": region_hint,
            "category_hint": "", "evidence": evidence, "query": page_url,
        })
    write_jsonl(source, records)


def harvest_aq_mtl():
    """Catégories montréalaises d'acheterquebecois.ca (source déjà couverte,
    pages régionales jamais moissonnées). Pagination /page/N/."""
    base = "https://acheterquebecois.ca"
    cats = ["fabrique-a-montreal", "artisanat", "vetements-2", "alimentation"]
    records, seen = [], set()
    for cat in cats:
        for page in range(1, 8):
            url = f"{base}/montreal/{cat}/" + (f"page/{page}/" if page > 1 else "")
            html = get(url, timeout=30)
            if not html:
                break
            found = 0
            for u, label in re.findall(
                    r'<a\s[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>', html, re.S | re.I):
                dom = norm_domain(u)
                if (not dom or dom in seen or "acheterquebecois" in dom
                        or BLOCK.search(dom) or BLOCK.search(u)):
                    continue
                seen.add(dom)
                found += 1
                records.append({
                    "name": strip_tags(label)[:80], "domain": dom, "url": u,
                    "region_hint": "Montréal", "category_hint": cat,
                    "evidence": "Répertorié « Fabriqué à Montréal » / catégorie "
                                "montréalaise sur acheterquebecois.ca",
                    "query": url,
                })
            time.sleep(0.4)
            if not found:
                break
    write_jsonl("acheterquebecois_mtl", records)


HARVESTERS = {
    "cartv_bio": harvest_cartv_bio,
    "ctaq": harvest_ctaq,
    "vendors_collectifs": harvest_vendors_collectifs,
    "cibim": lambda: harvest_single_page(
        "cibim", "https://cibim.org/membres/", "cibim.org", "",
        "Boulangerie artisanale membre de la CIBIM (Corporation des "
        "boulangers-pâtissiers indépendants)"),
    "canardduquebec": lambda: harvest_single_page(
        "canardduquebec", "https://canardduquebec.com/nos-producteurs/",
        "canardduquebec.com", "",
        "Producteur membre des Éleveurs de canards et d'oies du Québec"),
    "lebongoutfraisdesiles": lambda: harvest_single_page(
        "lebongoutfraisdesiles",
        "https://lebongoutfraisdesiles.com/production-et-transformation/",
        "lebongoutfraisdesiles.com", "Gaspésie–Îles-de-la-Madeleine",
        "Producteur/transformateur membre du Bon goût frais des Îles-de-la-Madeleine"),
    "acheterquebecois_mtl": harvest_aq_mtl,
}

WAVE4_SOURCES = list(HARVESTERS)

# Boutiques Shopify re-testées via curl (429 TLS python requests = gotcha
# connu) — grosses places de marché de produits québécois, connectées avec
# leur nature affichée (voir docs/CONFORMITE.md).
KNOWN_SHOPIFY = [
    {"id": "epipresto.ca", "name": "EPIPRESTO", "store_kind": "collectif",
     "origin_class": "C", "origin_confidence": 0.7,
     "origin_evidence": "Place de marché regroupant des épiceries et producteurs "
                        "locaux du Québec (re-testée via curl le 2026-08-19 ; "
                        "l'ancienne vérification avait échoué sur un 429 TLS)",
     "categories": ["epicerie"], "region": ""},
    {"id": "laboiteagrains.com", "name": "La Boite à Grains",
     "store_kind": "revendeur",
     "origin_class": "C", "origin_confidence": 0.7,
     "origin_evidence": "Épicerie santé de Gatineau (revendeur — produits "
                        "québécois et autres ; re-testée via curl le 2026-08-19)",
     "categories": ["epicerie"], "region": "Outaouais"},
]


def harvest(only=None):
    os.makedirs(RAW, exist_ok=True)
    for name, fn in HARVESTERS.items():
        if only and name not in only:
            continue
        print(f"[harvest] === {name} ===", flush=True)
        fn()


def integrate():
    """Intégration additive au registre + DB (mêmes garde-fous que la vague 3)."""
    from datetime import date
    import build_registry as br
    from fabrika import db as fdb

    for s in WAVE4_SOURCES:
        assert s in br.SOURCE_PRIORS, f"prior manquant dans build_registry: {s}"

    subprocess.run([sys.executable, os.path.join(ROOT, "scripts", "aggregate.py")],
                   check=True)

    cands = {}
    with open(os.path.join(ENR, "candidates.jsonl")) as f:
        for line in f:
            c = json.loads(line)
            cands[c["domain"]] = c

    verified_path = os.path.join(ENR, "verified.jsonl")
    verified = {}
    with open(verified_path) as f:
        for line in f:
            v = json.loads(line)
            verified[v["domain"]] = v

    reg_path = os.path.join(ROOT, "data", "stores.json")
    reg = json.load(open(reg_path))
    existing_ids = {s["id"] for s in reg["stores"]}

    new_domains = sorted(d for d in cands
                         if d not in verified and d not in existing_ids)
    print(f"[integrate] {len(new_domains)} nouveaux domaines à vérifier", flush=True)

    new_recs = []
    with cf.ThreadPoolExecutor(12) as ex:
        for i, rec in enumerate(ex.map(verify_domain, new_domains)):
            new_recs.append(rec)
            if (i + 1) % 100 == 0:
                print(f"  verify {i+1}/{len(new_domains)}", flush=True)
    with open(verified_path, "a") as f:
        for r in new_recs:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
            verified[r["domain"]] = r

    added, skipped_dup, skipped_qc, skipped_dead = [], 0, 0, 0
    for dom in new_domains:
        cand, ver = cands[dom], verified.get(dom, {})
        if not ver.get("active"):
            skipped_dead += 1
            continue
        final_dom = ver.get("final_domain") or dom
        if final_dom in existing_ids:
            skipped_dup += 1
            continue
        qc_signal = any(ver.get(k) for k in ("qc_postal", "qc_phone", "tld_quebec",
                                             "mentions_quebec", "made_in_qc_wording"))
        qc_source = any(s in br.QC_ONLY_SOURCES for s in cand.get("sources", []))
        if not qc_signal and not qc_source:
            skipped_qc += 1
            continue
        cls, conf, ev = br.classify(cand, ver)
        default_cat = next((br.SOURCE_PRIORS[s][3] for s in br.PRIORITY
                            if s in cand.get("sources", []) and br.SOURCE_PRIORS[s][3]), None)
        platform = ver.get("platform") or ""
        catalog_endpoint = ver.get("catalog_endpoint") or ""
        if platform == "wix" and not catalog_endpoint:
            catalog_endpoint = "/_api/wix-ecommerce-storefront-web/api"
        fu = urlparse(ver.get("final_url") or f"https://{final_dom}")
        store = {
            "id": final_dom,
            "name": br.clean_name(cand, ver),
            "url": f"{fu.scheme}://{fu.netloc}",
            "platform": platform,
            "catalog_endpoint": catalog_endpoint,
            "city": "",
            "region": br.pick_region(cand) or br.region_from_postal(cand, ver),
            "postal_prefix": cand.get("postal_prefix") or (ver.get("qc_postal") or "")[:3] or None,
            "phone": cand.get("phone") or ver.get("qc_phone"),
            "origin_class": cls,
            "origin_confidence": conf,
            "origin_evidence": ev,
            "categories": [default_cat] if default_cat else [],
            "socials": (cand.get("socials") or [])[:4] or ver.get("socials", []),
            "discovery_sources": cand.get("sources", []),
            "discovery_source_urls": cand.get("source_pages", [])[:5],
            "language": ver.get("language"),
            "ecommerce": bool(ver.get("has_cart") or catalog_endpoint),
            "verification_date": ver.get("checked_at") or str(date.today()),
            "status": "verified" if (conf >= 0.6 and ver.get("mentions_quebec")) else "probable",
            "enabled": bool(catalog_endpoint),
        }
        existing_ids.add(final_dom)
        added.append(store)

    # --- boutiques Shopify re-testées via curl (hors pipeline verify) ------
    for spec in KNOWN_SHOPIFY:
        if spec["id"] in existing_ids:
            continue
        probe = _curl(f"https://{spec['id']}/products.json?limit=1")
        if '"products"' not in probe[:200]:
            print(f"[integrate] {spec['id']}: /products.json injoignable — ignorée")
            continue
        store = {
            "id": spec["id"], "name": spec["name"], "url": f"https://{spec['id']}",
            "platform": "shopify", "catalog_endpoint": "/products.json",
            "city": "", "region": spec["region"], "postal_prefix": None,
            "phone": None, "origin_class": spec["origin_class"],
            "origin_confidence": spec["origin_confidence"],
            "origin_evidence": spec["origin_evidence"],
            "categories": spec["categories"], "socials": [],
            "discovery_sources": ["retest_curl_2026_08_19"],
            "discovery_source_urls": [], "language": "fr", "ecommerce": True,
            "verification_date": str(date.today()), "status": "verified",
            "enabled": True, "store_kind": spec["store_kind"],
        }
        existing_ids.add(spec["id"])
        added.append(store)

    reg["stores"].extend(added)
    reg["count"] = len(reg["stores"])
    reg["generated"] = str(date.today())
    json.dump(reg, open(reg_path, "w"), ensure_ascii=False, indent=1)

    con = fdb.connect()
    for s in added:
        fdb.upsert_store(con, s)
    con.commit(); con.close()

    enabled = [s["id"] for s in added if s["enabled"]]
    per_plat = Counter(s["platform"] or "(aucune)" for s in added)
    per_src = Counter(src for s in added for src in s["discovery_sources"])
    per_cls = Counter(s["origin_class"] for s in added)
    print(f"[integrate] boutiques ajoutées: {len(added)} | connectables (enabled): {len(enabled)}")
    print(f"[integrate] écartées — mortes/injoignables: {skipped_dead}, "
          f"dédup domaine final: {skipped_dup}, sans preuve QC: {skipped_qc}")
    print("[integrate] par plateforme:", dict(per_plat.most_common()))
    print("[integrate] par source:", dict(per_src.most_common()))
    print("[integrate] par classe:", dict(per_cls))
    with open(os.path.join(ROOT, "data", "wave4_new_enabled.txt"), "w") as f:
        f.write("\n".join(enabled) + "\n")
    if enabled:
        print("[integrate] à synchroniser: python run.py sync $(cat data/wave4_new_enabled.txt)")


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("cmd", choices=["harvest", "integrate"])
    ap.add_argument("only", nargs="*", help="sources précises (harvest)")
    args = ap.parse_args()
    harvest(args.only or None) if args.cmd == "harvest" else integrate()