#!/usr/bin/env python3
# -----------------------------------------------------------------------------
# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)
# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
# scripts/gen_connector_docs.py : documentation standardisée des connecteurs
#
#   Génère docs/connecteurs/ (INDEX.md + une fiche par MODULE connecteur +
#   fiches transverses) en croisant :
#     1. data/sources.json          — registre des sources (142 sources) ;
#     2. le code des connecteurs    — en-tête « mécanique » de chaque module,
#        backend de fetch, plafonds env (AUTOKA_*), classes exposées ;
#     3. la BD live data/autoka.db  — volumétrie, complétude par champ,
#        dernier sync et erreurs récentes (sync_log), doublons VIN, rappels
#        Transports Canada, coordonnées concessionnaires.
#
#   Rejouable à volonté (BD ouverte en lecture seule, docs régénérés) :
#       .venv/bin/python3 scripts/gen_connector_docs.py
# -----------------------------------------------------------------------------
from __future__ import annotations

import datetime
import json
import re
import sqlite3
import sys
from collections import OrderedDict
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

DB_PATH = ROOT / "data" / "autoka.db"
SOURCES_PATH = ROOT / "data" / "sources.json"
VILLES_PATH = ROOT / "data" / "villes_gps.json"
CONNECTORS_DIR = ROOT / "autoka" / "connectors"
OUT_DIR = ROOT / "docs" / "connecteurs"

# Champs de complétude (annonces actives) : libellé -> expression SQL "rempli"
FIELDS = OrderedDict([
    ("Marque", "make IS NOT NULL AND make != ''"),
    ("Modèle", "model IS NOT NULL AND model != ''"),
    ("Année", "year IS NOT NULL"),
    ("Kilométrage", "mileage_km IS NOT NULL"),
    ("Prix", "price IS NOT NULL"),
    ("Carburant", "fuel IS NOT NULL AND fuel != ''"),
    ("Boîte (transmission)", "transmission IS NOT NULL AND transmission != ''"),
    ("VIN", "vin IS NOT NULL AND vin != ''"),
    ("Photos", "images IS NOT NULL AND images NOT IN ('', '[]')"),
    ("Carfax", "carfax_url IS NOT NULL AND carfax_url != ''"),
    ("GPS (lat/lng)", "lat IS NOT NULL AND lng IS NOT NULL"),
])

# Historique des vagues d'enrichissement (source de vérité : git log)
HISTORIQUE = """\
## Historique des vagues (2026-08-18)

| Commit | Contenu |
|---|---|
| `2a35ac2` | Enrichissement connecteurs : Carfax, détails AED/LPDG, verticale moto, photos Magnetis, GPS concessionnaires |
| `bbc070b` | Vague 2 : dédup VIN inter-sources, Kijiji particuliers, rappels Transports Canada, coordonnées concessionnaires |
| `b6d5459` | Documentation standardisée des connecteurs (générateur + fiches) |
| Vague 3 | Grands portails : AutoTrader.ca/AutoHebdo (fenêtre récente), Otogo.ca (inventaire complet, VIN+GPS), CarGurus.ca (deal rating, via Scrapfly) |
"""


# --------------------------------------------------------------------------- #
#  Introspection du code
# --------------------------------------------------------------------------- #

def module_header(path: Path, marker: str) -> str:
    """En-tête « mécanique » d'un module : bloc de commentaires du haut du
    fichier, à partir de la ligne `# <marker>` (ex. connectors/x.py :) jusqu'à
    la ligne de tirets fermante. Rendu en bloc de citation Markdown."""
    lines = []
    started = False
    for raw in path.read_text(encoding="utf-8").splitlines():
        if not raw.startswith("#"):
            break
        body = raw.lstrip("#").strip()
        if not started:
            if body.startswith(marker):
                started = True
                lines.append(body)
            continue
        if set(body) <= {"-"} and len(body) > 10:      # ligne de tirets finale
            break
        lines.append(body)
    if not lines:
        return "_(pas d'en-tête trouvé)_"
    return "\n".join("> " + (l if l else "") for l in lines)


def detect_backend(src: str) -> str:
    low = src.lower()
    if "use_firecrawl" in low:
        return ("requests direct — Firecrawl par source au besoin "
                "(`use_firecrawl`, sites derrière Cloudflare)")
    if "firecrawl" in low:
        return "Firecrawl (contournement Cloudflare / rendu JS)"
    if "scrapfly" in low:
        return "Scrapfly"
    return "requests direct (aucun anti-bot)"


def detect_mechanics(src: str) -> str:
    low = src.lower()
    found = []
    for needle, label in [
        ("sitemap", "sitemap"),
        ("json-ld", "JSON-LD schema.org"),
        ("__next_f", "flux RSC Next.js"),
        ("__next_data__", "__NEXT_DATA__"),
        ("graphql", "GraphQL"),
        ("appsync", "AWS AppSync"),
        ("page-data.json", "page-data Gatsby"),
        ("rss", "flux RSS"),
        ("microdata", "microdata schema.org"),
        ("itemprop", "microdata schema.org"),
        ("beautifulsoup", "parsing HTML"),
        ("data-class=", "blob JSON data-class"),
        ("/api/", "API JSON interne"),
        (".json", "endpoint JSON"),
    ]:
        if needle in low and label not in found:
            found.append(label)
    return ", ".join(found) if found else "—"


def detect_caps(src: str) -> str:
    caps = sorted(set(re.findall(r"AUTOKA_[A-Z_]+", src)))
    caps = [c for c in caps if c not in ("AUTOKA_",)]
    if "detail_cache" in src or "cache_get" in src or "cached" in src.lower():
        caps.append("cache détail BD (detail_cache)")
    return ", ".join(caps) if caps else "aucun"


def module_classes(mod_name: str) -> list[str]:
    """Classes concrètes (source_id non vide) déclarées par le module."""
    import autoka.connectors as reg
    out = []
    for sid, cls in reg.CONNECTORS.items():
        if cls.__module__.rsplit(".", 1)[-1] == mod_name:
            out.append(f"`{cls.__name__}` → `{sid}`")
    return out


# --------------------------------------------------------------------------- #
#  BD live
# --------------------------------------------------------------------------- #

def q1(db, sql, args=()):
    return db.execute(sql, args).fetchone()


def completeness(db, source_ids: list[str]) -> tuple[int, OrderedDict]:
    ph = ",".join("?" * len(source_ids))
    parts = ", ".join(f"SUM(CASE WHEN {expr} THEN 1 ELSE 0 END)"
                      for expr in FIELDS.values())
    row = q1(db, f"SELECT COUNT(*), {parts} FROM vehicles "
                 f"WHERE active = 1 AND source IN ({ph})", source_ids)
    n = row[0] or 0
    out = OrderedDict()
    for (label, _), filled in zip(FIELDS.items(), row[1:]):
        out[label] = (100.0 * (filled or 0) / n) if n else 0.0
    return n, out


def source_stats(db, sid: str) -> dict:
    tot, act, dups = q1(db, "SELECT COUNT(*), SUM(active), "
                            "SUM(dup_of IS NOT NULL) FROM vehicles "
                            "WHERE source = ?", (sid,))
    last = q1(db, "SELECT ts, ok, found, added, updated, removed, message "
                  "FROM sync_log WHERE source = ? ORDER BY ts DESC LIMIT 1",
              (sid,))
    errs = q1(db, "SELECT COUNT(*) FROM (SELECT ok FROM sync_log "
                  "WHERE source = ? ORDER BY ts DESC LIMIT 10) WHERE ok = 0",
              (sid,))[0]
    return {"total": tot or 0, "active": act or 0, "dups": dups or 0,
            "last": last, "errs10": errs}


def fmt_ts(ts) -> str:
    if not ts:
        return "—"
    return datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M")


def sync_cell(st: dict) -> tuple[str, str]:
    last = st["last"]
    if not last:
        return "jamais", "—"
    ts, ok, found, *_ = last
    flag = "OK" if ok else "ERREUR"
    return fmt_ts(ts), f"{flag} ({found or 0} trouvés)"


def recent_errors(db, source_ids: list[str], limit=8) -> list[tuple]:
    ph = ",".join("?" * len(source_ids))
    return db.execute(
        f"SELECT source, ts, message FROM sync_log "
        f"WHERE ok = 0 AND source IN ({ph}) ORDER BY ts DESC LIMIT ?",
        (*source_ids, limit)).fetchall()


# --------------------------------------------------------------------------- #
#  Rendu des fiches
# --------------------------------------------------------------------------- #

def render_completeness(n: int, comp: OrderedDict) -> list[str]:
    n_fmt = f"{n:,}".replace(",", " ")
    out = [f"## Complétude des champs (annonces actives, N = {n_fmt})",
           "", "| Champ | % rempli |", "|---|---|"]
    for label, pct in comp.items():
        out.append(f"| {label} | {pct:.1f} % |")
    out.append("")
    return out


def render_module_fiche(db, mod: str, members: list[dict]) -> str:
    path = CONNECTORS_DIR / f"{mod}.py"
    src = path.read_text(encoding="utf-8")
    sids = [s["id"] for s in members]
    n, comp = completeness(db, sids)
    active_total = 0

    rows = []
    for s in sorted(members, key=lambda x: x["id"]):
        st = source_stats(db, s["id"])
        active_total += st["active"]
        when, status = sync_cell(st)
        rows.append(f"| `{s['id']}` | {s.get('name', '')} | "
                    f"{s.get('city', '') or '—'} | {s.get('region', '') or '—'} | "
                    f"{st['active']} | {st['dups']} | {when} | {status} |")

    from collections import Counter
    plat_counts = Counter(s.get("platform", "").split(" (")[0].strip()
                          for s in members if s.get("platform"))
    plat = plat_counts.most_common(1)[0][0] if plat_counts else mod
    lines = [
        f"# Module `{mod}` — {plat}",
        "",
        f"_Généré automatiquement par `scripts/gen_connector_docs.py` — ne pas éditer à la main._",
        "",
        "## Vue d'ensemble",
        "",
        f"- **Fichier** : `autoka/connectors/{mod}.py`",
        f"- **Sources membres** : {len(members)}",
        f"- **Annonces actives (BD)** : {active_total}",
        f"- **Backend de fetch** : {detect_backend(src)}",
        f"- **Mécanique détectée** : {detect_mechanics(src)}",
        f"- **Plafonds / cache** : {detect_caps(src)}",
    ]
    classes = module_classes(mod)
    if classes:
        lines.append(f"- **Classes** : {' ; '.join(classes[:6])}"
                     + (f" … (+{len(classes) - 6})" if len(classes) > 6 else ""))
    lines += ["", "## Mécanique (en-tête du module)", "",
              module_header(path, f"connectors/{mod}.py"), ""]
    lines += render_completeness(n, comp)
    lines += ["## Sources membres (BD live)", "",
              "| Source | Nom | Ville | Région | Actives | Masquées (dup VIN) | Dernier sync | Statut |",
              "|---|---|---|---|---|---|---|---|"] + rows + [""]

    errs = recent_errors(db, sids)
    if errs:
        lines += ["## Erreurs de synchronisation récentes (sync_log, ok = 0)", "",
                  "| Source | Quand | Message |", "|---|---|---|"]
        for sid, ts, msg in errs:
            msg = (msg or "").replace("|", "\\|")[:160]
            lines.append(f"| `{sid}` | {fmt_ts(ts)} | {msg} |")
        lines.append("")
    else:
        lines += ["## Erreurs de synchronisation récentes", "",
                  "Aucune erreur dans le sync_log pour les sources de ce module.", ""]
    return "\n".join(lines)


def render_transverse_dedup(db) -> str:
    total, masked = q1(db, "SELECT COUNT(*), SUM(dup_of IS NOT NULL) FROM vehicles")
    groups = q1(db, "SELECT COUNT(DISTINCT dup_of) FROM vehicles "
                    "WHERE dup_of IS NOT NULL")[0]
    top = db.execute(
        "SELECT source, COUNT(*) c FROM vehicles WHERE dup_of IS NOT NULL "
        "GROUP BY source ORDER BY c DESC LIMIT 12").fetchall()
    lines = ["# Transverse — Dédoublonnage VIN inter-sources", "",
             "_Généré automatiquement par `scripts/gen_connector_docs.py`._", "",
             "## Mécanique (`autoka/dedup.py`)", "",
             module_header(ROOT / "autoka" / "dedup.py", "dedup.py"), "",
             "## État live", "",
             f"- **Annonces en base** : {total}",
             f"- **Annonces masquées (`dup_of` non nul)** : {masked}",
             f"- **Annonces canoniques avec doublons** : {groups}",
             "- **Autorité** : concessionnaire direct > portail/regroupeur > petites annonces ;"
             " puis complétude de la fiche, puis ancienneté (`first_seen`).",
             "- Les listes filtrent `dup_of IS NULL` ; les fiches détail restent accessibles par uid.",
             "", "## Sources les plus masquées", "",
             "| Source | Annonces masquées |", "|---|---|"]
    lines += [f"| `{s}` | {c} |" for s, c in top]
    lines += ["", "Recalcul one-shot : `python3 -m autoka.dedup` (idempotent,"
              " relancé à la fin de chaque cycle d'ingestion).", ""]
    return "\n".join(lines)


def render_transverse_recalls(db) -> str:
    rows, nums = q1(db, "SELECT COUNT(*), COUNT(DISTINCT recall_number) FROM recalls")
    y0, y1 = q1(db, "SELECT MIN(year), MAX(year) FROM recalls WHERE year > 1900")
    cats = db.execute("SELECT category, COUNT(DISTINCT recall_number) c "
                      "FROM recalls GROUP BY category ORDER BY c DESC LIMIT 8").fetchall()
    lines = ["# Transverse — Rappels de sécurité Transports Canada", "",
             "_Généré automatiquement par `scripts/gen_connector_docs.py`._", "",
             "## Mécanique (`autoka/recalls.py`)", "",
             module_header(ROOT / "autoka" / "recalls.py", "recalls.py"), "",
             "## État live", "",
             f"- **Rappels distincts (`recall_number`)** : {nums}",
             f"- **Lignes en base (rappel × marque × modèle × année)** : {rows}",
             f"- **Années-modèles couvertes** : {y0}–{y1}",
             "- **Source** : CSV mensuel complet des données ouvertes TC"
             " (`opendatatc.tc.canada.ca/vrdb_full_monthly.csv`, sans clé API).",
             "", "## Répartition par catégorie TC", "",
             "| Catégorie | Rappels distincts |", "|---|---|"]
    lines += [f"| {c or '—'} | {n} |" for c, n in cats]
    lines += ["", "Peuplement one-shot / mensuel : `python3 -m autoka.recalls"
              " [--limit-years N]`.", ""]
    return "\n".join(lines)


def render_transverse_dealers(db) -> str:
    tot, ph, ad, gps = q1(db, "SELECT COUNT(*), "
                              "SUM(phone IS NOT NULL AND phone != ''), "
                              "SUM(address IS NOT NULL AND address != ''), "
                              "SUM(lat IS NOT NULL AND lng IS NOT NULL) FROM dealers")
    lines = ["# Transverse — Coordonnées des concessionnaires", "",
             "_Généré automatiquement par `scripts/gen_connector_docs.py`._", "",
             "## Mécanique (`autoka/dealers.py`)", "",
             module_header(ROOT / "autoka" / "dealers.py", "dealers.py"), "",
             "## État live (table `dealers`)", "",
             f"- **Concessionnaires en base** : {tot}",
             f"- **Avec téléphone** : {ph} ({100.0 * ph / tot:.0f} %)",
             f"- **Avec adresse** : {ad} ({100.0 * ad / tot:.0f} %)",
             f"- **Avec GPS (lat/lng)** : {gps} ({100.0 * gps / tot:.0f} %)",
             "",
             "Crawl « une page par site » (accueil : JSON-LD AutoDealer/LocalBusiness,"
             " replis regex `tel:` + code postal canadien) ; GPS de repli = centre-ville"
             " de la source (`data/villes_gps.json`). One-shot :"
             " `python3 -m autoka.dealers [--refresh]`.", ""]
    return "\n".join(lines)


def render_transverse_villes() -> str:
    villes = json.loads(VILLES_PATH.read_text(encoding="utf-8"))
    sample = ", ".join(sorted(villes)[:12])
    lines = ["# Transverse — Référentiel GPS des villes (`data/villes_gps.json`)", "",
             "_Généré automatiquement par `scripts/gen_connector_docs.py`._", "",
             f"- **Villes référencées** : {len(villes)}",
             "- **Format** : `\"ville en minuscules sans accents\" → [lat, lng]`"
             " (centres-villes du Québec).",
             "- **Usages** : GPS de repli des concessionnaires (`autoka/dealers.py`)"
             " et géolocalisation des annonces par la ville de leur source"
             " (les annonces Kijiji portent leur propre lat/lng exact).",
             "", f"Extrait : {sample}…", ""]
    return "\n".join(lines)


# --------------------------------------------------------------------------- #
#  INDEX
# --------------------------------------------------------------------------- #

def render_index(db, groups: OrderedDict, sources: list[dict]) -> str:
    total, active, masked = q1(db, "SELECT COUNT(*), SUM(active), "
                                   "SUM(dup_of IS NOT NULL) FROM vehicles")
    kinds = dict(db.execute("SELECT kind, COUNT(*) FROM vehicles "
                            "WHERE active = 1 GROUP BY kind").fetchall())
    recall_n = q1(db, "SELECT COUNT(DISTINCT recall_number) FROM recalls")[0]
    dealers_n = q1(db, "SELECT COUNT(*) FROM dealers")[0]
    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")

    lines = ["# Auto-Ka — Documentation des connecteurs", "",
             f"_Générée le {now} par `scripts/gen_connector_docs.py`"
             " (rejouable : `.venv/bin/python3 scripts/gen_connector_docs.py`)._", "",
             "## Vue d'ensemble", "",
             f"- **Sources** : {len(sources)} (registre `data/sources.json`)",
             f"- **Modules connecteurs** : {len(groups)} (`autoka/connectors/`)",
             f"- **Annonces en base** : {total} — dont {active} actives"
             f" ({kinds.get('auto', 0)} autos, {kinds.get('moto', 0)} motos,"
             f" {kinds.get('scooter', 0)} scooters)",
             f"- **Doublons VIN masqués** : {masked} (voir [dédup VIN](transverse-dedup-vin.md))",
             f"- **Rappels Transports Canada** : {recall_n} rappels distincts"
             " (voir [rappels TC](transverse-rappels-tc.md))",
             f"- **Concessionnaires géolocalisés** : {dealers_n}"
             " (voir [dealers](transverse-dealers.md))",
             "", "## Fiches par module connecteur", "",
             "| Module | Fiche | Sources | Annonces actives | Backend |",
             "|---|---|---|---|---|"]
    for mod, members in groups.items():
        sids = [s["id"] for s in members]
        ph_ = ",".join("?" * len(sids))
        act = q1(db, f"SELECT COALESCE(SUM(active), 0) FROM vehicles "
                     f"WHERE source IN ({ph_})", sids)[0]
        src = (CONNECTORS_DIR / f"{mod}.py").read_text(encoding="utf-8")
        backend = detect_backend(src).split(" (")[0]
        lines.append(f"| `{mod}` | [{mod}.md]({mod}.md) | {len(members)} | {act} | {backend} |")

    lines += ["", "## Fiches transverses", "",
              "| Sujet | Fiche |", "|---|---|",
              "| Dédoublonnage VIN inter-sources | [transverse-dedup-vin.md](transverse-dedup-vin.md) |",
              "| Rappels Transports Canada | [transverse-rappels-tc.md](transverse-rappels-tc.md) |",
              "| Coordonnées des concessionnaires | [transverse-dealers.md](transverse-dealers.md) |",
              "| Référentiel GPS des villes | [transverse-villes-gps.md](transverse-villes-gps.md) |",
              "", HISTORIQUE]
    return "\n".join(lines)


# --------------------------------------------------------------------------- #

def main() -> None:
    sources = json.loads(SOURCES_PATH.read_text(encoding="utf-8"))["sources"]
    groups: OrderedDict[str, list] = OrderedDict()
    for s in sources:
        groups.setdefault(s["connector"], []).append(s)
    # modules multi-sources d'abord (par taille), uniques ensuite (alpha)
    groups = OrderedDict(sorted(groups.items(),
                                key=lambda kv: (-len(kv[1]), kv[0])))

    db = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True)
    OUT_DIR.mkdir(parents=True, exist_ok=True)

    for mod, members in groups.items():
        (OUT_DIR / f"{mod}.md").write_text(
            render_module_fiche(db, mod, members), encoding="utf-8")
        print(f"  fiche {mod}.md ({len(members)} source(s))")

    (OUT_DIR / "transverse-dedup-vin.md").write_text(render_transverse_dedup(db), encoding="utf-8")
    (OUT_DIR / "transverse-rappels-tc.md").write_text(render_transverse_recalls(db), encoding="utf-8")
    (OUT_DIR / "transverse-dealers.md").write_text(render_transverse_dealers(db), encoding="utf-8")
    (OUT_DIR / "transverse-villes-gps.md").write_text(render_transverse_villes(), encoding="utf-8")
    (OUT_DIR / "INDEX.md").write_text(render_index(db, groups, sources), encoding="utf-8")
    print(f"OK — {len(groups)} fiches module + 4 transverses + INDEX.md dans {OUT_DIR}")


if __name__ == "__main__":
    main()