#!/usr/bin/env python3
# ==============================================================================
# Author: Simon-Pierre Boucher <contact@spboucher.ai>
# File:   scripts/gen_connector_docs.py
# Desc:   Documentation STANDARDISÉE des connecteurs Créa-Ka — génère
#         docs/connecteurs/INDEX.md + une fiche docs/connecteurs/<id>.md par
#         source du registre, de façon 100 % programmatique et rejouable :
#           1. registre data/sources.json (famille, palier, accès, cadence,
#              signal d'identité, notes, statut) ;
#           2. introspection STATIQUE (ast) de creaka/connectors/*.py :
#              classe, kind (discovery/enrichment), backend, endpoints,
#              pagination, constantes de budget — sans exécuter le code ;
#           3. BD live data/creaka.db : créateurs découverts/enrichis par
#              source (doc JSON), complétude des champs, comptes par
#              plateforme, dernier sync, cadence observée, alertes récentes.
#         Usage : python3 scripts/gen_connector_docs.py   (racine du projet)
# ==============================================================================
from __future__ import annotations

import ast
import json
import re
import sqlite3
import statistics
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
CONN_DIR = ROOT / "creaka" / "connectors"
DOCS_DIR = ROOT / "docs" / "connecteurs"
DB_PATH = ROOT / "data" / "creaka.db"
SOURCES_JSON = ROOT / "data" / "sources.json"

SKIP_MODULES = {"__init__", "base"}
URL_RE = re.compile(r"https?://[^\s\"'\\)>,;]+")
DATE_RE = re.compile(r"\d{4}-\d{2}-\d{2}")
INFRA_HOSTS = ("api.firecrawl.dev", "api.scrapfly.io", "crea-ka.com")

# plateformes de la table `accounts` alimentées par chaque source d'enrichissement
SOURCE_PLATFORMS = {
    "instagram-profil": ("instagram",),
    "tiktok-profil": ("tiktok",),
    "youtube": ("youtube",),
    "youtube-recherche": ("youtube",),
    "twitch": ("twitch",),
    "x-profil": ("x",),
    "balados-rss": ("podcast",),
    "balados-itunes": ("podcast",),
    "palmares-apple": ("podcast",),
    "podcastindex": ("podcast",),
    "onlyqueb": ("onlyfans",),
}

# champs du doc créateur documentés (complétude par source de découverte)
DOC_FIELDS = [
    ("display_name", "Nom public"),
    ("bio", "Bio"),
    ("region", "Région"),
    ("city", "Ville"),
    ("niches", "Niches (liste)"),
    ("languages", "Langues (liste)"),
    ("creator_type", "Type de créateur"),
    ("primary_platform", "Plateforme principale"),
    ("platforms", "Comptes sociaux (liste)"),
    ("avatar_url", "Avatar"),
    ("total_reach", "Portée totale (abonnés cumulés)"),
    ("link_in_bio_url", "Lien-en-bio"),
]

ACC_FIELDS = [
    ("handle", "Handle", "handle IS NOT NULL AND handle != ''"),
    ("url", "URL du profil", "url IS NOT NULL AND url != ''"),
    ("followers", "Abonnés", "followers IS NOT NULL"),
    ("verified", "Badge vérifié", "verified IS NOT NULL"),
    ("confidence", "Confiance d'identité", "confidence IS NOT NULL"),
    ("signal", "Signal d'identité", "signal IS NOT NULL AND signal != ''"),
    ("metrics", "Métriques détaillées (JSON)",
     "metrics IS NOT NULL AND length(metrics) > 4"),
    ("last_checked", "Dernière vérification",
     "last_checked IS NOT NULL AND last_checked != ''"),
]


def esc(s, limit: int = 100) -> str:
    s = str(s).replace("\\", "\\\\").replace("|", "\\|")
    s = re.sub(r"\s+", " ", s).strip()
    return s[: limit - 1] + "…" if len(s) > limit else s


def pct(n, d) -> str:
    return f"{100.0 * (n or 0) / d:.0f} %" if d else "—"


def parse_iso(ts: str) -> float | None:
    try:
        return datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp()
    except (ValueError, AttributeError):
        return None


def fmt_secs(sec: float) -> str:
    if sec < 5400:
        return f"≈ {sec / 60:.0f} min"
    if sec < 129600:
        return f"≈ {sec / 3600:.1f} h"
    return f"≈ {sec / 86400:.1f} j"


def filled(v) -> bool:
    return bool(v)


# -- introspection statique ------------------------------------------------------

def banner_description(text: str) -> str:
    lines, started = [], False
    for raw in text.splitlines():
        if not raw.startswith("#"):
            if started:
                break
            continue
        body = raw.lstrip("#").strip()
        if set(body) <= {"-", "="}:
            continue
        if not started:
            if body.startswith("Desc:"):
                started = True
                lines.append(body[len("Desc:"):].strip())
            continue
        if re.match(r"^(Author|File):", body):
            break
        lines.append(body)
    return " ".join(l for l in lines if l).strip()


def introspect_module(path: Path) -> list[dict]:
    text = path.read_text(encoding="utf-8")
    try:
        tree = ast.parse(text)
    except SyntaxError:
        return []
    constants: dict = {}
    for node in tree.body:
        if isinstance(node, ast.Assign) and len(node.targets) == 1 \
                and isinstance(node.targets[0], ast.Name) \
                and node.targets[0].id.isupper():
            try:
                constants[node.targets[0].id] = ast.literal_eval(node.value)
            except (ValueError, TypeError, SyntaxError):
                seg = ast.get_source_segment(text, node.value) or ""
                urls = URL_RE.findall(seg)
                constants[node.targets[0].id] = urls if len(urls) > 1 else \
                    (urls[0] if urls else None)
    urls_all = []
    for u in URL_RE.findall(text):
        u = u.rstrip('".')
        host = u.split("//", 1)[-1].split("/", 1)[0]
        if "." not in host:      # fragment de f-string, pas une vraie URL
            continue
        if not any(h in u for h in INFRA_HOSTS) and u not in urls_all:
            urls_all.append(u)
    endpoint = None
    for name in ("BASE", "BASE_URL", "API", "API_URL", "API_BASE", "SPARQL",
                 "ENDPOINT", "ROOT", "URL", "SEARCH_URL", "LOOKUP_URL"):
        v = constants.get(name)
        if isinstance(v, str) and v.startswith("http"):
            endpoint = v
            break
    if not endpoint and urls_all:
        endpoint = urls_all[0]
    budgets = {k: v for k, v in constants.items()
               if isinstance(v, (int, float)) and not isinstance(v, bool)
               and re.search(r"MAX|CAP|LIMIT|BUDGET|TTL|PER_PAGE|PAGES|DELAY|BATCH",
                             k)}
    detailed, family = [], "direct"
    if re.search(r"self\.(get|post)\(|requests\.(get|post)\(", text):
        detailed.append("requests direct (session UA CreaKaBot, throttling poli)")
    if ".scrapfly(" in text:
        detailed.append("Scrapfly (asp + render_js — contournement anti-bot)")
        family = "Scrapfly"
    if "get_rendered(" in text:
        detailed.append("Firecrawl (HTML rendu, JavaScript exécuté)")
        family = "Firecrawl" if family == "direct" else family
    if not detailed:
        detailed.append("requests direct")
    low = text.lower()
    if "sparql" in low:
        flavor = "API SPARQL publique (Wikidata)"
    elif "graphql" in low:
        flavor = "API GraphQL"
    elif "itunes.apple.com" in low:
        flavor = "API iTunes Search/Lookup (publique)"
    elif "rss" in low and "feedparser" in low or "<rss" in low:
        flavor = "flux RSS publics"
    elif "sitemap" in low:
        flavor = "sitemap XML + JSON-LD des pages profil (SSR)"
    elif "json-ld" in low or "jsonld" in low or "profilepage" in low:
        flavor = "JSON-LD des pages profil publiques"
    elif re.search(r"\.json\(\)", text) and re.search(r"api[./_]", low):
        flavor = "API JSON"
    elif "dataset" in low or "seed" in low and "csv" in low:
        flavor = "dataset local compilé (data/seed)"
    else:
        flavor = "pages HTML publiques"
    hits = []
    if re.search(r"[?&]page=|[\"']page[\"']\s*[:=]", low):
        hits.append("pagination par numéro de page")
    if re.search(r"[?&]offset=|[\"']offset[\"']", low):
        hits.append("pagination par offset")
    if "cursor" in low or "continuation" in low:
        hits.append("curseur / continuation")
    if not hits:
        hits.append("réponse unique (pas de pagination)")
    base = {"module": path.stem, "path": f"creaka/connectors/{path.stem}.py",
            "banner": banner_description(text), "endpoint": endpoint,
            "urls": urls_all[:5], "budgets": budgets, "backends": detailed,
            "backend_family": family, "flavor": flavor,
            "pagination": " ; ".join(hits)}
    entries = []
    for node in tree.body:
        if not isinstance(node, ast.ClassDef):
            continue
        bases = {getattr(b, "id", getattr(b, "attr", "")) for b in node.bases}
        if "BaseConnector" not in bases:
            continue
        attrs = {"request_delay": 0.8, "timeout": 30, "kind": "discovery",
                 "source_id": ""}
        for sub in node.body:
            if isinstance(sub, ast.Assign) and len(sub.targets) == 1 \
                    and isinstance(sub.targets[0], ast.Name):
                try:
                    attrs[sub.targets[0].id] = ast.literal_eval(sub.value)
                except (ValueError, TypeError, SyntaxError):
                    pass
        if attrs["source_id"]:
            entries.append({**base, "class": node.name,
                            "class_doc": ast.get_docstring(node) or "",
                            **attrs})
    return entries


# -- BD live ----------------------------------------------------------------------

def load_creators(con) -> list[dict]:
    out = []
    for (doc,) in con.execute("SELECT doc FROM creators"):
        try:
            out.append(json.loads(doc))
        except ValueError:
            pass
    return out


def db_stats(con, sid: str, creators: list[dict]) -> dict:
    mine = [c for c in creators if c.get("source") == sid]
    touched = sum(1 for c in creators
                  if sid in (c.get("source_ids") or []) or c.get("source") == sid)
    doc_fill = {f: sum(1 for c in mine if filled(c.get(f)))
                for f, _l in DOC_FIELDS}
    sample = max(mine, key=lambda c: c.get("total_reach") or 0) if mine else None

    acc = None
    acc_sample = None
    platforms = SOURCE_PLATFORMS.get(sid)
    if platforms:
        ph = ",".join("?" * len(platforms))
        parts = ", ".join(
            f"sum(CASE WHEN {cond} THEN 1 ELSE 0 END) AS f_{col}"
            for col, _l, cond in ACC_FIELDS)
        acc = con.execute(
            f"SELECT count(*) AS n, avg(confidence) AS conf, "
            f"max(last_checked) AS checked, {parts} FROM accounts "
            f"WHERE platform IN ({ph})", platforms).fetchone()
        acc_sample = con.execute(
            f"SELECT * FROM accounts WHERE platform IN ({ph}) AND followers IS "
            f"NOT NULL ORDER BY followers DESC LIMIT 1", platforms).fetchone()

    runs = con.execute(
        "SELECT ts, creators, accounts, avg_confidence, added, updated, "
        "errors, seconds, alert FROM sync_log WHERE source=? "
        "ORDER BY ts DESC LIMIT 60", (sid,)).fetchall()
    ok_ts = sorted(t for t in (parse_iso(r["ts"]) for r in runs) if t)
    cadence = None
    if len(ok_ts) >= 3:
        deltas = [b - a for a, b in zip(ok_ts, ok_ts[1:]) if b - a > 60]
        if deltas:
            cadence = statistics.median(deltas)
    alerts = [r for r in runs if (r["errors"] or 0) > 0 or r["alert"]][:5]
    return {"mine": len(mine), "touched": touched, "doc_fill": doc_fill,
            "sample": sample, "acc": acc, "acc_sample": acc_sample,
            "platforms": platforms, "runs": runs, "cadence": cadence,
            "last": runs[0] if runs else None, "alerts": alerts}


# -- rendu ---------------------------------------------------------------------------

def render_fiche(reg: dict, entry: dict | None, st: dict, now: str) -> str:
    sid = reg["id"]
    etat = reg.get("statut", "actif")
    fam = reg.get("famille", "—")
    out = [f"# `{sid}` — connecteur {fam} (palier {reg.get('palier', '—')})", "",
           f"_Fiche générée automatiquement par "
           f"`scripts/gen_connector_docs.py` le {now} — ne pas éditer à la "
           f"main, régénérer._", ""]
    kind = entry.get("kind") if entry else fam
    out.append(f"**État : {esc(etat, 80)}** · Famille : {fam} ({kind}) · "
               f"Backend : {entry['backend_family'] if entry else '—'} · "
               f"Créateurs découverts : {st['mine']} · touchés : "
               f"{st['touched']}")
    out.append("")

    out.append("## Description de la source")
    out.append("")
    if entry and entry["banner"]:
        out.append(entry["banner"])
        out.append("")
    out.append(f"- **Notes (registre)** : {reg.get('notes', '—')}")
    out.append(f"- **Signal d'identité** : {reg.get('signal_identite', '—')} "
               f"(colonne `accounts.signal` / `confidence`)")
    if entry:
        out.append(f"- **Module** : `{entry['path']}` — classe "
                   f"`{entry['class']}` (`kind = \"{entry['kind']}\"`)")
    else:
        out.append("- **Module** : introuvable (vérifier le registre)")
    out.append("")

    out.append("## Accès")
    out.append("")
    out.append(f"- **Accès (registre)** : {reg.get('acces', '—')}")
    if entry:
        out.append(f"- **Type d'accès (code)** : {entry['flavor']}")
        out.append(f"- **Endpoint de base** : {entry['endpoint'] or '—'}")
        if entry["urls"]:
            out.append("- **URLs du module** : " + " · ".join(entry["urls"][:4]))
        out.append(f"- **Pagination** : {entry['pagination']}")
        out.append(f"- **Backend anti-bot / rendu** : "
                   f"{' ; '.join(entry['backends'])}")
        out.append(f"- **Politesse** : {entry['request_delay']} s entre "
                   f"requêtes, timeout {entry['timeout']} s, UA "
                   f"`CreaKaBot/1.0 (+https://www.crea-ka.com/bot)`")
        needs_key = "clé" in (reg.get("acces") or "").lower() \
            or "clé" in str(etat).lower()
        auth = ("clé(s) API requise(s) — voir statut/registre" if needs_key
                else "aucune — contenu public / API anonyme")
        out.append(f"- **Authentification** : {auth}")
    out.append("")

    out.append("## Champs récupérés → schéma cible")
    out.append("")
    if st["mine"]:
        out.append(f"Source de **découverte** : produit des fiches `Creator` "
                   f"(tables `creators` + `accounts`). Complétude mesurée sur "
                   f"les {st['mine']} créateurs découverts par cette source "
                   f"(champ `doc.source`) ; exemple tiré d'une fiche réelle.")
        out.append("")
        out.append("| Champ du doc créateur | Contenu | Renseigné | Exemple réel |")
        out.append("|---|---|---|---|")
        s = st["sample"] or {}
        for f, label in DOC_FIELDS:
            v = s.get(f)
            if f == "platforms" and v:
                ex = esc(", ".join(f"{p.get('platform')}:@{p.get('handle')}"
                                   for p in v[:3]), 90)
            elif isinstance(v, list):
                ex = esc(", ".join(map(str, v[:4])), 90)
            else:
                ex = esc(v, 90) if filled(v) else "—"
            out.append(f"| `{f}` | {label} | "
                       f"{pct(st['doc_fill'][f], st['mine'])} | {ex} |")
        out.append("")
    if st["acc"] is not None and st["acc"]["n"]:
        a = st["acc"]
        plats = ", ".join(st["platforms"])
        out.append(f"Cible d'**enrichissement** : colonnes de la table "
                   f"`accounts` pour la/les plateforme(s) `{plats}` "
                   f"({a['n']} comptes en BD, confiance moyenne "
                   f"{a['conf']:.2f}).")
        out.append("")
        out.append("| Colonne `accounts` | Contenu | Renseignée | Exemple réel |")
        out.append("|---|---|---|---|")
        sm = st["acc_sample"]
        for col, label, _c in ACC_FIELDS:
            ex = esc(sm[col], 90) if sm is not None and sm[col] not in (
                None, "") else "—"
            out.append(f"| `{col}` | {label} | {pct(a[f'f_{col}'], a['n'])} "
                       f"| {ex} |")
        out.append("")
    if not st["mine"] and (st["acc"] is None or not st["acc"]["n"]):
        out.append("Aucune donnée attribuable à cette source en BD pour "
                   "l'instant (clés manquantes ou source en attente) — schéma "
                   "cible : `creators` + `accounts`.")
        out.append("")

    out.append("## Fréquence & budget")
    out.append("")
    out.append(f"- **Cadence déclarée (registre)** : {reg.get('cadence', '—')}")
    cad = fmt_secs(st["cadence"]) if st["cadence"] else "—"
    out.append(f"- **Cadence observée** (médiane sync_log) : {cad}")
    last = st["last"]
    if last:
        out.append(f"- **Dernier passage** : {esc(last['ts'], 20)} — "
                   f"{last['creators'] or 0} créateurs, {last['accounts'] or 0} "
                   f"comptes, +{last['added'] or 0} / ~{last['updated'] or 0}, "
                   f"{last['errors'] or 0} erreur(s), "
                   f"{(last['seconds'] or 0):.0f} s")
    if entry and entry["budgets"]:
        caps = ", ".join(f"`{k}` = {v}" for k, v in sorted(entry["budgets"].items()))
        out.append(f"- **Caps / budgets du module** : {caps}")
    if entry:
        out.append(f"- **Throttling** : {entry['request_delay']} s entre "
                   f"requêtes (`request_delay`)")
    out.append("")

    out.append("## Volumétrie & complétude")
    out.append("")
    out.append(f"- **Créateurs découverts par la source** (`doc.source`) : "
               f"{st['mine']} · **fiches touchées** (`source_ids`) : "
               f"{st['touched']}")
    if st["mine"]:
        out.append(f"- **Complétude clé (découverts)** : niches "
                   f"{pct(st['doc_fill']['niches'], st['mine'])} · avatar "
                   f"{pct(st['doc_fill']['avatar_url'], st['mine'])} · portée "
                   f"{pct(st['doc_fill']['total_reach'], st['mine'])} · région "
                   f"{pct(st['doc_fill']['region'], st['mine'])}")
    if st["acc"] is not None and st["acc"]["n"]:
        a = st["acc"]
        out.append(f"- **Comptes `{', '.join(st['platforms'])}` en BD** : "
                   f"{a['n']} — abonnés {pct(a['f_followers'], a['n'])} · "
                   f"métriques {pct(a['f_metrics'], a['n'])} · dernière "
                   f"vérification {esc(a['checked'] or '—', 20)}")
    out.append(f"- **Runs journalisés (60 derniers)** : {len(st['runs'])}, "
               f"dont {sum(1 for r in st['runs'] if (r['errors'] or 0) > 0)} "
               f"avec erreurs")
    out.append("")

    out.append("## Erreurs connues & dépannage")
    out.append("")
    if st["alerts"]:
        out.append("| Passage | Erreurs | Alerte (sync_log) |")
        out.append("|---|---|---|")
        for r in st["alerts"]:
            out.append(f"| {esc(r['ts'], 20)} | {r['errors'] or 0} | "
                       f"{esc(r['alert'] or '—', 160)} |")
        out.append("")
    else:
        out.append("Aucune erreur ni alerte dans les 60 derniers runs "
                   "journalisés.")
        out.append("")
    if str(etat) != "actif":
        out.append(f"**Statut du registre** : {etat}")
        out.append("")
    out.append(f"Rejouer la source seule : `python3 run.py sync {sid}` · "
               f"vérifier `sync_log` (`SELECT * FROM sync_log WHERE "
               f"source='{sid}' ORDER BY ts DESC LIMIT 5;`).")
    out.append("")

    out.append("## Licence, attribution & conditions")
    out.append("")
    out.append(f"- **Cadre d'accès (registre)** : {reg.get('acces', '—')}")
    low = (reg.get("acces") or "").lower()
    if "wikidata" in low or "sparql" in low:
        out.append("- **Licence** : données Wikidata sous CC0 — réutilisation "
                   "libre, mention « Source : Wikidata » affichée par "
                   "courtoisie.")
    elif "dataset local" in low or "listes" in sid:
        out.append("- **Données compilées** de listes et palmarès médias "
                   "**publics** ; seuls des faits publics (nom, handle, "
                   "audience approximative) sont conservés.")
    elif "api" in low and ("publique" in low or "itunes" in low):
        out.append("- **API publique** utilisée selon ses conditions (pas de "
                   "clé détournée, throttling poli) ; données limitées aux "
                   "profils publics.")
    else:
        out.append("- **Profils publics uniquement** : UA identifiable "
                   "`CreaKaBot/1.0 (+https://www.crea-ka.com/bot; "
                   "contact@spboucher.ai)`, throttling poli, respect des "
                   "opt-out (`data/optout.json`) et du volet éthique "
                   "(`creaka/ethics.py` — mineurs exclus via `is_minor`).")
    out.append("- Retrait sur demande : contact@spboucher.ai (opt-out honoré "
               "à la prochaine ingestion).")
    out.append("")

    out.append("## Historique")
    out.append("")
    blob = " ".join(str(reg.get(k, "")) for k in ("notes", "statut", "acces"))
    for d in sorted({m.group(0) for m in DATE_RE.finditer(blob)}):
        out.append(f"- {d} — date mentionnée au registre (voir notes/statut).")
    if st["runs"]:
        first = st["runs"][-1]["ts"]
        out.append(f"- {str(first)[:10]} — plus ancien passage journalisé dans "
                   f"`sync_log` (fenêtre des 60 derniers).")
    out.append("- 2026-08-18 — vague d'enrichissement : standardisation de la "
               "documentation des connecteurs (fiche générée par "
               "`scripts/gen_connector_docs.py`).")
    out.append("")
    return "\n".join(out)


def main() -> None:
    now = datetime.now().strftime("%Y-%m-%d %H:%M")
    registry = json.loads(SOURCES_JSON.read_text(encoding="utf-8"))["sources"]
    con = sqlite3.connect(DB_PATH)
    con.row_factory = sqlite3.Row
    creators = load_creators(con)

    by_sid: dict[str, dict] = {}
    for path in sorted(CONN_DIR.glob("*.py")):
        if path.stem in SKIP_MODULES:
            continue
        for e in introspect_module(path):
            by_sid[e["source_id"]] = e

    DOCS_DIR.mkdir(parents=True, exist_ok=True)
    for old in DOCS_DIR.glob("*.md"):
        old.unlink()

    rows = []
    for reg in registry:
        sid = reg["id"]
        entry = by_sid.get(sid)
        st = db_stats(con, sid, creators)
        (DOCS_DIR / f"{sid}.md").write_text(
            render_fiche(reg, entry, st, now), encoding="utf-8")
        last = st["last"]
        acc = st["acc"]
        comptes = (f"{acc['n']} ({', '.join(st['platforms'])})"
                   if acc is not None and acc["n"] else "—")
        rows.append(
            f"| [`{sid}`]({sid}.md) | {reg.get('famille', '—')} "
            f"| P{reg.get('palier', '—')} "
            f"| {esc(entry['flavor'] if entry else '—', 42)} "
            f"| {entry['backend_family'] if entry else '—'} "
            f"| {st['mine']} | {comptes} "
            f"| {esc(str(reg.get('statut', 'actif')).split('—')[0], 26)} "
            f"| {esc(last['ts'], 17) if last else '—'} |")

    n_creators = con.execute("SELECT count(*) FROM creators").fetchone()[0]
    n_accounts = con.execute("SELECT count(*) FROM accounts").fetchone()[0]
    idx = [
        "# Créa-Ka — Index des connecteurs", "",
        f"_Généré automatiquement par `scripts/gen_connector_docs.py` le {now} "
        f"— ne pas éditer à la main, régénérer._", "",
        f"**{len(registry)} sources au registre** · **{n_creators} créateurs** "
        f"et **{n_accounts} comptes** en BD.", "",
        "| Source | Famille | Palier | Type d'accès | Backend | Découverts "
        "| Comptes (plateforme) | État | Dernier passage |",
        "|---|---|---|---|---|---|---|---|---|",
    ]
    idx.extend(rows)
    idx.append("")
    (DOCS_DIR / "INDEX.md").write_text("\n".join(idx), encoding="utf-8")
    con.close()
    print(f"[gen_connector_docs] {len(registry)} fiches + INDEX.md écrits dans "
          f"{DOCS_DIR}")


if __name__ == "__main__":
    main()