#!/usr/bin/env python3
# -----------------------------------------------------------------------------
# Rent-Ka — Agrégateur de logements à louer (province de Québec)
# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
# scripts/gen_connector_docs.py : documentation STANDARDISÉE des connecteurs.
#
#   Génère docs/connecteurs/INDEX.md + une fiche docs/connecteurs/<source_id>.md
#   par connecteur, de façon 100 % programmatique et rejouable, en croisant :
#     1. le registre data/sources.json (nom, url, secteurs, statut, notes) ;
#     2. l'introspection STATIQUE (ast) du code de rentka/connectors/*.py :
#        classe, backend (direct / Firecrawl / Scrapfly), endpoint de base,
#        pagination, constantes de budget — sans exécuter les connecteurs ;
#     3. la BD live data/rentka.db : volumétrie, complétude des champs par
#        source (SQL), dernier sync OK, cadence observée, erreurs récentes.
#
#   Usage : python3 scripts/gen_connector_docs.py       (depuis la racine)
#   À relancer après tout changement de connecteur / registre / ingestion.
# -----------------------------------------------------------------------------
from __future__ import annotations

import ast
import json
import re
import sqlite3
import statistics
from datetime import datetime
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
CONN_DIR = ROOT / "rentka" / "connectors"
DOCS_DIR = ROOT / "docs" / "connecteurs"
DB_PATH = ROOT / "data" / "rentka.db"
SOURCES_JSON = ROOT / "data" / "sources.json"

SKIP_MODULES = {"__init__", "base", "_detailutil"}
URL_RE = re.compile(r"https?://[^\s\"'\\)>,;]+")
DATE_RE = re.compile(r"\d{4}-\d{2}-\d{2}")
INFRA_HOSTS = ("api.firecrawl.dev", "api.scrapfly.io", "rent-ka.com",
               "nominatim", "overpass")

# Colonnes de `listings` documentées dans la section « Champs récupérés » :
# (colonne, libellé, condition SQL de complétude sur les annonces actives)
FIELDS = [
    ("title", "Titre de l'annonce", "title IS NOT NULL AND title != ''"),
    ("address", "Adresse civique", "address IS NOT NULL AND address != ''"),
    ("city", "Ville (normalisée)", "city IS NOT NULL AND city != ''"),
    ("sector", "Secteur / quartier", "sector IS NOT NULL AND sector != ''"),
    ("unit_type", "Type d'unité (3½, 4½…)",
     "unit_type IS NOT NULL AND unit_type != ''"),
    ("price", "Loyer mensuel ($)", "price IS NOT NULL"),
    ("bedrooms", "Chambres", "bedrooms IS NOT NULL"),
    ("bathrooms", "Salles de bain", "bathrooms IS NOT NULL"),
    ("area_sqft", "Superficie (pi²)", "area_sqft IS NOT NULL"),
    ("availability_date", "Date de disponibilité",
     "availability_date IS NOT NULL AND availability_date != ''"),
    ("lat", "GPS (lat/lng)", "lat IS NOT NULL AND lng IS NOT NULL"),
    ("description", "Description", "description IS NOT NULL AND description != ''"),
    ("images", "Photos (JSON)", "images IS NOT NULL AND length(images) > 4"),
    ("amenities", "Commodités (JSON)",
     "amenities IS NOT NULL AND length(amenities) > 4"),
    ("details", "Détails additionnels (JSON)",
     "details IS NOT NULL AND length(details) > 4"),
    ("url", "URL de l'annonce source", "url IS NOT NULL AND url != ''"),
]


# -- helpers de formatage ------------------------------------------------------

def esc(s: str, limit: int = 100) -> str:
    """Échappe une valeur pour une cellule de tableau Markdown."""
    s = str(s).replace("\\", "\\\\").replace("|", "\\|")
    s = re.sub(r"\s+", " ", s).strip()
    return s[: limit - 1] + "…" if len(s) > limit else s


def pct(n, d) -> str:
    if not d:
        return "—"
    return f"{100.0 * (n or 0) / d:.0f} %"


def fmt_ts(ts) -> str:
    if not ts:
        return "—"
    return datetime.fromtimestamp(float(ts)).strftime("%Y-%m-%d %H:%M")


def fmt_secs(sec: float) -> str:
    if sec < 5400:
        return f"≈ {sec / 60:.0f} min"
    if sec < 129600:
        return f"≈ {sec / 3600:.1f} h"
    return f"≈ {sec / 86400:.1f} j"


def example_value(col: str, value) -> str:
    """Rend un exemple lisible tiré d'une ligne réelle de la BD."""
    if value is None or value == "":
        return "—"
    if col == "images":
        try:
            imgs = json.loads(value)
            if not imgs:
                return "—"
            return esc(f"{len(imgs)} photo(s) — {imgs[0]}", 90)
        except (ValueError, TypeError):
            return esc(value, 90)
    if col in ("amenities", "details"):
        return esc(value, 90)
    if col == "price":
        return f"{float(value):,.0f} $".replace(",", " ")
    if col in ("bedrooms", "bathrooms", "area_sqft"):
        v = float(value)
        return f"{v:g}"
    return esc(value, 90)


# -- introspection statique du code des connecteurs ----------------------------

def banner_description(text: str, module: str) -> str:
    """Extrait la description du bandeau de commentaires en tête de module."""
    lines, started = [], False
    for raw in text.splitlines():
        if not raw.startswith("#"):
            if started or not raw.strip():
                break
            continue
        body = raw.lstrip("#").strip()
        if set(body) <= {"-", "="}:
            continue
        marker = f"connectors/{module}.py"
        if not started:
            if marker in body:
                started = True
                lines.append(body.split(":", 1)[-1].strip())
            continue
        lines.append(body)
    return " ".join(l for l in lines if l).strip()


def detect_backends(text: str) -> tuple[list[str], str]:
    """(liste détaillée, famille courte) des backends de fetch utilisés."""
    detailed, family = [], "direct"
    if re.search(r"self\.(get|post)\(", text):
        detailed.append("requests direct (session UA RentKaBot, throttling poli)")
    if ".scrapfly(" in text:
        detailed.append("Scrapfly (asp + render_js — contournement anti-bot)")
        family = "Scrapfly"
    if "get_rendered(" in text:
        detailed.append("Scrapfly render_js (HTML rendu, JavaScript exécuté"
                        " — ex-Firecrawl, migré 2026-08-27)")
        family = "Scrapfly" if family == "direct" else family
    if not detailed:
        detailed.append("requests direct")
    return detailed, family


def detect_flavor(text: str) -> str:
    low = text.lower()
    if "graphql" in low:
        return "API GraphQL interne"
    if "admin-ajax" in low:
        return "API admin-ajax (WordPress)"
    if "/wp-json" in low:
        return "API REST WordPress (wp-json)"
    if "sitemap" in low:
        return "sitemap XML + pages HTML"
    if re.search(r"\.json\(\)", text) and re.search(r"api[./_]", low):
        return "API JSON interne du site"
    return "pages HTML (rendu serveur)"


def detect_pagination(text: str, constants: dict) -> str:
    hits = []
    low = text.lower()
    if "js_scenario" in text:
        hits.append("défilement simulé (js_scenario Scrapfly)")
    if re.search(r"[?&]page=|[\"']page[\"']\s*[:=]|paged", low):
        hits.append("pagination par numéro de page")
    if re.search(r"[?&]offset=|[\"']offset[\"']", low):
        hits.append("pagination par offset")
    if "load_more" in low or "loadmore" in low:
        hits.append("bouton « charger plus » rejoué")
    roots = [k for k, v in constants.items()
             if isinstance(v, (list, tuple)) and len(v) > 1
             and k in ("ROOTS", "PAGES", "SECTIONS", "SECTEURS", "CITIES",
                       "REGIONS", "URLS", "LISTS")]
    if roots:
        n = len(constants[roots[0]])
        hits.append(f"multi-racines ({n} pages de départ : {roots[0]})")
    if not hits:
        hits.append("page de liste unique (pas de pagination)")
    return " ; ".join(hits)


def introspect_module(path: Path) -> list[dict]:
    """Retourne une entrée par classe connecteur du module (souvent 1)."""
    text = path.read_text(encoding="utf-8")
    module = path.stem
    try:
        tree = ast.parse(text)
    except SyntaxError:
        return []

    constants: dict = {}
    for node in tree.body:
        if isinstance(node, ast.Assign) and len(node.targets) == 1 \
                and isinstance(node.targets[0], ast.Name) \
                and node.targets[0].id.isupper():
            try:
                constants[node.targets[0].id] = ast.literal_eval(node.value)
            except (ValueError, TypeError, SyntaxError):
                seg = ast.get_source_segment(text, node.value) or ""
                urls = URL_RE.findall(seg)
                constants[node.targets[0].id] = urls if len(urls) > 1 else \
                    (urls[0] if urls else None)

    urls_all = []
    for u in URL_RE.findall(text):
        u = u.rstrip('".')
        host = u.split("//", 1)[-1].split("/", 1)[0]
        if "." not in host:      # fragment de f-string, pas une vraie URL
            continue
        if not any(h in u for h in INFRA_HOSTS) and u not in urls_all:
            urls_all.append(u)

    endpoint = None
    for name in ("BASE", "BASE_URL", "API", "API_URL", "API_BASE", "ROOT",
                 "SITE", "URL", "LIST_URL", "HOST"):
        v = constants.get(name)
        if isinstance(v, str) and v.startswith("http"):
            endpoint = v
            break
    if not endpoint and urls_all:
        endpoint = urls_all[0]

    budgets = {k: v for k, v in constants.items()
               if isinstance(v, (int, float)) and not isinstance(v, bool)
               and re.search(r"MAX|CAP|LIMIT|BUDGET|TTL|PER_PAGE|PAGES|DELAY", k)}

    backends, family = detect_backends(text)
    entries = []
    for node in tree.body:
        if not isinstance(node, ast.ClassDef):
            continue
        bases = {getattr(b, "id", getattr(b, "attr", "")) for b in node.bases}
        if not (bases & {"BaseConnector"}) and not any(
                b.endswith("Connector") for b in bases if b):
            continue
        attrs = {"request_delay": 0.6, "timeout": 30, "use_detail_cache": True,
                 "disabled": False, "source_id": ""}
        for sub in node.body:
            if isinstance(sub, ast.Assign) and len(sub.targets) == 1 \
                    and isinstance(sub.targets[0], ast.Name):
                try:
                    attrs[sub.targets[0].id] = ast.literal_eval(sub.value)
                except (ValueError, TypeError, SyntaxError):
                    pass
        if not attrs.get("source_id"):
            continue
        entries.append({
            "module": module, "path": f"rentka/connectors/{module}.py",
            "class": node.name, "class_doc": ast.get_docstring(node) or "",
            "banner": banner_description(text, module),
            "source_id": attrs["source_id"], "disabled": attrs["disabled"],
            "request_delay": attrs["request_delay"], "timeout": attrs["timeout"],
            "use_detail_cache": attrs["use_detail_cache"],
            "endpoint": endpoint, "urls": urls_all[:4], "budgets": budgets,
            "backends": backends, "backend_family": family,
            "flavor": detect_flavor(text),
            "pagination": detect_pagination(text, constants),
        })
    return entries


# -- BD live -------------------------------------------------------------------

def db_stats(con: sqlite3.Connection, sid: str) -> dict:
    parts = ", ".join(
        f"sum(CASE WHEN active=1 AND {cond} THEN 1 ELSE 0 END) AS f_{col}"
        for col, _label, cond in FIELDS)
    row = con.execute(
        f"SELECT count(*) AS total, coalesce(sum(active),0) AS act, "
        f"min(first_seen) AS first_seen, max(last_seen) AS last_seen, {parts} "
        f"FROM listings WHERE source=?", (sid,)).fetchone()
    sample = con.execute(
        "SELECT * FROM listings WHERE source=? AND active=1 "
        "ORDER BY last_seen DESC LIMIT 1", (sid,)).fetchone()
    if sample is None:
        sample = con.execute(
            "SELECT * FROM listings WHERE source=? ORDER BY last_seen DESC "
            "LIMIT 1", (sid,)).fetchone()

    runs = con.execute(
        "SELECT ts, ok, found, added, updated, removed, message FROM sync_log "
        "WHERE source=? ORDER BY ts DESC LIMIT 60", (sid,)).fetchall()
    ok_ts = sorted(r["ts"] for r in runs if r["ok"])
    cadence = None
    if len(ok_ts) >= 3:
        deltas = [b - a for a, b in zip(ok_ts, ok_ts[1:]) if b - a > 60]
        if deltas:
            cadence = statistics.median(deltas)
    last_ok = next((r for r in runs if r["ok"]), None)
    errors = [r for r in runs if not r["ok"]][:5]
    return {"agg": row, "sample": sample, "runs": runs, "cadence": cadence,
            "last_ok": last_ok, "errors": errors,
            "err_count": sum(1 for r in runs if not r["ok"])}


# -- rendu des fiches ------------------------------------------------------------

def short_status(entry: dict, reg: list[dict]) -> str:
    if entry["disabled"]:
        return "désactivé (disabled=True)"
    status = (reg[0].get("status") or "") if reg else ""
    if status.startswith("actif (dégradé)"):
        return "actif (dégradé)"
    for p in ("actif", "suspendu", "non connectable"):
        if status.startswith(p):
            return p
    return status.split("—")[0].strip() or "actif (hors registre)"


def history_lines(reg: list[dict], stats: dict) -> list[str]:
    lines = []
    if stats["agg"]["first_seen"]:
        lines.append(f"- {fmt_ts(stats['agg']['first_seen'])[:10]} — premières "
                     "annonces de la source ingérées dans la BD.")
    seen = set()
    for r in reg:
        blob = " ".join(str(r.get(k, "")) for k in ("status", "notes"))
        for m in DATE_RE.finditer(blob):
            d = m.group(0)
            if d in seen:
                continue
            seen.add(d)
            ctx = blob[max(0, m.start() - 90): m.end() + 90]
            ctx = re.sub(r"\s+", " ", ctx).strip()
            lines.append(f"- {d} — mention au registre : « …{ctx}… »")
    lines.append("- 2026-08-18 — vague d'enrichissement : standardisation de la "
                 "documentation des connecteurs (fiche générée par "
                 "`scripts/gen_connector_docs.py`).")
    return lines


def render_fiche(entry: dict, reg: list[dict], stats: dict, now: str) -> str:
    sid = entry["source_id"]
    agg = stats["agg"]
    main = reg[0] if reg else {}
    name = main.get("name") or sid
    region = main.get("region") or "—"
    etat = short_status(entry, reg)

    out = [f"# {name} — connecteur `{sid}`", ""]
    out.append(f"_Fiche générée automatiquement par "
               f"`scripts/gen_connector_docs.py` le {now} — ne pas éditer à la "
               f"main, régénérer._")
    out.append("")
    out.append(f"**État : {etat}** · Région : {region} · Backend : "
               f"{entry['backend_family']} · Annonces actives : "
               f"{agg['act']}/{agg['total']}")
    out.append("")

    # -- Description ------------------------------------------------------------
    out.append("## Description de la source")
    out.append("")
    desc = entry["banner"] or entry["class_doc"].splitlines()[0] if (
        entry["banner"] or entry["class_doc"]) else ""
    if desc:
        out.append(desc)
        out.append("")
    out.append(f"- **Site** : {main.get('url', '—')}")
    out.append(f"- **Page des annonces** : {main.get('listing_url', '—')}")
    if main.get("sectors"):
        out.append(f"- **Secteurs couverts** : {main['sectors']}")
    out.append(f"- **Module** : `{entry['path']}` — classe `{entry['class']}`")
    if len(reg) > 1:
        others = ", ".join(f"`{r['id']}` ({r.get('name', '')})" for r in reg[1:])
        out.append(f"- **Entrées additionnelles du registre couvertes** : {others}")
    out.append("")

    # -- Accès --------------------------------------------------------------------
    out.append("## Accès")
    out.append("")
    out.append(f"- **Type d'accès** : {entry['flavor']}")
    out.append(f"- **Endpoint de base** : {entry['endpoint'] or '—'}")
    if entry["urls"]:
        out.append("- **URLs de départ (constantes du module)** : "
                   + " · ".join(entry["urls"]))
    out.append("- **Authentification** : aucune — contenu public"
               + (" (clé Scrapfly côté Rent-Ka)" if entry["backend_family"] ==
                  "Scrapfly" else ""))
    out.append(f"- **Pagination** : {entry['pagination']}")
    out.append(f"- **Backend anti-bot / rendu** : {' ; '.join(entry['backends'])}")
    out.append(f"- **Politesse** : {entry['request_delay']} s entre requêtes, "
               f"timeout {entry['timeout']} s, User-Agent identifiable "
               f"`RentKaBot/1.0 (+https://www.rent-ka.com/bot)`")
    out.append("")

    # -- Champs -------------------------------------------------------------------
    out.append("## Champs récupérés → schéma cible")
    out.append("")
    out.append(f"Le connecteur ({entry['backend_family']}, {entry['flavor']}) "
               f"alimente les colonnes de la table `listings`. Complétude "
               f"mesurée en SQL sur les {agg['act']} annonces actives ; exemple "
               f"tiré d'une ligne réelle de la BD.")
    out.append("")
    out.append("| Colonne `listings` | Contenu | Renseignée (actives) | Exemple réel |")
    out.append("|---|---|---|---|")
    sample = stats["sample"]
    for col, label, _cond in FIELDS:
        ex = example_value(col, sample[col]) if sample is not None else "—"
        out.append(f"| `{col}` | {label} | {pct(agg[f'f_{col}'], agg['act'])} "
                   f"| {ex} |")
    out.append("")

    # -- Fréquence & budget ---------------------------------------------------------
    out.append("## Fréquence & budget")
    out.append("")
    cad = fmt_secs(stats["cadence"]) if stats["cadence"] else "—"
    out.append(f"- **Cadence observée** (médiane des passages OK, sync_log) : {cad}")
    lo = stats["last_ok"]
    if lo:
        out.append(f"- **Dernier passage OK** : {fmt_ts(lo['ts'])} — "
                   f"{lo['found'] or 0} trouvées, +{lo['added'] or 0} / "
                   f"~{lo['updated'] or 0} / -{lo['removed'] or 0}")
    else:
        out.append("- **Dernier passage OK** : aucun dans les 60 derniers runs")
    out.append(f"- **Throttling** : {entry['request_delay']} s entre requêtes "
               f"(constante de classe `request_delay`)")
    out.append(f"- **Cache des pages détail (BD `detail_cache`)** : "
               f"{'activé' if entry['use_detail_cache'] else 'désactivé'} — "
               f"évite de re-visiter les fiches inchangées")
    if entry["budgets"]:
        caps = ", ".join(f"`{k}` = {v}" for k, v in sorted(entry["budgets"].items()))
        out.append(f"- **Caps / budgets du module** : {caps}")
    out.append("")

    # -- Volumétrie -------------------------------------------------------------------
    out.append("## Volumétrie & complétude")
    out.append("")
    out.append(f"- **Annonces en BD** : {agg['total']} au total, "
               f"**{agg['act']} actives**")
    out.append(f"- **Première ingestion** : {fmt_ts(agg['first_seen'])[:10]} · "
               f"**Dernière observation** : {fmt_ts(agg['last_seen'])[:10]}")
    out.append(f"- **Complétude clé (actives)** : GPS "
               f"{pct(agg['f_lat'], agg['act'])} · prix "
               f"{pct(agg['f_price'], agg['act'])} · photos "
               f"{pct(agg['f_images'], agg['act'])} · description "
               f"{pct(agg['f_description'], agg['act'])}")
    out.append(f"- **Runs journalisés (60 derniers)** : {len(stats['runs'])}, "
               f"dont {stats['err_count']} en erreur")
    out.append("")

    # -- Erreurs ------------------------------------------------------------------------
    out.append("## Erreurs connues & dépannage")
    out.append("")
    if stats["errors"]:
        out.append("| Date | Message (sync_log) |")
        out.append("|---|---|")
        for r in stats["errors"]:
            out.append(f"| {fmt_ts(r['ts'])} | {esc(r['message'] or '', 160)} |")
        out.append("")
    else:
        out.append("Aucune erreur dans les 60 derniers runs journalisés.")
        out.append("")
    status = main.get("status", "")
    if status and not status.strip() == "actif":
        out.append(f"**Note du registre** : {status}")
        out.append("")
    out.append(f"Rejouer la source seule : `python3 run.py sync {sid}` · "
               f"vérifier ensuite `sync_log` (`SELECT * FROM sync_log WHERE "
               f"source='{sid}' ORDER BY ts DESC LIMIT 5;`).")
    out.append("")

    # -- Licence ------------------------------------------------------------------------
    out.append("## Licence, attribution & conditions")
    out.append("")
    out.append("- Scraping poli d'annonces **publiques** (données factuelles : "
               "prix, adresse, disponibilité) publiées par le gestionnaire sur "
               "son propre site.")
    out.append("- User-Agent **identifiable** avec page d'information et "
               "contact : `RentKaBot/1.0 (+https://www.rent-ka.com/bot; "
               "contact@spboucher.ai)` ; throttling "
               f"{entry['request_delay']} s ; cache détail pour minimiser les "
               "requêtes.")
    out.append("- Chaque annonce affichée sur Rent-Ka **pointe vers l'annonce "
               "d'origine** (colonne `url`) — la source garde le trafic de "
               "conversion.")
    out.append("- Retrait sur demande : contact@spboucher.ai.")
    out.append("")

    # -- Historique ----------------------------------------------------------------------
    out.append("## Historique")
    out.append("")
    out.extend(history_lines(reg, stats))
    out.append("")
    return "\n".join(out)


# -- programme principal -----------------------------------------------------------

def main() -> None:
    now = datetime.now().strftime("%Y-%m-%d %H:%M")
    registry = json.loads(SOURCES_JSON.read_text(encoding="utf-8"))["sources"]
    by_connector: dict[str, list[dict]] = {}
    for r in registry:
        key = r.get("connector") or ""
        if key:
            by_connector.setdefault(key, []).append(r)
    # l'entrée dont id == connector d'abord (entrée « principale »)
    for key, lst in by_connector.items():
        lst.sort(key=lambda r: (r["id"] != key, r["id"]))

    con = sqlite3.connect(DB_PATH)
    con.row_factory = sqlite3.Row

    entries: list[dict] = []
    for path in sorted(CONN_DIR.glob("*.py")):
        if path.stem in SKIP_MODULES:
            continue
        entries.extend(introspect_module(path))
    entries.sort(key=lambda e: e["source_id"])

    DOCS_DIR.mkdir(parents=True, exist_ok=True)
    for old in DOCS_DIR.glob("*.md"):
        old.unlink()

    index_rows = []
    covered = set()
    tot_active = 0
    for entry in entries:
        sid = entry["source_id"]
        reg = by_connector.get(sid) or by_connector.get(entry["module"]) or []
        for r in reg:
            covered.add(r["id"])
        stats = db_stats(con, sid)
        (DOCS_DIR / f"{sid}.md").write_text(
            render_fiche(entry, reg, stats, now), encoding="utf-8")
        agg = stats["agg"]
        tot_active += agg["act"] or 0
        lo = stats["last_ok"]
        main_r = reg[0] if reg else {}
        index_rows.append(
            f"| [`{sid}`]({sid}.md) | {esc(main_r.get('name', sid), 40)} "
            f"| {esc(main_r.get('region', '—'), 20)} "
            f"| {esc(entry['flavor'], 34)} | {entry['backend_family']} "
            f"| {agg['act']}/{agg['total']} "
            f"| {pct(agg['f_lat'], agg['act'])} "
            f"| {pct(agg['f_price'], agg['act'])} "
            f"| {pct(agg['f_images'], agg['act'])} "
            f"| {esc(short_status(entry, reg), 26)} "
            f"| {fmt_ts(lo['ts']) if lo else '—'} |")

    # INDEX.md ------------------------------------------------------------------
    n_act = sum(1 for e in entries if not e["disabled"])
    idx = [
        "# Rent-Ka — Index des connecteurs", "",
        f"_Généré automatiquement par `scripts/gen_connector_docs.py` le {now} "
        f"— ne pas éditer à la main, régénérer._", "",
        f"**{len(entries)} connecteurs documentés** ({n_act} activés, "
        f"{len(entries) - n_act} désactivés) · **{tot_active} annonces "
        f"actives** au total · registre : {len(registry)} entrées.", "",
        "| Connecteur | Nom | Région | Type d'accès | Backend | Actives/Total "
        "| GPS | Prix | Photos | État | Dernier sync OK |",
        "|---|---|---|---|---|---|---|---|---|---|---|",
    ]
    idx.extend(index_rows)
    leftovers = [r for r in registry if r["id"] not in covered]
    if leftovers:
        idx += ["", "## Entrées du registre sans module connecteur", "",
                "Sources recensées mais non connectées (voir le champ `status` "
                "du registre pour la raison détaillée) :", ""]
        for r in sorted(leftovers, key=lambda x: x["id"]):
            reason = esc((r.get("status") or "").split("—")[0], 60)
            idx.append(f"- `{r['id']}` — {esc(r.get('name', ''), 60)} "
                       f"({reason or '—'})")
    idx.append("")
    (DOCS_DIR / "INDEX.md").write_text("\n".join(idx), encoding="utf-8")
    con.close()
    print(f"[gen_connector_docs] {len(entries)} fiches + INDEX.md écrits dans "
          f"{DOCS_DIR}")


if __name__ == "__main__":
    main()