Repository Public
1#!/usr/bin/env python32# -----------------------------------------------------------------------------3# Lou-Ka — Agrégateur de logements à louer (province de Québec)4# Auteur : Simon-Pierre Boucher — contact@spboucher.ai5# scripts/gen_connector_docs.py : documentation STANDARDISÉE des connecteurs.6#7# Génère docs/connecteurs/INDEX.md + une fiche docs/connecteurs/<source_id>.md8# par connecteur, de façon 100 % programmatique et rejouable, en croisant :9# 1. le registre data/sources.json (nom, url, secteurs, statut, notes) ;10# 2. l'introspection STATIQUE (ast) du code de louka/connectors/*.py :11# classe, backend (direct / Firecrawl / Scrapfly), endpoint de base,12# pagination, constantes de budget — sans exécuter les connecteurs ;13# 3. la BD live data/louka.db : volumétrie, complétude des champs par14# source (SQL), dernier sync OK, cadence observée, erreurs récentes.15#16# Usage : python3 scripts/gen_connector_docs.py (depuis la racine)17# À relancer après tout changement de connecteur / registre / ingestion.18# -----------------------------------------------------------------------------19from __future__ import annotations2021import ast22import json23import re24import sqlite325import statistics26from datetime import datetime27from pathlib import Path2829ROOT = Path(__file__).resolve().parents[1]30CONN_DIR = ROOT / "louka" / "connectors"31DOCS_DIR = ROOT / "docs" / "connecteurs"32DB_PATH = ROOT / "data" / "louka.db"33SOURCES_JSON = ROOT / "data" / "sources.json"3435SKIP_MODULES = {"__init__", "base", "_detailutil"}36URL_RE = re.compile(r"https?://[^\s\"'\\)>,;]+")37DATE_RE = re.compile(r"\d{4}-\d{2}-\d{2}")38INFRA_HOSTS = ("api.firecrawl.dev", "api.scrapfly.io", "lou-ka.com",39 "nominatim", "overpass")4041# Colonnes de `listings` documentées dans la section « Champs récupérés » :42# (colonne, libellé, condition SQL de complétude sur les annonces actives)43FIELDS = [44 ("title", "Titre de l'annonce", "title IS NOT NULL AND title != ''"),45 ("address", "Adresse civique", "address IS NOT NULL AND address != ''"),46 ("city", "Ville (normalisée)", "city IS NOT NULL AND city != ''"),47 ("sector", "Secteur / quartier", "sector IS NOT NULL AND sector != ''"),48 ("unit_type", "Type d'unité (3½, 4½…)",49 "unit_type IS NOT NULL AND unit_type != ''"),50 ("price", "Loyer mensuel ($)", "price IS NOT NULL"),51 ("bedrooms", "Chambres", "bedrooms IS NOT NULL"),52 ("bathrooms", "Salles de bain", "bathrooms IS NOT NULL"),53 ("area_sqft", "Superficie (pi²)", "area_sqft IS NOT NULL"),54 ("availability_date", "Date de disponibilité",55 "availability_date IS NOT NULL AND availability_date != ''"),56 ("lat", "GPS (lat/lng)", "lat IS NOT NULL AND lng IS NOT NULL"),57 ("description", "Description", "description IS NOT NULL AND description != ''"),58 ("images", "Photos (JSON)", "images IS NOT NULL AND length(images) > 4"),59 ("amenities", "Commodités (JSON)",60 "amenities IS NOT NULL AND length(amenities) > 4"),61 ("details", "Détails additionnels (JSON)",62 "details IS NOT NULL AND length(details) > 4"),63 ("url", "URL de l'annonce source", "url IS NOT NULL AND url != ''"),64]656667# -- helpers de formatage ------------------------------------------------------6869def esc(s: str, limit: int = 100) -> str:70 """Échappe une valeur pour une cellule de tableau Markdown."""71 s = str(s).replace("\\", "\\\\").replace("|", "\\|")72 s = re.sub(r"\s+", " ", s).strip()73 return s[: limit - 1] + "…" if len(s) > limit else s747576def pct(n, d) -> str:77 if not d:78 return "—"79 return f"{100.0 * (n or 0) / d:.0f} %"808182def fmt_ts(ts) -> str:83 if not ts:84 return "—"85 return datetime.fromtimestamp(float(ts)).strftime("%Y-%m-%d %H:%M")868788def fmt_secs(sec: float) -> str:89 if sec < 5400:90 return f"≈ {sec / 60:.0f} min"91 if sec < 129600:92 return f"≈ {sec / 3600:.1f} h"93 return f"≈ {sec / 86400:.1f} j"949596def example_value(col: str, value) -> str:97 """Rend un exemple lisible tiré d'une ligne réelle de la BD."""98 if value is None or value == "":99 return "—"100 if col == "images":101 try:102 imgs = json.loads(value)103 if not imgs:104 return "—"105 return esc(f"{len(imgs)} photo(s) — {imgs[0]}", 90)106 except (ValueError, TypeError):107 return esc(value, 90)108 if col in ("amenities", "details"):109 return esc(value, 90)110 if col == "price":111 return f"{float(value):,.0f} $".replace(",", " ")112 if col in ("bedrooms", "bathrooms", "area_sqft"):113 v = float(value)114 return f"{v:g}"115 return esc(value, 90)116117118# -- introspection statique du code des connecteurs ----------------------------119120def banner_description(text: str, module: str) -> str:121 """Extrait la description du bandeau de commentaires en tête de module."""122 lines, started = [], False123 for raw in text.splitlines():124 if not raw.startswith("#"):125 if started or not raw.strip():126 break127 continue128 body = raw.lstrip("#").strip()129 if set(body) <= {"-", "="}:130 continue131 marker = f"connectors/{module}.py"132 if not started:133 if marker in body:134 started = True135 lines.append(body.split(":", 1)[-1].strip())136 continue137 lines.append(body)138 return " ".join(l for l in lines if l).strip()139140141def detect_backends(text: str) -> tuple[list[str], str]:142 """(liste détaillée, famille courte) des backends de fetch utilisés."""143 detailed, family = [], "direct"144 if re.search(r"self\.(get|post)\(", text):145 detailed.append("requests direct (session UA LouKaBot, throttling poli)")146 if ".scrapfly(" in text:147 detailed.append("Scrapfly (asp + render_js — contournement anti-bot)")148 family = "Scrapfly"149 if "get_rendered(" in text:150 detailed.append("Firecrawl (HTML rendu, JavaScript exécuté)")151 family = "Firecrawl" if family == "direct" else family152 if not detailed:153 detailed.append("requests direct")154 return detailed, family155156157def detect_flavor(text: str) -> str:158 low = text.lower()159 if "graphql" in low:160 return "API GraphQL interne"161 if "admin-ajax" in low:162 return "API admin-ajax (WordPress)"163 if "/wp-json" in low:164 return "API REST WordPress (wp-json)"165 if "sitemap" in low:166 return "sitemap XML + pages HTML"167 if re.search(r"\.json\(\)", text) and re.search(r"api[./_]", low):168 return "API JSON interne du site"169 return "pages HTML (rendu serveur)"170171172def detect_pagination(text: str, constants: dict) -> str:173 hits = []174 low = text.lower()175 if "js_scenario" in text:176 hits.append("défilement simulé (js_scenario Scrapfly)")177 if re.search(r"[?&]page=|[\"']page[\"']\s*[:=]|paged", low):178 hits.append("pagination par numéro de page")179 if re.search(r"[?&]offset=|[\"']offset[\"']", low):180 hits.append("pagination par offset")181 if "load_more" in low or "loadmore" in low:182 hits.append("bouton « charger plus » rejoué")183 roots = [k for k, v in constants.items()184 if isinstance(v, (list, tuple)) and len(v) > 1185 and k in ("ROOTS", "PAGES", "SECTIONS", "SECTEURS", "CITIES",186 "REGIONS", "URLS", "LISTS")]187 if roots:188 n = len(constants[roots[0]])189 hits.append(f"multi-racines ({n} pages de départ : {roots[0]})")190 if not hits:191 hits.append("page de liste unique (pas de pagination)")192 return " ; ".join(hits)193194195def introspect_module(path: Path) -> list[dict]:196 """Retourne une entrée par classe connecteur du module (souvent 1)."""197 text = path.read_text(encoding="utf-8")198 module = path.stem199 try:200 tree = ast.parse(text)201 except SyntaxError:202 return []203204 constants: dict = {}205 for node in tree.body:206 if isinstance(node, ast.Assign) and len(node.targets) == 1 \207 and isinstance(node.targets[0], ast.Name) \208 and node.targets[0].id.isupper():209 try:210 constants[node.targets[0].id] = ast.literal_eval(node.value)211 except (ValueError, TypeError, SyntaxError):212 seg = ast.get_source_segment(text, node.value) or ""213 urls = URL_RE.findall(seg)214 constants[node.targets[0].id] = urls if len(urls) > 1 else \215 (urls[0] if urls else None)216217 urls_all = []218 for u in URL_RE.findall(text):219 u = u.rstrip('".')220 host = u.split("//", 1)[-1].split("/", 1)[0]221 if "." not in host: # fragment de f-string, pas une vraie URL222 continue223 if not any(h in u for h in INFRA_HOSTS) and u not in urls_all:224 urls_all.append(u)225226 endpoint = None227 for name in ("BASE", "BASE_URL", "API", "API_URL", "API_BASE", "ROOT",228 "SITE", "URL", "LIST_URL", "HOST"):229 v = constants.get(name)230 if isinstance(v, str) and v.startswith("http"):231 endpoint = v232 break233 if not endpoint and urls_all:234 endpoint = urls_all[0]235236 budgets = {k: v for k, v in constants.items()237 if isinstance(v, (int, float)) and not isinstance(v, bool)238 and re.search(r"MAX|CAP|LIMIT|BUDGET|TTL|PER_PAGE|PAGES|DELAY", k)}239240 backends, family = detect_backends(text)241 entries = []242 for node in tree.body:243 if not isinstance(node, ast.ClassDef):244 continue245 bases = {getattr(b, "id", getattr(b, "attr", "")) for b in node.bases}246 if not (bases & {"BaseConnector"}) and not any(247 b.endswith("Connector") for b in bases if b):248 continue249 attrs = {"request_delay": 0.6, "timeout": 30, "use_detail_cache": True,250 "disabled": False, "source_id": ""}251 for sub in node.body:252 if isinstance(sub, ast.Assign) and len(sub.targets) == 1 \253 and isinstance(sub.targets[0], ast.Name):254 try:255 attrs[sub.targets[0].id] = ast.literal_eval(sub.value)256 except (ValueError, TypeError, SyntaxError):257 pass258 if not attrs.get("source_id"):259 continue260 entries.append({261 "module": module, "path": f"louka/connectors/{module}.py",262 "class": node.name, "class_doc": ast.get_docstring(node) or "",263 "banner": banner_description(text, module),264 "source_id": attrs["source_id"], "disabled": attrs["disabled"],265 "request_delay": attrs["request_delay"], "timeout": attrs["timeout"],266 "use_detail_cache": attrs["use_detail_cache"],267 "endpoint": endpoint, "urls": urls_all[:4], "budgets": budgets,268 "backends": backends, "backend_family": family,269 "flavor": detect_flavor(text),270 "pagination": detect_pagination(text, constants),271 })272 return entries273274275# -- BD live -------------------------------------------------------------------276277def db_stats(con: sqlite3.Connection, sid: str) -> dict:278 parts = ", ".join(279 f"sum(CASE WHEN active=1 AND {cond} THEN 1 ELSE 0 END) AS f_{col}"280 for col, _label, cond in FIELDS)281 row = con.execute(282 f"SELECT count(*) AS total, coalesce(sum(active),0) AS act, "283 f"min(first_seen) AS first_seen, max(last_seen) AS last_seen, {parts} "284 f"FROM listings WHERE source=?", (sid,)).fetchone()285 sample = con.execute(286 "SELECT * FROM listings WHERE source=? AND active=1 "287 "ORDER BY last_seen DESC LIMIT 1", (sid,)).fetchone()288 if sample is None:289 sample = con.execute(290 "SELECT * FROM listings WHERE source=? ORDER BY last_seen DESC "291 "LIMIT 1", (sid,)).fetchone()292293 runs = con.execute(294 "SELECT ts, ok, found, added, updated, removed, message FROM sync_log "295 "WHERE source=? ORDER BY ts DESC LIMIT 60", (sid,)).fetchall()296 ok_ts = sorted(r["ts"] for r in runs if r["ok"])297 cadence = None298 if len(ok_ts) >= 3:299 deltas = [b - a for a, b in zip(ok_ts, ok_ts[1:]) if b - a > 60]300 if deltas:301 cadence = statistics.median(deltas)302 last_ok = next((r for r in runs if r["ok"]), None)303 errors = [r for r in runs if not r["ok"]][:5]304 return {"agg": row, "sample": sample, "runs": runs, "cadence": cadence,305 "last_ok": last_ok, "errors": errors,306 "err_count": sum(1 for r in runs if not r["ok"])}307308309# -- rendu des fiches ------------------------------------------------------------310311def short_status(entry: dict, reg: list[dict]) -> str:312 if entry["disabled"]:313 return "désactivé (disabled=True)"314 status = (reg[0].get("status") or "") if reg else ""315 if status.startswith("actif (dégradé)"):316 return "actif (dégradé)"317 for p in ("actif", "suspendu", "non connectable"):318 if status.startswith(p):319 return p320 return status.split("—")[0].strip() or "actif (hors registre)"321322323def history_lines(reg: list[dict], stats: dict) -> list[str]:324 lines = []325 if stats["agg"]["first_seen"]:326 lines.append(f"- {fmt_ts(stats['agg']['first_seen'])[:10]} — premières "327 "annonces de la source ingérées dans la BD.")328 seen = set()329 for r in reg:330 blob = " ".join(str(r.get(k, "")) for k in ("status", "notes"))331 for m in DATE_RE.finditer(blob):332 d = m.group(0)333 if d in seen:334 continue335 seen.add(d)336 ctx = blob[max(0, m.start() - 90): m.end() + 90]337 ctx = re.sub(r"\s+", " ", ctx).strip()338 lines.append(f"- {d} — mention au registre : « …{ctx}… »")339 lines.append("- 2026-08-18 — vague d'enrichissement : standardisation de la "340 "documentation des connecteurs (fiche générée par "341 "`scripts/gen_connector_docs.py`).")342 return lines343344345def render_fiche(entry: dict, reg: list[dict], stats: dict, now: str) -> str:346 sid = entry["source_id"]347 agg = stats["agg"]348 main = reg[0] if reg else {}349 name = main.get("name") or sid350 region = main.get("region") or "—"351 etat = short_status(entry, reg)352353 out = [f"# {name} — connecteur `{sid}`", ""]354 out.append(f"_Fiche générée automatiquement par "355 f"`scripts/gen_connector_docs.py` le {now} — ne pas éditer à la "356 f"main, régénérer._")357 out.append("")358 out.append(f"**État : {etat}** · Région : {region} · Backend : "359 f"{entry['backend_family']} · Annonces actives : "360 f"{agg['act']}/{agg['total']}")361 out.append("")362363 # -- Description ------------------------------------------------------------364 out.append("## Description de la source")365 out.append("")366 desc = entry["banner"] or entry["class_doc"].splitlines()[0] if (367 entry["banner"] or entry["class_doc"]) else ""368 if desc:369 out.append(desc)370 out.append("")371 out.append(f"- **Site** : {main.get('url', '—')}")372 out.append(f"- **Page des annonces** : {main.get('listing_url', '—')}")373 if main.get("sectors"):374 out.append(f"- **Secteurs couverts** : {main['sectors']}")375 out.append(f"- **Module** : `{entry['path']}` — classe `{entry['class']}`")376 if len(reg) > 1:377 others = ", ".join(f"`{r['id']}` ({r.get('name', '')})" for r in reg[1:])378 out.append(f"- **Entrées additionnelles du registre couvertes** : {others}")379 out.append("")380381 # -- Accès --------------------------------------------------------------------382 out.append("## Accès")383 out.append("")384 out.append(f"- **Type d'accès** : {entry['flavor']}")385 out.append(f"- **Endpoint de base** : {entry['endpoint'] or '—'}")386 if entry["urls"]:387 out.append("- **URLs de départ (constantes du module)** : "388 + " · ".join(entry["urls"]))389 out.append("- **Authentification** : aucune — contenu public"390 + (" (clé Scrapfly côté Lou-Ka)" if entry["backend_family"] ==391 "Scrapfly" else392 " (clé Firecrawl côté Lou-Ka)" if entry["backend_family"] ==393 "Firecrawl" else ""))394 out.append(f"- **Pagination** : {entry['pagination']}")395 out.append(f"- **Backend anti-bot / rendu** : {' ; '.join(entry['backends'])}")396 out.append(f"- **Politesse** : {entry['request_delay']} s entre requêtes, "397 f"timeout {entry['timeout']} s, User-Agent identifiable "398 f"`LouKaBot/1.0 (+https://www.lou-ka.com/bot)`")399 out.append("")400401 # -- Champs -------------------------------------------------------------------402 out.append("## Champs récupérés → schéma cible")403 out.append("")404 out.append(f"Le connecteur ({entry['backend_family']}, {entry['flavor']}) "405 f"alimente les colonnes de la table `listings`. Complétude "406 f"mesurée en SQL sur les {agg['act']} annonces actives ; exemple "407 f"tiré d'une ligne réelle de la BD.")408 out.append("")409 out.append("| Colonne `listings` | Contenu | Renseignée (actives) | Exemple réel |")410 out.append("|---|---|---|---|")411 sample = stats["sample"]412 for col, label, _cond in FIELDS:413 ex = example_value(col, sample[col]) if sample is not None else "—"414 out.append(f"| `{col}` | {label} | {pct(agg[f'f_{col}'], agg['act'])} "415 f"| {ex} |")416 out.append("")417418 # -- Fréquence & budget ---------------------------------------------------------419 out.append("## Fréquence & budget")420 out.append("")421 cad = fmt_secs(stats["cadence"]) if stats["cadence"] else "—"422 out.append(f"- **Cadence observée** (médiane des passages OK, sync_log) : {cad}")423 lo = stats["last_ok"]424 if lo:425 out.append(f"- **Dernier passage OK** : {fmt_ts(lo['ts'])} — "426 f"{lo['found'] or 0} trouvées, +{lo['added'] or 0} / "427 f"~{lo['updated'] or 0} / -{lo['removed'] or 0}")428 else:429 out.append("- **Dernier passage OK** : aucun dans les 60 derniers runs")430 out.append(f"- **Throttling** : {entry['request_delay']} s entre requêtes "431 f"(constante de classe `request_delay`)")432 out.append(f"- **Cache des pages détail (BD `detail_cache`)** : "433 f"{'activé' if entry['use_detail_cache'] else 'désactivé'} — "434 f"évite de re-visiter les fiches inchangées")435 if entry["budgets"]:436 caps = ", ".join(f"`{k}` = {v}" for k, v in sorted(entry["budgets"].items()))437 out.append(f"- **Caps / budgets du module** : {caps}")438 out.append("")439440 # -- Volumétrie -------------------------------------------------------------------441 out.append("## Volumétrie & complétude")442 out.append("")443 out.append(f"- **Annonces en BD** : {agg['total']} au total, "444 f"**{agg['act']} actives**")445 out.append(f"- **Première ingestion** : {fmt_ts(agg['first_seen'])[:10]} · "446 f"**Dernière observation** : {fmt_ts(agg['last_seen'])[:10]}")447 out.append(f"- **Complétude clé (actives)** : GPS "448 f"{pct(agg['f_lat'], agg['act'])} · prix "449 f"{pct(agg['f_price'], agg['act'])} · photos "450 f"{pct(agg['f_images'], agg['act'])} · description "451 f"{pct(agg['f_description'], agg['act'])}")452 out.append(f"- **Runs journalisés (60 derniers)** : {len(stats['runs'])}, "453 f"dont {stats['err_count']} en erreur")454 out.append("")455456 # -- Erreurs ------------------------------------------------------------------------457 out.append("## Erreurs connues & dépannage")458 out.append("")459 if stats["errors"]:460 out.append("| Date | Message (sync_log) |")461 out.append("|---|---|")462 for r in stats["errors"]:463 out.append(f"| {fmt_ts(r['ts'])} | {esc(r['message'] or '', 160)} |")464 out.append("")465 else:466 out.append("Aucune erreur dans les 60 derniers runs journalisés.")467 out.append("")468 status = main.get("status", "")469 if status and not status.strip() == "actif":470 out.append(f"**Note du registre** : {status}")471 out.append("")472 out.append(f"Rejouer la source seule : `python3 run.py sync {sid}` · "473 f"vérifier ensuite `sync_log` (`SELECT * FROM sync_log WHERE "474 f"source='{sid}' ORDER BY ts DESC LIMIT 5;`).")475 out.append("")476477 # -- Licence ------------------------------------------------------------------------478 out.append("## Licence, attribution & conditions")479 out.append("")480 out.append("- Scraping poli d'annonces **publiques** (données factuelles : "481 "prix, adresse, disponibilité) publiées par le gestionnaire sur "482 "son propre site.")483 out.append("- User-Agent **identifiable** avec page d'information et "484 "contact : `LouKaBot/1.0 (+https://www.lou-ka.com/bot; "485 "contact@spboucher.ai)` ; throttling "486 f"{entry['request_delay']} s ; cache détail pour minimiser les "487 "requêtes.")488 out.append("- Chaque annonce affichée sur Lou-Ka **pointe vers l'annonce "489 "d'origine** (colonne `url`) — la source garde le trafic de "490 "conversion.")491 out.append("- Retrait sur demande : contact@spboucher.ai.")492 out.append("")493494 # -- Historique ----------------------------------------------------------------------495 out.append("## Historique")496 out.append("")497 out.extend(history_lines(reg, stats))498 out.append("")499 return "\n".join(out)500501502# -- programme principal -----------------------------------------------------------503504def main() -> None:505 now = datetime.now().strftime("%Y-%m-%d %H:%M")506 registry = json.loads(SOURCES_JSON.read_text(encoding="utf-8"))["sources"]507 by_connector: dict[str, list[dict]] = {}508 for r in registry:509 key = r.get("connector") or ""510 if key:511 by_connector.setdefault(key, []).append(r)512 # l'entrée dont id == connector d'abord (entrée « principale »)513 for key, lst in by_connector.items():514 lst.sort(key=lambda r: (r["id"] != key, r["id"]))515516 con = sqlite3.connect(DB_PATH)517 con.row_factory = sqlite3.Row518519 entries: list[dict] = []520 for path in sorted(CONN_DIR.glob("*.py")):521 if path.stem in SKIP_MODULES:522 continue523 entries.extend(introspect_module(path))524 entries.sort(key=lambda e: e["source_id"])525526 DOCS_DIR.mkdir(parents=True, exist_ok=True)527 for old in DOCS_DIR.glob("*.md"):528 old.unlink()529530 index_rows = []531 covered = set()532 tot_active = 0533 for entry in entries:534 sid = entry["source_id"]535 reg = by_connector.get(sid) or by_connector.get(entry["module"]) or []536 for r in reg:537 covered.add(r["id"])538 stats = db_stats(con, sid)539 (DOCS_DIR / f"{sid}.md").write_text(540 render_fiche(entry, reg, stats, now), encoding="utf-8")541 agg = stats["agg"]542 tot_active += agg["act"] or 0543 lo = stats["last_ok"]544 main_r = reg[0] if reg else {}545 index_rows.append(546 f"| [`{sid}`]({sid}.md) | {esc(main_r.get('name', sid), 40)} "547 f"| {esc(main_r.get('region', '—'), 20)} "548 f"| {esc(entry['flavor'], 34)} | {entry['backend_family']} "549 f"| {agg['act']}/{agg['total']} "550 f"| {pct(agg['f_lat'], agg['act'])} "551 f"| {pct(agg['f_price'], agg['act'])} "552 f"| {pct(agg['f_images'], agg['act'])} "553 f"| {esc(short_status(entry, reg), 26)} "554 f"| {fmt_ts(lo['ts']) if lo else '—'} |")555556 # INDEX.md ------------------------------------------------------------------557 n_act = sum(1 for e in entries if not e["disabled"])558 idx = [559 "# Lou-Ka — Index des connecteurs", "",560 f"_Généré automatiquement par `scripts/gen_connector_docs.py` le {now} "561 f"— ne pas éditer à la main, régénérer._", "",562 f"**{len(entries)} connecteurs documentés** ({n_act} activés, "563 f"{len(entries) - n_act} désactivés) · **{tot_active} annonces "564 f"actives** au total · registre : {len(registry)} entrées.", "",565 "| Connecteur | Nom | Région | Type d'accès | Backend | Actives/Total "566 "| GPS | Prix | Photos | État | Dernier sync OK |",567 "|---|---|---|---|---|---|---|---|---|---|---|",568 ]569 idx.extend(index_rows)570 leftovers = [r for r in registry if r["id"] not in covered]571 if leftovers:572 idx += ["", "## Entrées du registre sans module connecteur", "",573 "Sources recensées mais non connectées (voir le champ `status` "574 "du registre pour la raison détaillée) :", ""]575 for r in sorted(leftovers, key=lambda x: x["id"]):576 reason = esc((r.get("status") or "").split("—")[0], 60)577 idx.append(f"- `{r['id']}` — {esc(r.get('name', ''), 60)} "578 f"({reason or '—'})")579 idx.append("")580 (DOCS_DIR / "INDEX.md").write_text("\n".join(idx), encoding="utf-8")581 con.close()582 print(f"[gen_connector_docs] {len(entries)} fiches + INDEX.md écrits dans "583 f"{DOCS_DIR}")584585586if __name__ == "__main__":587 main()588