Python 61.6%
TypeScript 20.9%
CSS 11.4%
JavaScript 5.1%
HTML 1.1%
1#!/usr/bin/env python32# -----------------------------------------------------------------------------3# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)4# Auteur : Simon-Pierre Boucher — contact@spboucher.ai5# scripts/gen_connector_docs.py : documentation standardisée des connecteurs6#7# Génère docs/connecteurs/ (INDEX.md + une fiche par MODULE connecteur +8# fiches transverses) en croisant :9# 1. data/sources.json — registre des sources (142 sources) ;10# 2. le code des connecteurs — en-tête « mécanique » de chaque module,11# backend de fetch, plafonds env (AUTOKA_*), classes exposées ;12# 3. la BD live data/autoka.db — volumétrie, complétude par champ,13# dernier sync et erreurs récentes (sync_log), doublons VIN, rappels14# Transports Canada, coordonnées concessionnaires.15#16# Rejouable à volonté (BD ouverte en lecture seule, docs régénérés) :17# .venv/bin/python3 scripts/gen_connector_docs.py18# -----------------------------------------------------------------------------19from __future__ import annotations2021import datetime22import json23import re24import sqlite325import sys26from collections import OrderedDict27from pathlib import Path2829ROOT = Path(__file__).resolve().parent.parent30sys.path.insert(0, str(ROOT))3132DB_PATH = ROOT / "data" / "autoka.db"33SOURCES_PATH = ROOT / "data" / "sources.json"34VILLES_PATH = ROOT / "data" / "villes_gps.json"35CONNECTORS_DIR = ROOT / "autoka" / "connectors"36OUT_DIR = ROOT / "docs" / "connecteurs"3738# Champs de complétude (annonces actives) : libellé -> expression SQL "rempli"39FIELDS = OrderedDict([40 ("Marque", "make IS NOT NULL AND make != ''"),41 ("Modèle", "model IS NOT NULL AND model != ''"),42 ("Année", "year IS NOT NULL"),43 ("Kilométrage", "mileage_km IS NOT NULL"),44 ("Prix", "price IS NOT NULL"),45 ("Carburant", "fuel IS NOT NULL AND fuel != ''"),46 ("Boîte (transmission)", "transmission IS NOT NULL AND transmission != ''"),47 ("VIN", "vin IS NOT NULL AND vin != ''"),48 ("Photos", "images IS NOT NULL AND images NOT IN ('', '[]')"),49 ("Carfax", "carfax_url IS NOT NULL AND carfax_url != ''"),50 ("GPS (lat/lng)", "lat IS NOT NULL AND lng IS NOT NULL"),51])5253# Historique des vagues d'enrichissement (source de vérité : git log)54HISTORIQUE = """\55## Historique des vagues (2026-08-18)5657| Commit | Contenu |58|---|---|59| `2a35ac2` | Enrichissement connecteurs : Carfax, détails AED/LPDG, verticale moto, photos Magnetis, GPS concessionnaires |60| `bbc070b` | Vague 2 : dédup VIN inter-sources, Kijiji particuliers, rappels Transports Canada, coordonnées concessionnaires |61| `b6d5459` | Documentation standardisée des connecteurs (générateur + fiches) |62| Vague 3 | Grands portails : AutoTrader.ca/AutoHebdo (fenêtre récente), Otogo.ca (inventaire complet, VIN+GPS), CarGurus.ca (deal rating, via Scrapfly) |63"""646566# --------------------------------------------------------------------------- #67# Introspection du code68# --------------------------------------------------------------------------- #6970def module_header(path: Path, marker: str) -> str:71 """En-tête « mécanique » d'un module : bloc de commentaires du haut du72 fichier, à partir de la ligne `# <marker>` (ex. connectors/x.py :) jusqu'à73 la ligne de tirets fermante. Rendu en bloc de citation Markdown."""74 lines = []75 started = False76 for raw in path.read_text(encoding="utf-8").splitlines():77 if not raw.startswith("#"):78 break79 body = raw.lstrip("#").strip()80 if not started:81 if body.startswith(marker):82 started = True83 lines.append(body)84 continue85 if set(body) <= {"-"} and len(body) > 10: # ligne de tirets finale86 break87 lines.append(body)88 if not lines:89 return "_(pas d'en-tête trouvé)_"90 return "\n".join("> " + (l if l else "") for l in lines)919293def detect_backend(src: str) -> str:94 low = src.lower()95 if "use_firecrawl" in low:96 return ("requests direct — Firecrawl par source au besoin "97 "(`use_firecrawl`, sites derrière Cloudflare)")98 if "firecrawl" in low:99 return "Firecrawl (contournement Cloudflare / rendu JS)"100 if "scrapfly" in low:101 return "Scrapfly"102 return "requests direct (aucun anti-bot)"103104105def detect_mechanics(src: str) -> str:106 low = src.lower()107 found = []108 for needle, label in [109 ("sitemap", "sitemap"),110 ("json-ld", "JSON-LD schema.org"),111 ("__next_f", "flux RSC Next.js"),112 ("__next_data__", "__NEXT_DATA__"),113 ("graphql", "GraphQL"),114 ("appsync", "AWS AppSync"),115 ("page-data.json", "page-data Gatsby"),116 ("rss", "flux RSS"),117 ("microdata", "microdata schema.org"),118 ("itemprop", "microdata schema.org"),119 ("beautifulsoup", "parsing HTML"),120 ("data-class=", "blob JSON data-class"),121 ("/api/", "API JSON interne"),122 (".json", "endpoint JSON"),123 ]:124 if needle in low and label not in found:125 found.append(label)126 return ", ".join(found) if found else "—"127128129def detect_caps(src: str) -> str:130 caps = sorted(set(re.findall(r"AUTOKA_[A-Z_]+", src)))131 caps = [c for c in caps if c not in ("AUTOKA_",)]132 if "detail_cache" in src or "cache_get" in src or "cached" in src.lower():133 caps.append("cache détail BD (detail_cache)")134 return ", ".join(caps) if caps else "aucun"135136137def module_classes(mod_name: str) -> list[str]:138 """Classes concrètes (source_id non vide) déclarées par le module."""139 import autoka.connectors as reg140 out = []141 for sid, cls in reg.CONNECTORS.items():142 if cls.__module__.rsplit(".", 1)[-1] == mod_name:143 out.append(f"`{cls.__name__}` → `{sid}`")144 return out145146147# --------------------------------------------------------------------------- #148# BD live149# --------------------------------------------------------------------------- #150151def q1(db, sql, args=()):152 return db.execute(sql, args).fetchone()153154155def completeness(db, source_ids: list[str]) -> tuple[int, OrderedDict]:156 ph = ",".join("?" * len(source_ids))157 parts = ", ".join(f"SUM(CASE WHEN {expr} THEN 1 ELSE 0 END)"158 for expr in FIELDS.values())159 row = q1(db, f"SELECT COUNT(*), {parts} FROM vehicles "160 f"WHERE active = 1 AND source IN ({ph})", source_ids)161 n = row[0] or 0162 out = OrderedDict()163 for (label, _), filled in zip(FIELDS.items(), row[1:]):164 out[label] = (100.0 * (filled or 0) / n) if n else 0.0165 return n, out166167168def source_stats(db, sid: str) -> dict:169 tot, act, dups = q1(db, "SELECT COUNT(*), SUM(active), "170 "SUM(dup_of IS NOT NULL) FROM vehicles "171 "WHERE source = ?", (sid,))172 last = q1(db, "SELECT ts, ok, found, added, updated, removed, message "173 "FROM sync_log WHERE source = ? ORDER BY ts DESC LIMIT 1",174 (sid,))175 errs = q1(db, "SELECT COUNT(*) FROM (SELECT ok FROM sync_log "176 "WHERE source = ? ORDER BY ts DESC LIMIT 10) WHERE ok = 0",177 (sid,))[0]178 return {"total": tot or 0, "active": act or 0, "dups": dups or 0,179 "last": last, "errs10": errs}180181182def fmt_ts(ts) -> str:183 if not ts:184 return "—"185 return datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M")186187188def sync_cell(st: dict) -> tuple[str, str]:189 last = st["last"]190 if not last:191 return "jamais", "—"192 ts, ok, found, *_ = last193 flag = "OK" if ok else "ERREUR"194 return fmt_ts(ts), f"{flag} ({found or 0} trouvés)"195196197def recent_errors(db, source_ids: list[str], limit=8) -> list[tuple]:198 ph = ",".join("?" * len(source_ids))199 return db.execute(200 f"SELECT source, ts, message FROM sync_log "201 f"WHERE ok = 0 AND source IN ({ph}) ORDER BY ts DESC LIMIT ?",202 (*source_ids, limit)).fetchall()203204205# --------------------------------------------------------------------------- #206# Rendu des fiches207# --------------------------------------------------------------------------- #208209def render_completeness(n: int, comp: OrderedDict) -> list[str]:210 n_fmt = f"{n:,}".replace(",", " ")211 out = [f"## Complétude des champs (annonces actives, N = {n_fmt})",212 "", "| Champ | % rempli |", "|---|---|"]213 for label, pct in comp.items():214 out.append(f"| {label} | {pct:.1f} % |")215 out.append("")216 return out217218219def render_module_fiche(db, mod: str, members: list[dict]) -> str:220 path = CONNECTORS_DIR / f"{mod}.py"221 src = path.read_text(encoding="utf-8")222 sids = [s["id"] for s in members]223 n, comp = completeness(db, sids)224 active_total = 0225226 rows = []227 for s in sorted(members, key=lambda x: x["id"]):228 st = source_stats(db, s["id"])229 active_total += st["active"]230 when, status = sync_cell(st)231 rows.append(f"| `{s['id']}` | {s.get('name', '')} | "232 f"{s.get('city', '') or '—'} | {s.get('region', '') or '—'} | "233 f"{st['active']} | {st['dups']} | {when} | {status} |")234235 from collections import Counter236 plat_counts = Counter(s.get("platform", "").split(" (")[0].strip()237 for s in members if s.get("platform"))238 plat = plat_counts.most_common(1)[0][0] if plat_counts else mod239 lines = [240 f"# Module `{mod}` — {plat}",241 "",242 f"_Généré automatiquement par `scripts/gen_connector_docs.py` — ne pas éditer à la main._",243 "",244 "## Vue d'ensemble",245 "",246 f"- **Fichier** : `autoka/connectors/{mod}.py`",247 f"- **Sources membres** : {len(members)}",248 f"- **Annonces actives (BD)** : {active_total}",249 f"- **Backend de fetch** : {detect_backend(src)}",250 f"- **Mécanique détectée** : {detect_mechanics(src)}",251 f"- **Plafonds / cache** : {detect_caps(src)}",252 ]253 classes = module_classes(mod)254 if classes:255 lines.append(f"- **Classes** : {' ; '.join(classes[:6])}"256 + (f" … (+{len(classes) - 6})" if len(classes) > 6 else ""))257 lines += ["", "## Mécanique (en-tête du module)", "",258 module_header(path, f"connectors/{mod}.py"), ""]259 lines += render_completeness(n, comp)260 lines += ["## Sources membres (BD live)", "",261 "| Source | Nom | Ville | Région | Actives | Masquées (dup VIN) | Dernier sync | Statut |",262 "|---|---|---|---|---|---|---|---|"] + rows + [""]263264 errs = recent_errors(db, sids)265 if errs:266 lines += ["## Erreurs de synchronisation récentes (sync_log, ok = 0)", "",267 "| Source | Quand | Message |", "|---|---|---|"]268 for sid, ts, msg in errs:269 msg = (msg or "").replace("|", "\\|")[:160]270 lines.append(f"| `{sid}` | {fmt_ts(ts)} | {msg} |")271 lines.append("")272 else:273 lines += ["## Erreurs de synchronisation récentes", "",274 "Aucune erreur dans le sync_log pour les sources de ce module.", ""]275 return "\n".join(lines)276277278def render_transverse_dedup(db) -> str:279 total, masked = q1(db, "SELECT COUNT(*), SUM(dup_of IS NOT NULL) FROM vehicles")280 groups = q1(db, "SELECT COUNT(DISTINCT dup_of) FROM vehicles "281 "WHERE dup_of IS NOT NULL")[0]282 top = db.execute(283 "SELECT source, COUNT(*) c FROM vehicles WHERE dup_of IS NOT NULL "284 "GROUP BY source ORDER BY c DESC LIMIT 12").fetchall()285 lines = ["# Transverse — Dédoublonnage VIN inter-sources", "",286 "_Généré automatiquement par `scripts/gen_connector_docs.py`._", "",287 "## Mécanique (`autoka/dedup.py`)", "",288 module_header(ROOT / "autoka" / "dedup.py", "dedup.py"), "",289 "## État live", "",290 f"- **Annonces en base** : {total}",291 f"- **Annonces masquées (`dup_of` non nul)** : {masked}",292 f"- **Annonces canoniques avec doublons** : {groups}",293 "- **Autorité** : concessionnaire direct > portail/regroupeur > petites annonces ;"294 " puis complétude de la fiche, puis ancienneté (`first_seen`).",295 "- Les listes filtrent `dup_of IS NULL` ; les fiches détail restent accessibles par uid.",296 "", "## Sources les plus masquées", "",297 "| Source | Annonces masquées |", "|---|---|"]298 lines += [f"| `{s}` | {c} |" for s, c in top]299 lines += ["", "Recalcul one-shot : `python3 -m autoka.dedup` (idempotent,"300 " relancé à la fin de chaque cycle d'ingestion).", ""]301 return "\n".join(lines)302303304def render_transverse_recalls(db) -> str:305 rows, nums = q1(db, "SELECT COUNT(*), COUNT(DISTINCT recall_number) FROM recalls")306 y0, y1 = q1(db, "SELECT MIN(year), MAX(year) FROM recalls WHERE year > 1900")307 cats = db.execute("SELECT category, COUNT(DISTINCT recall_number) c "308 "FROM recalls GROUP BY category ORDER BY c DESC LIMIT 8").fetchall()309 lines = ["# Transverse — Rappels de sécurité Transports Canada", "",310 "_Généré automatiquement par `scripts/gen_connector_docs.py`._", "",311 "## Mécanique (`autoka/recalls.py`)", "",312 module_header(ROOT / "autoka" / "recalls.py", "recalls.py"), "",313 "## État live", "",314 f"- **Rappels distincts (`recall_number`)** : {nums}",315 f"- **Lignes en base (rappel × marque × modèle × année)** : {rows}",316 f"- **Années-modèles couvertes** : {y0}–{y1}",317 "- **Source** : CSV mensuel complet des données ouvertes TC"318 " (`opendatatc.tc.canada.ca/vrdb_full_monthly.csv`, sans clé API).",319 "", "## Répartition par catégorie TC", "",320 "| Catégorie | Rappels distincts |", "|---|---|"]321 lines += [f"| {c or '—'} | {n} |" for c, n in cats]322 lines += ["", "Peuplement one-shot / mensuel : `python3 -m autoka.recalls"323 " [--limit-years N]`.", ""]324 return "\n".join(lines)325326327def render_transverse_dealers(db) -> str:328 tot, ph, ad, gps = q1(db, "SELECT COUNT(*), "329 "SUM(phone IS NOT NULL AND phone != ''), "330 "SUM(address IS NOT NULL AND address != ''), "331 "SUM(lat IS NOT NULL AND lng IS NOT NULL) FROM dealers")332 lines = ["# Transverse — Coordonnées des concessionnaires", "",333 "_Généré automatiquement par `scripts/gen_connector_docs.py`._", "",334 "## Mécanique (`autoka/dealers.py`)", "",335 module_header(ROOT / "autoka" / "dealers.py", "dealers.py"), "",336 "## État live (table `dealers`)", "",337 f"- **Concessionnaires en base** : {tot}",338 f"- **Avec téléphone** : {ph} ({100.0 * ph / tot:.0f} %)",339 f"- **Avec adresse** : {ad} ({100.0 * ad / tot:.0f} %)",340 f"- **Avec GPS (lat/lng)** : {gps} ({100.0 * gps / tot:.0f} %)",341 "",342 "Crawl « une page par site » (accueil : JSON-LD AutoDealer/LocalBusiness,"343 " replis regex `tel:` + code postal canadien) ; GPS de repli = centre-ville"344 " de la source (`data/villes_gps.json`). One-shot :"345 " `python3 -m autoka.dealers [--refresh]`.", ""]346 return "\n".join(lines)347348349def render_transverse_villes() -> str:350 villes = json.loads(VILLES_PATH.read_text(encoding="utf-8"))351 sample = ", ".join(sorted(villes)[:12])352 lines = ["# Transverse — Référentiel GPS des villes (`data/villes_gps.json`)", "",353 "_Généré automatiquement par `scripts/gen_connector_docs.py`._", "",354 f"- **Villes référencées** : {len(villes)}",355 "- **Format** : `\"ville en minuscules sans accents\" → [lat, lng]`"356 " (centres-villes du Québec).",357 "- **Usages** : GPS de repli des concessionnaires (`autoka/dealers.py`)"358 " et géolocalisation des annonces par la ville de leur source"359 " (les annonces Kijiji portent leur propre lat/lng exact).",360 "", f"Extrait : {sample}…", ""]361 return "\n".join(lines)362363364# --------------------------------------------------------------------------- #365# INDEX366# --------------------------------------------------------------------------- #367368def render_index(db, groups: OrderedDict, sources: list[dict]) -> str:369 total, active, masked = q1(db, "SELECT COUNT(*), SUM(active), "370 "SUM(dup_of IS NOT NULL) FROM vehicles")371 kinds = dict(db.execute("SELECT kind, COUNT(*) FROM vehicles "372 "WHERE active = 1 GROUP BY kind").fetchall())373 recall_n = q1(db, "SELECT COUNT(DISTINCT recall_number) FROM recalls")[0]374 dealers_n = q1(db, "SELECT COUNT(*) FROM dealers")[0]375 now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")376377 lines = ["# Auto-Ka — Documentation des connecteurs", "",378 f"_Générée le {now} par `scripts/gen_connector_docs.py`"379 " (rejouable : `.venv/bin/python3 scripts/gen_connector_docs.py`)._", "",380 "## Vue d'ensemble", "",381 f"- **Sources** : {len(sources)} (registre `data/sources.json`)",382 f"- **Modules connecteurs** : {len(groups)} (`autoka/connectors/`)",383 f"- **Annonces en base** : {total} — dont {active} actives"384 f" ({kinds.get('auto', 0)} autos, {kinds.get('moto', 0)} motos,"385 f" {kinds.get('scooter', 0)} scooters)",386 f"- **Doublons VIN masqués** : {masked} (voir [dédup VIN](transverse-dedup-vin.md))",387 f"- **Rappels Transports Canada** : {recall_n} rappels distincts"388 " (voir [rappels TC](transverse-rappels-tc.md))",389 f"- **Concessionnaires géolocalisés** : {dealers_n}"390 " (voir [dealers](transverse-dealers.md))",391 "", "## Fiches par module connecteur", "",392 "| Module | Fiche | Sources | Annonces actives | Backend |",393 "|---|---|---|---|---|"]394 for mod, members in groups.items():395 sids = [s["id"] for s in members]396 ph_ = ",".join("?" * len(sids))397 act = q1(db, f"SELECT COALESCE(SUM(active), 0) FROM vehicles "398 f"WHERE source IN ({ph_})", sids)[0]399 src = (CONNECTORS_DIR / f"{mod}.py").read_text(encoding="utf-8")400 backend = detect_backend(src).split(" (")[0]401 lines.append(f"| `{mod}` | [{mod}.md]({mod}.md) | {len(members)} | {act} | {backend} |")402403 lines += ["", "## Fiches transverses", "",404 "| Sujet | Fiche |", "|---|---|",405 "| Dédoublonnage VIN inter-sources | [transverse-dedup-vin.md](transverse-dedup-vin.md) |",406 "| Rappels Transports Canada | [transverse-rappels-tc.md](transverse-rappels-tc.md) |",407 "| Coordonnées des concessionnaires | [transverse-dealers.md](transverse-dealers.md) |",408 "| Référentiel GPS des villes | [transverse-villes-gps.md](transverse-villes-gps.md) |",409 "", HISTORIQUE]410 return "\n".join(lines)411412413# --------------------------------------------------------------------------- #414415def main() -> None:416 sources = json.loads(SOURCES_PATH.read_text(encoding="utf-8"))["sources"]417 groups: OrderedDict[str, list] = OrderedDict()418 for s in sources:419 groups.setdefault(s["connector"], []).append(s)420 # modules multi-sources d'abord (par taille), uniques ensuite (alpha)421 groups = OrderedDict(sorted(groups.items(),422 key=lambda kv: (-len(kv[1]), kv[0])))423424 db = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True)425 OUT_DIR.mkdir(parents=True, exist_ok=True)426427 for mod, members in groups.items():428 (OUT_DIR / f"{mod}.md").write_text(429 render_module_fiche(db, mod, members), encoding="utf-8")430 print(f" fiche {mod}.md ({len(members)} source(s))")431432 (OUT_DIR / "transverse-dedup-vin.md").write_text(render_transverse_dedup(db), encoding="utf-8")433 (OUT_DIR / "transverse-rappels-tc.md").write_text(render_transverse_recalls(db), encoding="utf-8")434 (OUT_DIR / "transverse-dealers.md").write_text(render_transverse_dealers(db), encoding="utf-8")435 (OUT_DIR / "transverse-villes-gps.md").write_text(render_transverse_villes(), encoding="utf-8")436 (OUT_DIR / "INDEX.md").write_text(render_index(db, groups, sources), encoding="utf-8")437 print(f"OK — {len(groups)} fiches module + 4 transverses + INDEX.md dans {OUT_DIR}")438439440if __name__ == "__main__":441 main()442