#!/usr/bin/env python3 # ============================================================================== # Author: Simon-Pierre Boucher # File: scripts/gen_connector_docs.py # Desc: Documentation STANDARDISÉE des connecteurs Resto-Ka — génère # docs/connecteurs/INDEX.md + une fiche docs/connecteurs/.md par # source du registre, de façon 100 % programmatique et rejouable : # 1. registre data/sources.json (nom, tier, extraction, acces_legal, # cadence déclarée, statut) ; # 2. introspection STATIQUE (ast) de restoka/connectors/*.py et # restoka/inspections.py : classe, backend, endpoints, pagination, # constantes de budget — sans exécuter les connecteurs ; # 3. BD live data/restoka.db : volumétrie, complétude des champs par # source (SQL), menus, dernier sync OK, cadence observée, erreurs. # Usage : python3 scripts/gen_connector_docs.py (racine du projet) # ============================================================================== from __future__ import annotations import ast import json import re import sqlite3 import statistics from datetime import datetime from pathlib import Path ROOT = Path(__file__).resolve().parents[1] CONN_DIR = ROOT / "restoka" / "connectors" EXTRA_MODULES = [ROOT / "restoka" / "inspections.py", # mapaq (enrichissement) ROOT / "restoka" / "permits.py"] # racj (enrichissement) DOCS_DIR = ROOT / "docs" / "connecteurs" DB_PATH = ROOT / "data" / "restoka.db" SOURCES_JSON = ROOT / "data" / "sources.json" SKIP_MODULES = {"__init__", "base"} URL_RE = re.compile(r"https?://[^\s\"'\\)>,;]+") DATE_RE = re.compile(r"\d{4}-\d{2}-\d{2}") INFRA_HOSTS = ("api.firecrawl.dev", "api.scrapfly.io", "resto-ka.com", "nominatim") FIELDS = [ ("name", "Nom de l'établissement", "name IS NOT NULL AND name != ''"), ("chain", "Chaîne / bannière", "chain IS NOT NULL AND chain != ''"), ("cuisines", "Cuisines (JSON, taxonomie §6.1)", "cuisines IS NOT NULL AND length(cuisines) > 4"), ("establishment_type", "Type d'établissement", "establishment_type IS NOT NULL AND establishment_type != ''"), ("price_range", "Fourchette de prix", "price_range IS NOT NULL AND price_range != ''"), ("address", "Adresse civique", "address IS NOT NULL AND address != ''"), ("city", "Ville", "city IS NOT NULL AND city != ''"), ("region", "Région administrative", "region IS NOT NULL AND region != ''"), ("postal_code", "Code postal", "postal_code IS NOT NULL AND postal_code != ''"), ("lat", "GPS (lat/lng)", "lat IS NOT NULL AND lng IS NOT NULL"), ("phone", "Téléphone", "phone IS NOT NULL AND phone != ''"), ("website", "Site web", "website IS NOT NULL AND website != ''"), ("hours", "Horaires structurés (JSON)", "hours IS NOT NULL AND length(hours) > 4"), ("services", "Services (JSON)", "services IS NOT NULL AND length(services) > 4"), ("dietary_options", "Options alimentaires (JSON)", "dietary_options IS NOT NULL AND length(dietary_options) > 4"), ("images", "Photos (JSON)", "images IS NOT NULL AND length(images) > 4"), ("url", "URL de la fiche source", "url IS NOT NULL AND url != ''"), ] def esc(s: str, limit: int = 100) -> str: s = str(s).replace("\\", "\\\\").replace("|", "\\|") s = re.sub(r"\s+", " ", s).strip() return s[: limit - 1] + "…" if len(s) > limit else s def pct(n, d) -> str: return f"{100.0 * (n or 0) / d:.0f} %" if d else "—" def fmt_ts(ts) -> str: if not ts: return "—" return datetime.fromtimestamp(float(ts)).strftime("%Y-%m-%d %H:%M") def fmt_secs(sec: float) -> str: if sec < 5400: return f"≈ {sec / 60:.0f} min" if sec < 129600: return f"≈ {sec / 3600:.1f} h" return f"≈ {sec / 86400:.1f} j" def example_value(col: str, value) -> str: if value is None or value == "": return "—" if col == "images": try: imgs = json.loads(value) return esc(f"{len(imgs)} photo(s) — {imgs[0]}", 90) if imgs else "—" except (ValueError, TypeError): return esc(value, 90) return esc(value, 90) # -- introspection statique ------------------------------------------------------ def banner_description(text: str, filename: str) -> str: lines, started = [], False for raw in text.splitlines(): if not raw.startswith("#"): if started: break continue body = raw.lstrip("#").strip() if set(body) <= {"-", "="}: continue if not started: if body.startswith("Desc:"): started = True lines.append(body[len("Desc:"):].strip()) continue if re.match(r"^(Author|File):", body): break lines.append(body) return " ".join(l for l in lines if l).strip() def detect_backends(text: str) -> tuple[list[str], str]: detailed, family = [], "direct" if re.search(r"self\.(get|post)\(|requests\.(get|post)\(", text): detailed.append("requests direct (session UA RestoKaBot, throttling poli)") if ".scrapfly(" in text: detailed.append("Scrapfly (asp + render_js — contournement anti-bot)") family = "Scrapfly" if "get_rendered(" in text: detailed.append("Firecrawl (HTML rendu, JavaScript exécuté)") family = "Firecrawl" if family == "direct" else family if not detailed: detailed.append("requests direct") return detailed, family def detect_flavor(text: str) -> str: low = text.lower() if "graphql" in low: return "API GraphQL interne" if "overpass" in low: return "API Overpass (OpenStreetMap)" if "listecondamnation" in low or "donneesquebec" in low or "données québec" in low: return "jeu de données ouvert (CSV, Données Québec)" if "sitemap" in low: return "sitemap XML + pages HTML" if re.search(r"\.json\(\)", text) and re.search(r"api[./_]", low): return "API JSON" return "pages HTML (rendu serveur)" def detect_pagination(text: str, constants: dict) -> str: hits, low = [], text.lower() if re.search(r"[?&]page=|[\"']page[\"']\s*[:=]|paged", low): hits.append("pagination par numéro de page") if re.search(r"[?&]offset=|[\"']offset[\"']", low): hits.append("pagination par offset") if "cursor" in low: hits.append("curseur de pagination") lists = [k for k, v in constants.items() if isinstance(v, (list, tuple)) and len(v) > 1] if lists: hits.append(f"itération sur {len(constants[lists[0]])} racines " f"(constante `{lists[0]}`)") if not hits: hits.append("réponse unique (pas de pagination)") return " ; ".join(hits) def introspect_module(path: Path) -> list[dict]: text = path.read_text(encoding="utf-8") try: tree = ast.parse(text) except SyntaxError: return [] constants: dict = {} for node in tree.body: if isinstance(node, ast.Assign) and len(node.targets) == 1 \ and isinstance(node.targets[0], ast.Name) \ and node.targets[0].id.isupper(): try: constants[node.targets[0].id] = ast.literal_eval(node.value) except (ValueError, TypeError, SyntaxError): seg = ast.get_source_segment(text, node.value) or "" urls = URL_RE.findall(seg) constants[node.targets[0].id] = urls if len(urls) > 1 else \ (urls[0] if urls else None) urls_all = [] for u in URL_RE.findall(text): u = u.rstrip('".') host = u.split("//", 1)[-1].split("/", 1)[0] if "." not in host: # fragment de f-string, pas une vraie URL continue if not any(h in u for h in INFRA_HOSTS) and u not in urls_all: urls_all.append(u) endpoint = None for name in ("BASE", "BASE_URL", "API", "API_URL", "API_BASE", "GRAPHQL", "CSV_URL", "DATASET_URL", "ROOT", "URL"): v = constants.get(name) if isinstance(v, str) and v.startswith("http"): endpoint = v break if not endpoint and urls_all: endpoint = urls_all[0] budgets = {k: v for k, v in constants.items() if isinstance(v, (int, float)) and not isinstance(v, bool) and re.search(r"MAX|CAP|LIMIT|BUDGET|TTL|PER_PAGE|PAGES|DELAY|BATCH", k)} backends, family = detect_backends(text) rel = path.relative_to(ROOT) base = { "module": path.stem, "path": str(rel), "banner": banner_description(text, path.name), "endpoint": endpoint, "urls": urls_all[:5], "budgets": budgets, "backends": backends, "backend_family": family, "flavor": detect_flavor(text), "pagination": detect_pagination(text, constants), } entries = [] for node in tree.body: if not isinstance(node, ast.ClassDef): continue bases = {getattr(b, "id", getattr(b, "attr", "")) for b in node.bases} if "BaseConnector" not in bases: continue attrs = {"request_delay": 0.6, "timeout": 30, "use_detail_cache": True, "source_id": ""} for sub in node.body: if isinstance(sub, ast.Assign) and len(sub.targets) == 1 \ and isinstance(sub.targets[0], ast.Name): try: attrs[sub.targets[0].id] = ast.literal_eval(sub.value) except (ValueError, TypeError, SyntaxError): pass if attrs["source_id"]: entries.append({**base, "class": node.name, "class_doc": ast.get_docstring(node) or "", **attrs}) if not entries and isinstance(constants.get("SOURCE_ID"), str): # module d'enrichissement sans classe (ex. inspections.py / mapaq) entries.append({**base, "class": "(module fonctionnel, pas de classe)", "class_doc": ast.get_docstring(tree) or "", "source_id": constants["SOURCE_ID"], "request_delay": None, "timeout": None, "use_detail_cache": False}) return entries # -- BD live ---------------------------------------------------------------------- def db_stats(con: sqlite3.Connection, sid: str) -> dict: parts = ", ".join( f"sum(CASE WHEN active=1 AND {cond} THEN 1 ELSE 0 END) AS f_{col}" for col, _l, cond in FIELDS) agg = con.execute( f"SELECT count(*) AS total, coalesce(sum(active),0) AS act, " f"min(first_seen) AS first_seen, max(last_seen) AS last_seen, {parts} " f"FROM restaurants WHERE source=?", (sid,)).fetchone() sample = con.execute( "SELECT * FROM restaurants WHERE source=? AND active=1 " "ORDER BY last_seen DESC LIMIT 1", (sid,)).fetchone() menus = con.execute( "SELECT count(*) AS n, coalesce(sum(m.item_count),0) AS items, " "max(m.captured_at) AS fresh, count(DISTINCT m.price_context) AS ctx " "FROM menus m JOIN restaurants r ON r.uid = m.uid WHERE r.source=?", (sid,)).fetchone() inspections = None if sid == "mapaq": inspections = con.execute( "SELECT count(*) AS n, sum(CASE WHEN uid IS NOT NULL THEN 1 ELSE 0 " "END) AS matched, min(date_infraction) AS d0, " "max(date_infraction) AS d1, coalesce(sum(montant_amende),0) " "AS amendes " "FROM inspections").fetchone() runs = con.execute( "SELECT ts, ok, found, added, updated, removed, message FROM sync_log " "WHERE source=? ORDER BY ts DESC LIMIT 60", (sid,)).fetchall() ok_ts = sorted(r["ts"] for r in runs if r["ok"]) cadence = None if len(ok_ts) >= 3: deltas = [b - a for a, b in zip(ok_ts, ok_ts[1:]) if b - a > 60] if deltas: cadence = statistics.median(deltas) return {"agg": agg, "sample": sample, "menus": menus, "inspections": inspections, "runs": runs, "cadence": cadence, "last_ok": next((r for r in runs if r["ok"]), None), "errors": [r for r in runs if not r["ok"]][:5], "err_count": sum(1 for r in runs if not r["ok"])} # -- rendu --------------------------------------------------------------------------- def licence_lines(reg: dict, entry: dict | None) -> list[str]: out = [f"- **Cadre d'accès (registre, `acces_legal`)** : " f"{reg.get('acces_legal', '—')}"] blob = (reg.get("acces_legal") or "") + (reg.get("extraction") or "") low = blob.lower() if "odbl" in low or "openstreetmap" in low: out.append("- **Licence** : ODbL — « Données © contributeurs " "OpenStreetMap » ; attribution affichée sur les pages " "Sources et le pied de page de Resto-Ka.") elif "cc-by" in low or "données québec" in low or ( entry and "donneesquebec" in " ".join(entry.get("urls", []))): out.append("- **Licence** : donnée ouverte CC-BY 4.0 (Données Québec) " "— mention de la source « MAPAQ / Données Québec » affichée.") else: out.append("- **Scraping / API** : User-Agent identifiable " "`RestoKaBot/1.0 (+https://www.resto-ka.com/bot; " "contact@spboucher.ai)`, throttling poli, aucun " "contournement d'accès ; les fiches pointent vers la " "source d'origine.") out.append("- Retrait sur demande : contact@spboucher.ai.") return out def render_fiche(reg: dict, entry: dict | None, stats: dict | None, now: str) -> str: sid = reg["id"] name = reg.get("name", sid) status = reg.get("status", "—") etat = status.split("—")[0].split("(")[0].strip() out = [f"# {name} — connecteur `{sid}`", "", f"_Fiche générée automatiquement par " f"`scripts/gen_connector_docs.py` le {now} — ne pas éditer à la " f"main, régénérer._", ""] agg = stats["agg"] if stats else None vol = f" · Restos actifs : {agg['act']}/{agg['total']}" if agg and \ agg["total"] else "" fam = entry["backend_family"] if entry else "—" out.append(f"**État : {etat}** · Palier (tier) : {reg.get('tier', '—')} · " f"Backend : {fam}{vol}") out.append("") out.append("## Description de la source") out.append("") if entry and entry["banner"]: out.append(entry["banner"]) out.append("") out.append(f"- **Plateforme** : {reg.get('platform', '—')} · **Site** : " f"{reg.get('url', '—')}") out.append(f"- **Extraction (registre)** : {reg.get('extraction', '—')}") out.append(f"- **Contexte de prix** : {reg.get('price_context') or '—'}") if entry: out.append(f"- **Module** : `{entry['path']}` — `{entry['class']}`") else: out.append("- **Module** : aucun (connecteur à écrire — voir statut)") if reg.get("integrations"): n = len(reg["integrations"]) inact = sum(1 for i in reg["integrations"] if i.get("status") == "inactif") out.append(f"- **Intégrations recensées** : {n} clés GUID " f"({n - inact} actives, {inact} retirées) — voir " f"`data/sources.json` et `data/ueat-discovered.json`") out.append("") out.append("## Accès") out.append("") if entry: out.append(f"- **Type d'accès** : {entry['flavor']}") out.append(f"- **Endpoint de base** : {entry['endpoint'] or '—'}") if entry["urls"]: out.append("- **URLs du module** : " + " · ".join(entry["urls"][:4])) out.append(f"- **Pagination** : {entry['pagination']}") out.append(f"- **Backend anti-bot / rendu** : " f"{' ; '.join(entry['backends'])}") if entry["request_delay"] is not None: out.append(f"- **Politesse** : {entry['request_delay']} s entre " f"requêtes, timeout {entry['timeout']} s, UA " f"`RestoKaBot/1.0 (+https://www.resto-ka.com/bot)`") auth = "clé API requise (voir statut)" if "clé" in status else \ "aucune — accès public/anonyme" out.append(f"- **Authentification** : {auth}") else: out.append(f"- Connecteur non écrit — accès prévu (registre) : " f"{reg.get('extraction', '—')}") out.append("") out.append("## Champs récupérés → schéma cible") out.append("") if sid == "mapaq": out.append("Source d'**enrichissement** : alimente la table " "`inspections` (exploitant, établissement, adresse, dates, " "amende, motif) puis croisement conservateur avec " "`restaurants.uid` (nom normalisé + ville/code postal, ou " "code postal + civique + similarité de nom — colonne " "`matched_by`). Pas de fiches restaurant propres.") out.append("") elif agg and agg["total"]: out.append(f"Le connecteur alimente la table `restaurants` (et `menus` " f"le cas échéant). Complétude mesurée en SQL sur les " f"{agg['act']} fiches actives ; exemple tiré d'une ligne " f"réelle de la BD.") out.append("") out.append("| Colonne `restaurants` | Contenu | Renseignée (actives) " "| Exemple réel |") out.append("|---|---|---|---|") sample = stats["sample"] for col, label, _c in FIELDS: ex = example_value(col, sample[col]) if sample is not None else "—" out.append(f"| `{col}` | {label} | {pct(agg[f'f_{col}'], agg['act'])} " f"| {ex} |") out.append("") m = stats["menus"] if m and m["n"]: out.append(f"**Menus** : {m['n']} menus rattachés " f"({m['items']} items, {m['ctx']} contexte(s) de prix, " f"dernière capture {esc(m['fresh'] or '—', 20)}) — table " f"`menus` (sections → items → options, prix CAD).") out.append("") else: out.append("Aucune fiche en BD pour cette source (connecteur en " "attente ou clé manquante) — schéma cible : table " "`restaurants` + `menus`.") out.append("") out.append("## Fréquence & budget") out.append("") out.append(f"- **Cadence déclarée (registre)** : {reg.get('cadence', '—')}") if stats: cad = fmt_secs(stats["cadence"]) if stats["cadence"] else "—" out.append(f"- **Cadence observée** (médiane sync_log) : {cad}") lo = stats["last_ok"] if lo: out.append(f"- **Dernier passage OK** : {fmt_ts(lo['ts'])} — " f"{lo['found'] or 0} trouvées, +{lo['added'] or 0} / " f"~{lo['updated'] or 0} / -{lo['removed'] or 0}") if entry and entry["budgets"]: caps = ", ".join(f"`{k}` = {v}" for k, v in sorted(entry["budgets"].items())) out.append(f"- **Caps / budgets du module** : {caps}") if entry and entry.get("use_detail_cache"): out.append("- **Cache des payloads détail** : activé (table " "`detail_cache`)") out.append("") out.append("## Volumétrie & complétude") out.append("") if agg and agg["total"]: out.append(f"- **Fiches en BD** : {agg['total']} au total, " f"**{agg['act']} actives**") out.append(f"- **Première ingestion** : {fmt_ts(agg['first_seen'])[:10]} " f"· **Dernière observation** : {fmt_ts(agg['last_seen'])[:10]}") out.append(f"- **Complétude clé (actives)** : GPS " f"{pct(agg['f_lat'], agg['act'])} · adresse " f"{pct(agg['f_address'], agg['act'])} · cuisines " f"{pct(agg['f_cuisines'], agg['act'])} · horaires " f"{pct(agg['f_hours'], agg['act'])} · site web " f"{pct(agg['f_website'], agg['act'])}") if stats and stats["inspections"] and stats["inspections"]["n"]: i = stats["inspections"] amendes = f"{i['amendes']:,.0f}".replace(",", " ") out.append(f"- **Inspections MAPAQ** : {i['n']} condamnations " f"({i['matched']} croisées avec un resto, amendes cumulées " f"{amendes} $), infractions de {i['d0']} à {i['d1']}") if not (agg and agg["total"]) and not (stats and stats["inspections"]): out.append("- Aucune donnée en BD pour cette source.") if stats: out.append(f"- **Runs journalisés (60 derniers)** : " f"{len(stats['runs'])}, dont {stats['err_count']} en erreur") out.append("") out.append("## Erreurs connues & dépannage") out.append("") if stats and stats["errors"]: out.append("| Date | Message (sync_log) |") out.append("|---|---|") for r in stats["errors"]: out.append(f"| {fmt_ts(r['ts'])} | {esc(r['message'] or '', 160)} |") out.append("") else: out.append("Aucune erreur dans les 60 derniers runs journalisés.") out.append("") if etat != "actif": out.append(f"**Note du registre** : {status}") out.append("") out.append(f"Rejouer la source seule : `python3 run.py sync {sid}` · " f"vérifier `sync_log` (`SELECT * FROM sync_log WHERE " f"source='{sid}' ORDER BY ts DESC LIMIT 5;`).") out.append("") out.append("## Licence, attribution & conditions") out.append("") out.extend(licence_lines(reg, entry)) out.append("") out.append("## Historique") out.append("") if agg and agg["first_seen"]: out.append(f"- {fmt_ts(agg['first_seen'])[:10]} — premières fiches de " f"la source ingérées dans la BD.") blob = " ".join(str(reg.get(k, "")) for k in ("status", "notes", "extraction", "acces_legal")) for d in sorted({m.group(0) for m in DATE_RE.finditer(blob)}): out.append(f"- {d} — date mentionnée au registre (voir `status`/notes).") out.append("- 2026-08-18 — vague d'enrichissement : standardisation de la " "documentation des connecteurs (fiche générée par " "`scripts/gen_connector_docs.py`).") out.append("") return "\n".join(out) def main() -> None: now = datetime.now().strftime("%Y-%m-%d %H:%M") registry = json.loads(SOURCES_JSON.read_text(encoding="utf-8"))["sources"] con = sqlite3.connect(DB_PATH) con.row_factory = sqlite3.Row by_sid: dict[str, dict] = {} for path in sorted(CONN_DIR.glob("*.py")) + EXTRA_MODULES: if path.stem in SKIP_MODULES or not path.exists(): continue for e in introspect_module(path): by_sid[e["source_id"]] = e DOCS_DIR.mkdir(parents=True, exist_ok=True) for old in DOCS_DIR.glob("*.md"): old.unlink() rows = [] for reg in registry: sid = reg["id"] entry = by_sid.get(sid) stats = db_stats(con, sid) (DOCS_DIR / f"{sid}.md").write_text( render_fiche(reg, entry, stats, now), encoding="utf-8") agg = stats["agg"] lo = stats["last_ok"] etat = reg.get("status", "—").split("—")[0].split("(")[0].strip() rows.append( f"| [`{sid}`]({sid}.md) | {esc(reg.get('name', sid), 40)} " f"| T{reg.get('tier', '—')} " f"| {esc(entry['flavor'] if entry else '—', 40)} " f"| {entry['backend_family'] if entry else '—'} " f"| {agg['act']}/{agg['total']} " f"| {pct(agg['f_lat'], agg['act'])} " f"| {pct(agg['f_hours'], agg['act'])} " f"| {pct(agg['f_cuisines'], agg['act'])} " f"| {esc(etat, 30)} | {fmt_ts(lo['ts']) if lo else '—'} |") total_act = con.execute( "SELECT coalesce(sum(active),0) FROM restaurants").fetchone()[0] idx = [ "# Resto-Ka — Index des connecteurs", "", f"_Généré automatiquement par `scripts/gen_connector_docs.py` le {now} " f"— ne pas éditer à la main, régénérer._", "", f"**{len(registry)} sources au registre** · **{total_act} restos " f"actifs** en BD.", "", "| Source | Nom | Tier | Type d'accès | Backend | Actifs/Total | GPS " "| Horaires | Cuisines | État | Dernier sync OK |", "|---|---|---|---|---|---|---|---|---|---|---|", ] idx.extend(rows) idx.append("") (DOCS_DIR / "INDEX.md").write_text("\n".join(idx), encoding="utf-8") con.close() print(f"[gen_connector_docs] {len(registry)} fiches + INDEX.md écrits dans " f"{DOCS_DIR}") if __name__ == "__main__": main()