Créa·Ka — annuaire public cross-plateforme des créateurs de contenu québécois (crea-ka.com)
1#!/usr/bin/env python32# ==============================================================================3# Author: Simon-Pierre Boucher <contact@spboucher.ai>4# File: scripts/gen_connector_docs.py5# Desc: Documentation STANDARDISÉE des connecteurs Créa-Ka — génère6# docs/connecteurs/INDEX.md + une fiche docs/connecteurs/<id>.md par7# source du registre, de façon 100 % programmatique et rejouable :8# 1. registre data/sources.json (famille, palier, accès, cadence,9# signal d'identité, notes, statut) ;10# 2. introspection STATIQUE (ast) de creaka/connectors/*.py :11# classe, kind (discovery/enrichment), backend, endpoints,12# pagination, constantes de budget — sans exécuter le code ;13# 3. BD live data/creaka.db : créateurs découverts/enrichis par14# source (doc JSON), complétude des champs, comptes par15# plateforme, dernier sync, cadence observée, alertes récentes.16# Usage : python3 scripts/gen_connector_docs.py (racine du projet)17# ==============================================================================18from __future__ import annotations1920import ast21import json22import re23import sqlite324import statistics25from datetime import datetime, timezone26from pathlib import Path2728ROOT = Path(__file__).resolve().parents[1]29CONN_DIR = ROOT / "creaka" / "connectors"30DOCS_DIR = ROOT / "docs" / "connecteurs"31DB_PATH = ROOT / "data" / "creaka.db"32SOURCES_JSON = ROOT / "data" / "sources.json"3334SKIP_MODULES = {"__init__", "base"}35URL_RE = re.compile(r"https?://[^\s\"'\\)>,;]+")36DATE_RE = re.compile(r"\d{4}-\d{2}-\d{2}")37INFRA_HOSTS = ("api.firecrawl.dev", "api.scrapfly.io", "crea-ka.com")3839# plateformes de la table `accounts` alimentées par chaque source d'enrichissement40SOURCE_PLATFORMS = {41 "instagram-profil": ("instagram",),42 "tiktok-profil": ("tiktok",),43 "youtube": ("youtube",),44 "youtube-recherche": ("youtube",),45 "twitch": ("twitch",),46 "x-profil": ("x",),47 "balados-rss": ("podcast",),48 "balados-itunes": ("podcast",),49 "podcastindex": ("podcast",),50 "onlyqueb": ("onlyfans",),51}5253# champs du doc créateur documentés (complétude par source de découverte)54DOC_FIELDS = [55 ("display_name", "Nom public"),56 ("bio", "Bio"),57 ("region", "Région"),58 ("city", "Ville"),59 ("niches", "Niches (liste)"),60 ("languages", "Langues (liste)"),61 ("creator_type", "Type de créateur"),62 ("primary_platform", "Plateforme principale"),63 ("platforms", "Comptes sociaux (liste)"),64 ("avatar_url", "Avatar"),65 ("total_reach", "Portée totale (abonnés cumulés)"),66 ("link_in_bio_url", "Lien-en-bio"),67]6869ACC_FIELDS = [70 ("handle", "Handle", "handle IS NOT NULL AND handle != ''"),71 ("url", "URL du profil", "url IS NOT NULL AND url != ''"),72 ("followers", "Abonnés", "followers IS NOT NULL"),73 ("verified", "Badge vérifié", "verified IS NOT NULL"),74 ("confidence", "Confiance d'identité", "confidence IS NOT NULL"),75 ("signal", "Signal d'identité", "signal IS NOT NULL AND signal != ''"),76 ("metrics", "Métriques détaillées (JSON)",77 "metrics IS NOT NULL AND length(metrics) > 4"),78 ("last_checked", "Dernière vérification",79 "last_checked IS NOT NULL AND last_checked != ''"),80]818283def esc(s, limit: int = 100) -> str:84 s = str(s).replace("\\", "\\\\").replace("|", "\\|")85 s = re.sub(r"\s+", " ", s).strip()86 return s[: limit - 1] + "…" if len(s) > limit else s878889def pct(n, d) -> str:90 return f"{100.0 * (n or 0) / d:.0f} %" if d else "—"919293def parse_iso(ts: str) -> float | None:94 try:95 return datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp()96 except (ValueError, AttributeError):97 return None9899100def fmt_secs(sec: float) -> str:101 if sec < 5400:102 return f"≈ {sec / 60:.0f} min"103 if sec < 129600:104 return f"≈ {sec / 3600:.1f} h"105 return f"≈ {sec / 86400:.1f} j"106107108def filled(v) -> bool:109 return bool(v)110111112# -- introspection statique ------------------------------------------------------113114def banner_description(text: str) -> str:115 lines, started = [], False116 for raw in text.splitlines():117 if not raw.startswith("#"):118 if started:119 break120 continue121 body = raw.lstrip("#").strip()122 if set(body) <= {"-", "="}:123 continue124 if not started:125 if body.startswith("Desc:"):126 started = True127 lines.append(body[len("Desc:"):].strip())128 continue129 if re.match(r"^(Author|File):", body):130 break131 lines.append(body)132 return " ".join(l for l in lines if l).strip()133134135def introspect_module(path: Path) -> list[dict]:136 text = path.read_text(encoding="utf-8")137 try:138 tree = ast.parse(text)139 except SyntaxError:140 return []141 constants: dict = {}142 for node in tree.body:143 if isinstance(node, ast.Assign) and len(node.targets) == 1 \144 and isinstance(node.targets[0], ast.Name) \145 and node.targets[0].id.isupper():146 try:147 constants[node.targets[0].id] = ast.literal_eval(node.value)148 except (ValueError, TypeError, SyntaxError):149 seg = ast.get_source_segment(text, node.value) or ""150 urls = URL_RE.findall(seg)151 constants[node.targets[0].id] = urls if len(urls) > 1 else \152 (urls[0] if urls else None)153 urls_all = []154 for u in URL_RE.findall(text):155 u = u.rstrip('".')156 host = u.split("//", 1)[-1].split("/", 1)[0]157 if "." not in host: # fragment de f-string, pas une vraie URL158 continue159 if not any(h in u for h in INFRA_HOSTS) and u not in urls_all:160 urls_all.append(u)161 endpoint = None162 for name in ("BASE", "BASE_URL", "API", "API_URL", "API_BASE", "SPARQL",163 "ENDPOINT", "ROOT", "URL", "SEARCH_URL", "LOOKUP_URL"):164 v = constants.get(name)165 if isinstance(v, str) and v.startswith("http"):166 endpoint = v167 break168 if not endpoint and urls_all:169 endpoint = urls_all[0]170 budgets = {k: v for k, v in constants.items()171 if isinstance(v, (int, float)) and not isinstance(v, bool)172 and re.search(r"MAX|CAP|LIMIT|BUDGET|TTL|PER_PAGE|PAGES|DELAY|BATCH",173 k)}174 detailed, family = [], "direct"175 if re.search(r"self\.(get|post)\(|requests\.(get|post)\(", text):176 detailed.append("requests direct (session UA CreaKaBot, throttling poli)")177 if ".scrapfly(" in text:178 detailed.append("Scrapfly (asp + render_js — contournement anti-bot)")179 family = "Scrapfly"180 if "get_rendered(" in text:181 detailed.append("Firecrawl (HTML rendu, JavaScript exécuté)")182 family = "Firecrawl" if family == "direct" else family183 if not detailed:184 detailed.append("requests direct")185 low = text.lower()186 if "sparql" in low:187 flavor = "API SPARQL publique (Wikidata)"188 elif "graphql" in low:189 flavor = "API GraphQL"190 elif "itunes.apple.com" in low:191 flavor = "API iTunes Search/Lookup (publique)"192 elif "rss" in low and "feedparser" in low or "<rss" in low:193 flavor = "flux RSS publics"194 elif "sitemap" in low:195 flavor = "sitemap XML + JSON-LD des pages profil (SSR)"196 elif "json-ld" in low or "jsonld" in low or "profilepage" in low:197 flavor = "JSON-LD des pages profil publiques"198 elif re.search(r"\.json\(\)", text) and re.search(r"api[./_]", low):199 flavor = "API JSON"200 elif "dataset" in low or "seed" in low and "csv" in low:201 flavor = "dataset local compilé (data/seed)"202 else:203 flavor = "pages HTML publiques"204 hits = []205 if re.search(r"[?&]page=|[\"']page[\"']\s*[:=]", low):206 hits.append("pagination par numéro de page")207 if re.search(r"[?&]offset=|[\"']offset[\"']", low):208 hits.append("pagination par offset")209 if "cursor" in low or "continuation" in low:210 hits.append("curseur / continuation")211 if not hits:212 hits.append("réponse unique (pas de pagination)")213 base = {"module": path.stem, "path": f"creaka/connectors/{path.stem}.py",214 "banner": banner_description(text), "endpoint": endpoint,215 "urls": urls_all[:5], "budgets": budgets, "backends": detailed,216 "backend_family": family, "flavor": flavor,217 "pagination": " ; ".join(hits)}218 entries = []219 for node in tree.body:220 if not isinstance(node, ast.ClassDef):221 continue222 bases = {getattr(b, "id", getattr(b, "attr", "")) for b in node.bases}223 if "BaseConnector" not in bases:224 continue225 attrs = {"request_delay": 0.8, "timeout": 30, "kind": "discovery",226 "source_id": ""}227 for sub in node.body:228 if isinstance(sub, ast.Assign) and len(sub.targets) == 1 \229 and isinstance(sub.targets[0], ast.Name):230 try:231 attrs[sub.targets[0].id] = ast.literal_eval(sub.value)232 except (ValueError, TypeError, SyntaxError):233 pass234 if attrs["source_id"]:235 entries.append({**base, "class": node.name,236 "class_doc": ast.get_docstring(node) or "",237 **attrs})238 return entries239240241# -- BD live ----------------------------------------------------------------------242243def load_creators(con) -> list[dict]:244 out = []245 for (doc,) in con.execute("SELECT doc FROM creators"):246 try:247 out.append(json.loads(doc))248 except ValueError:249 pass250 return out251252253def db_stats(con, sid: str, creators: list[dict]) -> dict:254 mine = [c for c in creators if c.get("source") == sid]255 touched = sum(1 for c in creators256 if sid in (c.get("source_ids") or []) or c.get("source") == sid)257 doc_fill = {f: sum(1 for c in mine if filled(c.get(f)))258 for f, _l in DOC_FIELDS}259 sample = max(mine, key=lambda c: c.get("total_reach") or 0) if mine else None260261 acc = None262 acc_sample = None263 platforms = SOURCE_PLATFORMS.get(sid)264 if platforms:265 ph = ",".join("?" * len(platforms))266 parts = ", ".join(267 f"sum(CASE WHEN {cond} THEN 1 ELSE 0 END) AS f_{col}"268 for col, _l, cond in ACC_FIELDS)269 acc = con.execute(270 f"SELECT count(*) AS n, avg(confidence) AS conf, "271 f"max(last_checked) AS checked, {parts} FROM accounts "272 f"WHERE platform IN ({ph})", platforms).fetchone()273 acc_sample = con.execute(274 f"SELECT * FROM accounts WHERE platform IN ({ph}) AND followers IS "275 f"NOT NULL ORDER BY followers DESC LIMIT 1", platforms).fetchone()276277 runs = con.execute(278 "SELECT ts, creators, accounts, avg_confidence, added, updated, "279 "errors, seconds, alert FROM sync_log WHERE source=? "280 "ORDER BY ts DESC LIMIT 60", (sid,)).fetchall()281 ok_ts = sorted(t for t in (parse_iso(r["ts"]) for r in runs) if t)282 cadence = None283 if len(ok_ts) >= 3:284 deltas = [b - a for a, b in zip(ok_ts, ok_ts[1:]) if b - a > 60]285 if deltas:286 cadence = statistics.median(deltas)287 alerts = [r for r in runs if (r["errors"] or 0) > 0 or r["alert"]][:5]288 return {"mine": len(mine), "touched": touched, "doc_fill": doc_fill,289 "sample": sample, "acc": acc, "acc_sample": acc_sample,290 "platforms": platforms, "runs": runs, "cadence": cadence,291 "last": runs[0] if runs else None, "alerts": alerts}292293294# -- rendu ---------------------------------------------------------------------------295296def render_fiche(reg: dict, entry: dict | None, st: dict, now: str) -> str:297 sid = reg["id"]298 etat = reg.get("statut", "actif")299 fam = reg.get("famille", "—")300 out = [f"# `{sid}` — connecteur {fam} (palier {reg.get('palier', '—')})", "",301 f"_Fiche générée automatiquement par "302 f"`scripts/gen_connector_docs.py` le {now} — ne pas éditer à la "303 f"main, régénérer._", ""]304 kind = entry.get("kind") if entry else fam305 out.append(f"**État : {esc(etat, 80)}** · Famille : {fam} ({kind}) · "306 f"Backend : {entry['backend_family'] if entry else '—'} · "307 f"Créateurs découverts : {st['mine']} · touchés : "308 f"{st['touched']}")309 out.append("")310311 out.append("## Description de la source")312 out.append("")313 if entry and entry["banner"]:314 out.append(entry["banner"])315 out.append("")316 out.append(f"- **Notes (registre)** : {reg.get('notes', '—')}")317 out.append(f"- **Signal d'identité** : {reg.get('signal_identite', '—')} "318 f"(colonne `accounts.signal` / `confidence`)")319 if entry:320 out.append(f"- **Module** : `{entry['path']}` — classe "321 f"`{entry['class']}` (`kind = \"{entry['kind']}\"`)")322 else:323 out.append("- **Module** : introuvable (vérifier le registre)")324 out.append("")325326 out.append("## Accès")327 out.append("")328 out.append(f"- **Accès (registre)** : {reg.get('acces', '—')}")329 if entry:330 out.append(f"- **Type d'accès (code)** : {entry['flavor']}")331 out.append(f"- **Endpoint de base** : {entry['endpoint'] or '—'}")332 if entry["urls"]:333 out.append("- **URLs du module** : " + " · ".join(entry["urls"][:4]))334 out.append(f"- **Pagination** : {entry['pagination']}")335 out.append(f"- **Backend anti-bot / rendu** : "336 f"{' ; '.join(entry['backends'])}")337 out.append(f"- **Politesse** : {entry['request_delay']} s entre "338 f"requêtes, timeout {entry['timeout']} s, UA "339 f"`CreaKaBot/1.0 (+https://www.crea-ka.com/bot)`")340 needs_key = "clé" in (reg.get("acces") or "").lower() \341 or "clé" in str(etat).lower()342 auth = ("clé(s) API requise(s) — voir statut/registre" if needs_key343 else "aucune — contenu public / API anonyme")344 out.append(f"- **Authentification** : {auth}")345 out.append("")346347 out.append("## Champs récupérés → schéma cible")348 out.append("")349 if st["mine"]:350 out.append(f"Source de **découverte** : produit des fiches `Creator` "351 f"(tables `creators` + `accounts`). Complétude mesurée sur "352 f"les {st['mine']} créateurs découverts par cette source "353 f"(champ `doc.source`) ; exemple tiré d'une fiche réelle.")354 out.append("")355 out.append("| Champ du doc créateur | Contenu | Renseigné | Exemple réel |")356 out.append("|---|---|---|---|")357 s = st["sample"] or {}358 for f, label in DOC_FIELDS:359 v = s.get(f)360 if f == "platforms" and v:361 ex = esc(", ".join(f"{p.get('platform')}:@{p.get('handle')}"362 for p in v[:3]), 90)363 elif isinstance(v, list):364 ex = esc(", ".join(map(str, v[:4])), 90)365 else:366 ex = esc(v, 90) if filled(v) else "—"367 out.append(f"| `{f}` | {label} | "368 f"{pct(st['doc_fill'][f], st['mine'])} | {ex} |")369 out.append("")370 if st["acc"] is not None and st["acc"]["n"]:371 a = st["acc"]372 plats = ", ".join(st["platforms"])373 out.append(f"Cible d'**enrichissement** : colonnes de la table "374 f"`accounts` pour la/les plateforme(s) `{plats}` "375 f"({a['n']} comptes en BD, confiance moyenne "376 f"{a['conf']:.2f}).")377 out.append("")378 out.append("| Colonne `accounts` | Contenu | Renseignée | Exemple réel |")379 out.append("|---|---|---|---|")380 sm = st["acc_sample"]381 for col, label, _c in ACC_FIELDS:382 ex = esc(sm[col], 90) if sm is not None and sm[col] not in (383 None, "") else "—"384 out.append(f"| `{col}` | {label} | {pct(a[f'f_{col}'], a['n'])} "385 f"| {ex} |")386 out.append("")387 if not st["mine"] and (st["acc"] is None or not st["acc"]["n"]):388 out.append("Aucune donnée attribuable à cette source en BD pour "389 "l'instant (clés manquantes ou source en attente) — schéma "390 "cible : `creators` + `accounts`.")391 out.append("")392393 out.append("## Fréquence & budget")394 out.append("")395 out.append(f"- **Cadence déclarée (registre)** : {reg.get('cadence', '—')}")396 cad = fmt_secs(st["cadence"]) if st["cadence"] else "—"397 out.append(f"- **Cadence observée** (médiane sync_log) : {cad}")398 last = st["last"]399 if last:400 out.append(f"- **Dernier passage** : {esc(last['ts'], 20)} — "401 f"{last['creators'] or 0} créateurs, {last['accounts'] or 0} "402 f"comptes, +{last['added'] or 0} / ~{last['updated'] or 0}, "403 f"{last['errors'] or 0} erreur(s), "404 f"{(last['seconds'] or 0):.0f} s")405 if entry and entry["budgets"]:406 caps = ", ".join(f"`{k}` = {v}" for k, v in sorted(entry["budgets"].items()))407 out.append(f"- **Caps / budgets du module** : {caps}")408 if entry:409 out.append(f"- **Throttling** : {entry['request_delay']} s entre "410 f"requêtes (`request_delay`)")411 out.append("")412413 out.append("## Volumétrie & complétude")414 out.append("")415 out.append(f"- **Créateurs découverts par la source** (`doc.source`) : "416 f"{st['mine']} · **fiches touchées** (`source_ids`) : "417 f"{st['touched']}")418 if st["mine"]:419 out.append(f"- **Complétude clé (découverts)** : niches "420 f"{pct(st['doc_fill']['niches'], st['mine'])} · avatar "421 f"{pct(st['doc_fill']['avatar_url'], st['mine'])} · portée "422 f"{pct(st['doc_fill']['total_reach'], st['mine'])} · région "423 f"{pct(st['doc_fill']['region'], st['mine'])}")424 if st["acc"] is not None and st["acc"]["n"]:425 a = st["acc"]426 out.append(f"- **Comptes `{', '.join(st['platforms'])}` en BD** : "427 f"{a['n']} — abonnés {pct(a['f_followers'], a['n'])} · "428 f"métriques {pct(a['f_metrics'], a['n'])} · dernière "429 f"vérification {esc(a['checked'] or '—', 20)}")430 out.append(f"- **Runs journalisés (60 derniers)** : {len(st['runs'])}, "431 f"dont {sum(1 for r in st['runs'] if (r['errors'] or 0) > 0)} "432 f"avec erreurs")433 out.append("")434435 out.append("## Erreurs connues & dépannage")436 out.append("")437 if st["alerts"]:438 out.append("| Passage | Erreurs | Alerte (sync_log) |")439 out.append("|---|---|---|")440 for r in st["alerts"]:441 out.append(f"| {esc(r['ts'], 20)} | {r['errors'] or 0} | "442 f"{esc(r['alert'] or '—', 160)} |")443 out.append("")444 else:445 out.append("Aucune erreur ni alerte dans les 60 derniers runs "446 "journalisés.")447 out.append("")448 if str(etat) != "actif":449 out.append(f"**Statut du registre** : {etat}")450 out.append("")451 out.append(f"Rejouer la source seule : `python3 run.py sync {sid}` · "452 f"vérifier `sync_log` (`SELECT * FROM sync_log WHERE "453 f"source='{sid}' ORDER BY ts DESC LIMIT 5;`).")454 out.append("")455456 out.append("## Licence, attribution & conditions")457 out.append("")458 out.append(f"- **Cadre d'accès (registre)** : {reg.get('acces', '—')}")459 low = (reg.get("acces") or "").lower()460 if "wikidata" in low or "sparql" in low:461 out.append("- **Licence** : données Wikidata sous CC0 — réutilisation "462 "libre, mention « Source : Wikidata » affichée par "463 "courtoisie.")464 elif "dataset local" in low or "listes" in sid:465 out.append("- **Données compilées** de listes et palmarès médias "466 "**publics** ; seuls des faits publics (nom, handle, "467 "audience approximative) sont conservés.")468 elif "api" in low and ("publique" in low or "itunes" in low):469 out.append("- **API publique** utilisée selon ses conditions (pas de "470 "clé détournée, throttling poli) ; données limitées aux "471 "profils publics.")472 else:473 out.append("- **Profils publics uniquement** : UA identifiable "474 "`CreaKaBot/1.0 (+https://www.crea-ka.com/bot; "475 "contact@spboucher.ai)`, throttling poli, respect des "476 "opt-out (`data/optout.json`) et du volet éthique "477 "(`creaka/ethics.py` — mineurs exclus via `is_minor`).")478 out.append("- Retrait sur demande : contact@spboucher.ai (opt-out honoré "479 "à la prochaine ingestion).")480 out.append("")481482 out.append("## Historique")483 out.append("")484 blob = " ".join(str(reg.get(k, "")) for k in ("notes", "statut", "acces"))485 for d in sorted({m.group(0) for m in DATE_RE.finditer(blob)}):486 out.append(f"- {d} — date mentionnée au registre (voir notes/statut).")487 if st["runs"]:488 first = st["runs"][-1]["ts"]489 out.append(f"- {str(first)[:10]} — plus ancien passage journalisé dans "490 f"`sync_log` (fenêtre des 60 derniers).")491 out.append("- 2026-08-18 — vague d'enrichissement : standardisation de la "492 "documentation des connecteurs (fiche générée par "493 "`scripts/gen_connector_docs.py`).")494 out.append("")495 return "\n".join(out)496497498def main() -> None:499 now = datetime.now().strftime("%Y-%m-%d %H:%M")500 registry = json.loads(SOURCES_JSON.read_text(encoding="utf-8"))["sources"]501 con = sqlite3.connect(DB_PATH)502 con.row_factory = sqlite3.Row503 creators = load_creators(con)504505 by_sid: dict[str, dict] = {}506 for path in sorted(CONN_DIR.glob("*.py")):507 if path.stem in SKIP_MODULES:508 continue509 for e in introspect_module(path):510 by_sid[e["source_id"]] = e511512 DOCS_DIR.mkdir(parents=True, exist_ok=True)513 for old in DOCS_DIR.glob("*.md"):514 old.unlink()515516 rows = []517 for reg in registry:518 sid = reg["id"]519 entry = by_sid.get(sid)520 st = db_stats(con, sid, creators)521 (DOCS_DIR / f"{sid}.md").write_text(522 render_fiche(reg, entry, st, now), encoding="utf-8")523 last = st["last"]524 acc = st["acc"]525 comptes = (f"{acc['n']} ({', '.join(st['platforms'])})"526 if acc is not None and acc["n"] else "—")527 rows.append(528 f"| [`{sid}`]({sid}.md) | {reg.get('famille', '—')} "529 f"| P{reg.get('palier', '—')} "530 f"| {esc(entry['flavor'] if entry else '—', 42)} "531 f"| {entry['backend_family'] if entry else '—'} "532 f"| {st['mine']} | {comptes} "533 f"| {esc(str(reg.get('statut', 'actif')).split('—')[0], 26)} "534 f"| {esc(last['ts'], 17) if last else '—'} |")535536 n_creators = con.execute("SELECT count(*) FROM creators").fetchone()[0]537 n_accounts = con.execute("SELECT count(*) FROM accounts").fetchone()[0]538 idx = [539 "# Créa-Ka — Index des connecteurs", "",540 f"_Généré automatiquement par `scripts/gen_connector_docs.py` le {now} "541 f"— ne pas éditer à la main, régénérer._", "",542 f"**{len(registry)} sources au registre** · **{n_creators} créateurs** "543 f"et **{n_accounts} comptes** en BD.", "",544 "| Source | Famille | Palier | Type d'accès | Backend | Découverts "545 "| Comptes (plateforme) | État | Dernier passage |",546 "|---|---|---|---|---|---|---|---|---|",547 ]548 idx.extend(rows)549 idx.append("")550 (DOCS_DIR / "INDEX.md").write_text("\n".join(idx), encoding="utf-8")551 con.close()552 print(f"[gen_connector_docs] {len(registry)} fiches + INDEX.md écrits dans "553 f"{DOCS_DIR}")554555556if __name__ == "__main__":557 main()558