SPB Git
spboucher.ai

spb/crea-ka

Public

Créa·Ka — annuaire public cross-plateforme des créateurs de contenu québécois (crea-ka.com)

quebeccreateursinfluenceursannuaireka

Correctifs audit : fix 500 threads, durées sync, twitch sauté proprement, onlyqueb RSC, avatars YouTube, balados enrichis, archivage §13

- web.py : connexion SQLite PAR THREAD (threading.local) — corrige le 500
  intermittent sur /api/stats (curseurs entrelacés sur connexion partagée)
- db.py : sync_log.seconds/errors réels persistés ; log des comptes retirés
  dans _write_creator ; archive_missing() (§13 — non revu depuis 72 h →
  inactive, réversible au retour, opt-out intouchés)
- ingest.py : durée+erreurs transmises à sync_source ; archivage en fin de
  passage complet ; SkipSource journalisé sans fausse alerte
- twitch : clés absentes → SkipSource (« sauté : clés manquantes »), plus de
  run 0 ni d alerte « source bloquée »
- onlyqueb : parseur du payload Next.js/RSC — displayName réel, bio+headline,
  categories→niches, mym/fansly/ofUrl→comptes, websiteUrl→compte site-web
- normalize : plateformes mym + fansly (URL canoniques + détection)
- dedup : le statut de l observation fraîche gagne (dormant/retour) ; un vrai
  nom public remplace un display_name pseudo-handle
- youtube-recherche : avatar_url extrait du channelRenderer (meilleure rés.)
- balados-itunes : feed_url/artist_id/last_episode dans le doc, genres[]
  complets → niches, dormant >12 mois → status inactive (§13)
- wikidata-qc : échec SPARQL journalisé + compté dans sync_log.errors
- frontend : libellés MYM/Fansly

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Aug 18, 2026) parent 5f6f958 Browse files

13 changed files +227 −31

13 changed files
Unified Split
modified creaka/connectors/balados_itunes.py +33 −4
@@ -14,6 +14,8 @@ Apple Podcasts. Le balado est le compte SOURCE de la fiche → profil_source.
14 """ 14 """
15 from __future__ import annotations 15 from __future__ import annotations
16 16
17+from datetime import datetime, timedelta, timezone
18+
17 from ..identity import account 19 from ..identity import account
18 from ..normalize import slugify 20 from ..normalize import slugify
19 from ..schema import Creator 21 from ..schema import Creator
@@ -22,6 +24,21 @@ from .youtube_recherche import is_quebec
22 24
23 SEARCH_URL = "https://itunes.apple.com/search" 25 SEARCH_URL = "https://itunes.apple.com/search"
24 26
27+# au-delà de 12 mois sans épisode : balado dormant (flag public dans le doc ;
28+# la fiche reste dans l'annuaire — §13, on ne supprime jamais brutalement)
29+DORMANT_AFTER_DAYS = 365
30+
31+
32+def _dormant(release_date: str | None) -> bool:
33+ """Vrai si le dernier épisode (releaseDate iTunes) date de > 12 mois."""
34+ if not release_date:
35+ return False
36+ try:
37+ last = datetime.fromisoformat(release_date.replace("Z", "+00:00"))
38+ except ValueError:
39+ return False
40+ return datetime.now(timezone.utc) - last > timedelta(days=DORMANT_AFTER_DAYS)
41+
25 QUERIES = [ 42 QUERIES = [
26 "québécois", "québécoise", "balado québec", "podcast québec", "montréal", 43 "québécois", "québécoise", "balado québec", "podcast québec", "montréal",
27 "humour québécois", "balado montréal", "saguenay", "gatineau", "sherbrooke", 44 "humour québécois", "balado montréal", "saguenay", "gatineau", "sherbrooke",
@@ -90,24 +107,36 @@ class BaladosItunesConnector(BaseConnector):
90 if not is_quebec(f"{name} {artist}"): 107 if not is_quebec(f"{name} {artist}"):
91 continue 108 continue
92 seen.add(cid) 109 seen.add(cid)
93− url = r.get("collectionViewUrl") or r.get("feedUrl") or "" 110+ feed = r.get("feedUrl") or ""
111+ url = r.get("collectionViewUrl") or feed
94 if not url: 112 if not url:
95 continue 113 continue
96 acc = account("podcast", str(cid), "profil_source", url=url) 114 acc = account("podcast", str(cid), "profil_source", url=url)
115+ dormant = _dormant(r.get("releaseDate"))
97 acc.metrics = {k: v for k, v in { 116 acc.metrics = {k: v for k, v in {
98 "episodes": r.get("trackCount"), 117 "episodes": r.get("trackCount"),
99 "genre": r.get("primaryGenreName"), 118 "genre": r.get("primaryGenreName"),
119+ "feed_url": feed, # site du balado (RSS)
120+ "artist_id": r.get("artistId"),
121+ "last_episode": r.get("releaseDate"),
122+ "dormant": dormant or None, # >12 mois sans épisode
100 }.items() if v} 123 }.items() if v}
101− niche = _GENRE_NICHE.get(r.get("primaryGenreName") or "", 124+ # genres[] complets → niches §6.1 (le genre principal inclus)
102− "actualite-opinion") 125+ genres = [g for g in (r.get("genres") or []) if isinstance(g, str)]
126+ niches = sorted({_GENRE_NICHE[g] for g in
127+ [r.get("primaryGenreName"), *genres]
128+ if g in _GENRE_NICHE}) or ["actualite-opinion"]
103 creators.append(Creator( 129 creators.append(Creator(
104 source=self.source_id, 130 source=self.source_id,
105 external_id=str(cid), 131 external_id=str(cid),
106 display_name=name, 132 display_name=name,
107 bio=f"Balado de {artist}." if artist and artist != name else "", 133 bio=f"Balado de {artist}." if artist and artist != name else "",
108− niches=[niche], 134+ niches=niches,
109 creator_type="podcasteur", 135 creator_type="podcasteur",
110 avatar_url=r.get("artworkUrl600") or r.get("artworkUrl100"), 136 avatar_url=r.get("artworkUrl600") or r.get("artworkUrl100"),
137+ # §13 : plus de publication depuis 12 mois → inactive
138+ # (réversible : redevient actif dès un nouvel épisode)
139+ status="inactive" if dormant else "active",
111 platforms=[acc], 140 platforms=[acc],
112 notes="Référencé sur Apple Podcasts (API de recherche officielle).", 141 notes="Référencé sur Apple Podcasts (API de recherche officielle).",
113 source_ids=[f"{self.source_id}:{cid}"], 142 source_ids=[f"{self.source_id}:{cid}"],
modified creaka/connectors/base.py +8 −0
@@ -20,6 +20,14 @@ FIRECRAWL_API = "https://api.firecrawl.dev/v1/scrape"
20 SCRAPFLY_API = "https://api.scrapfly.io/scrape" 20 SCRAPFLY_API = "https://api.scrapfly.io/scrape"
21 21
22 22
23+class SkipSource(Exception):
24+ """Passage sauté VOLONTAIREMENT (ex. clés API absentes) — pas un échec.
25+
26+ Le pipeline (ingest.py) le journalise clairement, sans alerte de blocage
27+ ni run « 0 créateur » dans sync_log (§16, §18 monitoring).
28+ """
29+
30+
23 class BaseConnector: 31 class BaseConnector:
24 """Un connecteur = un adaptateur propre à une source (CLAUDE.md §8). 32 """Un connecteur = un adaptateur propre à une source (CLAUDE.md §8).
25 33
modified creaka/connectors/onlyqueb.py +61 −4
@@ -20,7 +20,7 @@ import json
20 import re 20 import re
21 21
22 from ..identity import account 22 from ..identity import account
23−from ..normalize import platform_from_url, slugify 23+from ..normalize import map_niche, platform_from_url, slugify
24 from ..schema import Creator 24 from ..schema import Creator
25 from .base import BaseConnector 25 from .base import BaseConnector
26 26
@@ -28,6 +28,13 @@ SITEMAP_URL = "https://onlyqueb.com/sitemap.xml"
28 _LOC_RE = re.compile(r"<loc>([^<]+)</loc>") 28 _LOC_RE = re.compile(r"<loc>([^<]+)</loc>")
29 _LD_RE = re.compile(r'application/ld\+json"[^>]*>(.*?)</script>', re.S) 29 _LD_RE = re.compile(r'application/ld\+json"[^>]*>(.*?)</script>', re.S)
30 _EXCLUDED = ("/en", "/categories", "/explore", "/seo", "/signup", "/blog") 30 _EXCLUDED = ("/en", "/categories", "/explore", "/seo", "/signup", "/blog")
31+# fragments du payload Next.js/RSC embarqué dans la page (self.__next_f.push)
32+_PUSH_RE = re.compile(r'self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)')
33+
34+# catégories OnlyQueb → niches §6.1 quand le lien de sens existe ; les
35+# descripteurs physiques (blonde, petite…) ne sont PAS des niches → « autre »
36+_CAT_NICHE = {"cosplay": "arts", "gamer": "gaming", "gaming": "gaming",
37+ "fitness": "sport-fitness", "athletic": "sport-fitness"}
31 38
32 39
33 def parse_profile_ld(html: str) -> dict | None: 40 def parse_profile_ld(html: str) -> dict | None:
@@ -49,6 +56,32 @@ def parse_profile_ld(html: str) -> dict | None:
49 return None 56 return None
50 57
51 58
59+def parse_profile_next(html: str) -> dict:
60+ """Page profil → objet `profile` du payload Next.js/RSC ({} si absent).
61+
62+ La même page expose, au-delà du JSON-LD, la fiche complète auto-déclarée :
63+ displayName réel, bio, headline, categories[], mymUrl/fanslyUrl/ofUrl,
64+ websiteUrl, updatedAt. Chaque fragment `self.__next_f.push([1,"…"])` est
65+ une chaîne JS → décodée comme chaîne JSON, puis l'objet `"profile":{…}`
66+ est extrait par décodage équilibré (raw_decode).
67+ """
68+ for m in _PUSH_RE.finditer(html):
69+ try:
70+ chunk = json.loads(f'"{m.group(1)}"')
71+ except Exception:
72+ continue
73+ i = chunk.find('"profile":{')
74+ if i < 0:
75+ continue
76+ try:
77+ obj, _ = json.JSONDecoder().raw_decode(chunk, i + len('"profile":'))
78+ except Exception:
79+ continue
80+ if isinstance(obj, dict) and obj.get("displayName") is not None:
81+ return obj
82+ return {}
83+
84+
52 class OnlyQuebConnector(BaseConnector): 85 class OnlyQuebConnector(BaseConnector):
53 source_id = "onlyqueb" 86 source_id = "onlyqueb"
54 kind = "discovery" 87 kind = "discovery"
@@ -75,9 +108,15 @@ class OnlyQuebConnector(BaseConnector):
75 prof = parse_profile_ld(html) 108 prof = parse_profile_ld(html)
76 if not prof or not prof["name"]: 109 if not prof or not prof["name"]:
77 continue 110 continue
111+ extra = parse_profile_next(html) # payload RSC : fiche complète
112+ links = list(prof["links"])
113+ for key in ("ofUrl", "mymUrl", "fanslyUrl", "instagramUrl",
114+ "xUrl", "tiktokUrl", "snapchatUrl"):
115+ if (extra.get(key) or "").startswith("http"):
116+ links.append(extra[key])
78 accounts = [] 117 accounts = []
79 seen: set[str] = set() 118 seen: set[str] = set()
80− for link in prof["links"]: 119+ for link in links:
81 hit = platform_from_url(link) 120 hit = platform_from_url(link)
82 if not hit: 121 if not hit:
83 continue 122 continue
@@ -86,14 +125,32 @@ class OnlyQuebConnector(BaseConnector):
86 continue 125 continue
87 seen.add(f"{platform}:{handle}") 126 seen.add(f"{platform}:{handle}")
88 accounts.append(account(platform, handle, "cross_link", url=link)) 127 accounts.append(account(platform, handle, "cross_link", url=link))
128+ # site web personnel auto-déclaré → compte site-web (annuaire de liens)
129+ site = (extra.get("websiteUrl") or "").strip()
130+ if site.startswith("http") and not platform_from_url(site):
131+ domain = site.split("://", 1)[-1].split("/", 1)[0]
132+ domain = domain.removeprefix("www.")
133+ if domain and f"site-web:{domain}" not in seen:
134+ seen.add(f"site-web:{domain}")
135+ accounts.append(account("site-web", domain, "cross_link",
136+ url=site))
89 if not accounts: 137 if not accounts:
90 continue # sans lien vérifiable, pas de fiche (annuaire de liens) 138 continue # sans lien vérifiable, pas de fiche (annuaire de liens)
139+ name = (extra.get("displayName") or "").strip() or prof["name"]
140+ bio = "\n".join(
141+ t.strip() for t in ((extra.get("headline") or ""),
142+ (extra.get("bio") or "")) if t.strip())
143+ cats = [c for c in (extra.get("categories") or [])
144+ if isinstance(c, str)]
145+ niches = sorted({_CAT_NICHE.get(c, map_niche(c)) for c in cats})
146+ niches = [n for n in niches if n != "autre"] or ["autre"]
91 slug = slugify(prof["name"]) 147 slug = slugify(prof["name"])
92 creators.append(Creator( 148 creators.append(Creator(
93 source=self.source_id, 149 source=self.source_id,
94 external_id=slug, 150 external_id=slug,
95− display_name=prof["name"], 151+ display_name=name,
96− niches=["autre"], 152+ bio=bio,
153+ niches=niches,
97 creator_type="influenceur", 154 creator_type="influenceur",
98 avatar_url=prof["image"], 155 avatar_url=prof["image"],
99 platforms=accounts, 156 platforms=accounts,
modified creaka/connectors/twitch.py +5 −2
@@ -16,7 +16,7 @@ from __future__ import annotations
16 import os 16 import os
17 17
18 from ..schema import Creator, now_iso 18 from ..schema import Creator, now_iso
19−from .base import BaseConnector 19+from .base import BaseConnector, SkipSource
20 20
21 TOKEN_URL = "https://id.twitch.tv/oauth2/token" 21 TOKEN_URL = "https://id.twitch.tv/oauth2/token"
22 USERS_URL = "https://api.twitch.tv/helix/users" 22 USERS_URL = "https://api.twitch.tv/helix/users"
@@ -38,7 +38,10 @@ class TwitchConnector(BaseConnector):
38 cid = os.environ.get("TWITCH_CLIENT_ID", "") 38 cid = os.environ.get("TWITCH_CLIENT_ID", "")
39 secret = os.environ.get("TWITCH_CLIENT_SECRET", "") 39 secret = os.environ.get("TWITCH_CLIENT_SECRET", "")
40 if not cid or not secret: 40 if not cid or not secret:
41− return [] # pas de clés → on saute (jamais de contournement, §15) 41+ # pas de clés → passage sauté PROPREMENT (jamais de contournement,
42+ # §15) ; SkipSource évite la fausse alerte « source bloquée »
43+ raise SkipSource("clés TWITCH_CLIENT_ID/TWITCH_CLIENT_SECRET "
44+ "manquantes (.env)")
42 token = self._token(cid, secret) 45 token = self._token(cid, secret)
43 headers = {"Client-Id": cid, "Authorization": f"Bearer {token}"} 46 headers = {"Client-Id": cid, "Authorization": f"Bearer {token}"}
44 enriched: list[Creator] = [] 47 enriched: list[Creator] = []
modified creaka/connectors/wikidata_qc.py +8 −2
@@ -18,6 +18,7 @@ Prudence mineurs (§15) : l'année de naissance publique permet de lever
18 """ 18 """
19 from __future__ import annotations 19 from __future__ import annotations
20 20
21+import sys
21 from datetime import datetime, timezone 22 from datetime import datetime, timezone
22 23
23 from ..identity import account 24 from ..identity import account
@@ -71,11 +72,16 @@ class WikidataQcConnector(BaseConnector):
71 72
72 def fetch(self) -> list[Creator]: 73 def fetch(self) -> list[Creator]:
73 rows: list[dict] = [] 74 rows: list[dict] = []
75+ self.errors = 0 # comptabilisé dans sync_log.errors par le pipeline (§16)
74 for prop in ("P19", "P551"): # naissance, résidence 76 for prop in ("P19", "P551"): # naissance, résidence
75 try: 77 try:
76 rows.extend(self._query(prop)) 78 rows.extend(self._query(prop))
77− except Exception: 79+ except Exception as exc:
78− continue # une branche en échec ne bloque pas l'autre 80+ # une branche en échec ne bloque pas l'autre — mais JAMAIS en
81+ # silence : la couverture chute si une requête SPARQL casse
82+ self.errors += 1
83+ print(f"[crea-ka] ⚠ wikidata-qc : requête SPARQL {prop} en "
84+ f"échec : {exc}", file=sys.stderr)
79 by_person: dict[str, dict] = {} 85 by_person: dict[str, dict] = {}
80 for r in rows: 86 for r in rows:
81 def val(key): 87 def val(key):
modified creaka/connectors/youtube_recherche.py +11 −1
@@ -248,9 +248,18 @@ def channels_from_data(data) -> list[dict]:
248 ((r.get("descriptionSnippet") or {}).get("runs") or [])) 248 ((r.get("descriptionSnippet") or {}).get("runs") or []))
249 subs_txt = ((r.get("videoCountText") or {}).get("simpleText")) or \ 249 subs_txt = ((r.get("videoCountText") or {}).get("simpleText")) or \
250 ((r.get("subscriberCountText") or {}).get("simpleText")) or "" 250 ((r.get("subscriberCountText") or {}).get("simpleText")) or ""
251+ # avatar « gratuit » : le renderer embarque les miniatures du profil
252+ # → meilleure résolution (URLs souvent protocol-relative « //yt3… »)
253+ thumbs = (r.get("thumbnail") or {}).get("thumbnails") or []
254+ avatar = None
255+ if thumbs:
256+ avatar = max(thumbs,
257+ key=lambda t: t.get("width") or 0).get("url")
258+ if avatar and avatar.startswith("//"):
259+ avatar = "https:" + avatar
251 channels.append({"handle": handle, "channel_id": r.get("channelId", ""), 260 channels.append({"handle": handle, "channel_id": r.get("channelId", ""),
252 "name": name, "subs": parse_count(subs_txt), 261 "name": name, "subs": parse_count(subs_txt),
253− "description": desc}) 262+ "description": desc, "avatar": avatar})
254 except Exception: 263 except Exception:
255 continue 264 continue
256 return channels 265 return channels
@@ -287,6 +296,7 @@ class YouTubeRechercheConnector(BaseConnector):
287 bio=ch["description"][:1200], 296 bio=ch["description"][:1200],
288 niches=[niche], 297 niches=[niche],
289 creator_type="youtubeur", 298 creator_type="youtubeur",
299+ avatar_url=ch.get("avatar"),
290 platforms=[acc], 300 platforms=[acc],
291 source_ids=[f"{self.source_id}:{key}"], 301 source_ids=[f"{self.source_id}:{key}"],
292 )) 302 ))
modified creaka/db.py +48 −4
@@ -20,7 +20,10 @@ from __future__ import annotations
20 20
21 import json 21 import json
22 import sqlite3 22 import sqlite3
23+import sys
24+import time
23 from dataclasses import asdict 25 from dataclasses import asdict
26+from datetime import datetime, timedelta, timezone
24 from pathlib import Path 27 from pathlib import Path
25 28
26 from . import ethics 29 from . import ethics
@@ -135,6 +138,14 @@ def _unique_id(con: sqlite3.Connection, base: str) -> str:
135 138
136 def _write_creator(con: sqlite3.Connection, cid: str, cr: Creator, 139 def _write_creator(con: sqlite3.Connection, cid: str, cr: Creator,
137 first_seen: str, ts: str) -> None: 140 first_seen: str, ts: str) -> None:
141+ # comptes retirés de la fiche : jamais en silence (§13 — historique) ;
142+ # merge_accounts fait l'union, donc une perte ici est anormale → journalisée
143+ old_keys = {f"{r['platform']}:{r['handle']}" for r in con.execute(
144+ "SELECT platform, handle FROM accounts WHERE creator_id=?", (cid,))}
145+ lost = old_keys - {a.key for a in cr.platforms}
146+ if lost:
147+ print(f"[crea-ka] ⚠ {cid} : compte(s) retiré(s) de la fiche : "
148+ f"{', '.join(sorted(lost))}", file=sys.stderr)
138 con.execute( 149 con.execute(
139 """INSERT INTO creators (id, display_name, status, is_minor, region, city, 150 """INSERT INTO creators (id, display_name, status, is_minor, region, city,
140 niches, languages, creator_type, primary_platform, audience_tier, 151 niches, languages, creator_type, primary_platform, audience_tier,
@@ -186,8 +197,14 @@ def _find_existing(con: sqlite3.Connection, cr: Creator) -> str | None:
186 197
187 198
188 def sync_source(con: sqlite3.Connection, source_id: str, 199 def sync_source(con: sqlite3.Connection, source_id: str,
189− creators: list[Creator]) -> dict: 200+ creators: list[Creator], *, started: float | None = None,
190− """Synchronise le lot d'une source : ajouts, fusions, mises à jour, opt-out.""" 201+ errors: int = 0) -> dict:
202+ """Synchronise le lot d'une source : ajouts, fusions, mises à jour, opt-out.
203+
204+ `started` (time.time() du début du passage) → durée réelle persistée dans
205+ sync_log.seconds ; `errors` → erreurs non bloquantes signalées par le
206+ connecteur (§16 logging structuré).
207+ """
191 ts = now_iso() 208 ts = now_iso()
192 added = updated = skipped_optout = 0 209 added = updated = skipped_optout = 0
193 confs: list[float] = [] 210 confs: list[float] = []
@@ -226,15 +243,42 @@ def sync_source(con: sqlite3.Connection, source_id: str,
226 if not creators: 243 if not creators:
227 alert = "0 créateur retourné — source possiblement bloquée" 244 alert = "0 créateur retourné — source possiblement bloquée"
228 stats["alert"] = alert 245 stats["alert"] = alert
246+ seconds = round(time.time() - started, 1) if started else 0.0
247+ stats["seconds"] = seconds
248+ if errors:
249+ stats["errors"] = errors
229 con.execute( 250 con.execute(
230 "INSERT INTO sync_log (ts, source, creators, accounts, avg_confidence," 251 "INSERT INTO sync_log (ts, source, creators, accounts, avg_confidence,"
231− " added, updated, errors, seconds, alert) VALUES (?,?,?,?,?,?,?,0,0,?)", 252+ " added, updated, errors, seconds, alert) VALUES (?,?,?,?,?,?,?,?,?,?)",
232 (ts, source_id, len(creators), n_acc, stats["avg_confidence"], 253 (ts, source_id, len(creators), n_acc, stats["avg_confidence"],
233− added, updated, alert)) 254+ added, updated, errors, seconds, alert))
234 con.commit() 255 con.commit()
235 return stats 256 return stats
236 257
237 258
259+def archive_missing(con: sqlite3.Connection, *, hours: int = 72) -> int:
260+ """Archivage des disparus (§13, politique de grâce).
261+
262+ Un créateur qu'AUCUNE source de découverte n'a revu depuis ~3 passages
263+ complets (cadence quotidienne §14 → 72 h) passe `status='inactive'`.
264+ Réversible : dès qu'une source le revoit, la fusion (sync_source →
265+ _write_creator) réécrit son statut actif et son `last_seen`.
266+ Les opt-out ne sont JAMAIS touchés (status != 'active').
267+ """
268+ cutoff = (datetime.now(timezone.utc) - timedelta(hours=hours)) \
269+ .strftime("%Y-%m-%dT%H:%M:%SZ")
270+ ts = now_iso()
271+ cur = con.execute(
272+ "UPDATE creators SET status='inactive', updated_at=? "
273+ "WHERE status='active' AND last_seen IS NOT NULL AND last_seen<?",
274+ (ts, cutoff))
275+ con.commit()
276+ if cur.rowcount:
277+ print(f"[crea-ka] archivage §13 : {cur.rowcount} fiche(s) non revue(s) "
278+ f"depuis {hours} h → inactive (réversible au retour)")
279+ return cur.rowcount
280+
281+
238 def log_failure(con: sqlite3.Connection, source_id: str, message: str) -> None: 282 def log_failure(con: sqlite3.Connection, source_id: str, message: str) -> None:
239 con.execute( 283 con.execute(
240 "INSERT INTO sync_log (ts, source, creators, accounts, errors, alert)" 284 "INSERT INTO sync_log (ts, source, creators, accounts, errors, alert)"
modified creaka/dedup.py +12 −1
@@ -45,6 +45,13 @@ def merge_creators(canon: Creator, other: Creator) -> Creator:
45 """ 45 """
46 canon.platforms = merge_accounts(canon.platforms, other.platforms) 46 canon.platforms = merge_accounts(canon.platforms, other.platforms)
47 canon.source_ids = sorted(set(canon.source_ids) | set(other.source_ids)) 47 canon.source_ids = sorted(set(canon.source_ids) | set(other.source_ids))
48+ # nom d'affichage : un VRAI nom public remplace un simple pseudo-handle
49+ # (ex. « misslavoie » → « Jade Lavoie ») ; jamais l'inverse
50+ handles = {a.handle for a in canon.platforms}
51+ if (other.display_name and other.display_name != canon.display_name
52+ and canon.display_name.lower() in handles
53+ and other.display_name.lower() not in handles):
54+ canon.display_name = other.display_name
48 if len(other.bio or "") > len(canon.bio or ""): 55 if len(other.bio or "") > len(canon.bio or ""):
49 canon.bio = other.bio 56 canon.bio = other.bio
50 if len(other.notes or "") > len(canon.notes or ""): 57 if len(other.notes or "") > len(canon.notes or ""):
@@ -57,9 +64,13 @@ def merge_creators(canon: Creator, other: Creator) -> Creator:
57 setattr(canon, field_, getattr(other, field_)) 64 setattr(canon, field_, getattr(other, field_))
58 # prudence maximale : si UNE source signale un mineur, le régime s'applique 65 # prudence maximale : si UNE source signale un mineur, le régime s'applique
59 canon.is_minor = canon.is_minor or other.is_minor 66 canon.is_minor = canon.is_minor or other.is_minor
60− # opt-out prime sur tout (§15) 67+ # opt-out prime sur tout (§15) ; sinon l'observation la plus FRAÎCHE
68+ # (`other`) fixe le statut — un balado dormant passe inactive, une fiche
69+ # archivée (§13) redevient active dès qu'une source la revoit
61 if "opted_out" in (canon.status, other.status): 70 if "opted_out" in (canon.status, other.status):
62 canon.status = "opted_out" 71 canon.status = "opted_out"
72+ elif other.status:
73+ canon.status = other.status
63 # recalculer les agrégats (tier, portée, plateforme principale) 74 # recalculer les agrégats (tier, portée, plateforme principale)
64 canon.audience_tier = "" 75 canon.audience_tier = ""
65 canon.primary_platform = "" 76 canon.primary_platform = ""
modified creaka/ingest.py +10 −2
@@ -24,6 +24,7 @@ import traceback
24 24
25 from . import db 25 from . import db
26 from .connectors import CONNECTORS 26 from .connectors import CONNECTORS
27+from .connectors.base import SkipSource
27 from .dedup import dedupe 28 from .dedup import dedupe
28 29
29 # ordre d'exécution des connecteurs d'enrichissement (link-in-bio en dernier) 30 # ordre d'exécution des connecteurs d'enrichissement (link-in-bio en dernier)
@@ -74,18 +75,25 @@ def run(sources: list[str] | None = None) -> list[dict]:
74 batch = connector.enrich(load_active(con)) 75 batch = connector.enrich(load_active(con))
75 finalized, dropped = _finalize_batch(batch) 76 finalized, dropped = _finalize_batch(batch)
76 finalized = dedupe(finalized) # dédup intra-lot avant la base 77 finalized = dedupe(finalized) # dédup intra-lot avant la base
77− stats = db.sync_source(con, sid, finalized) 78+ stats = db.sync_source(con, sid, finalized, started=t0,
78− stats["seconds"] = round(time.time() - t0, 1) 79+ errors=getattr(connector, "errors", 0))
79 if dropped: 80 if dropped:
80 stats["dropped"] = dropped 81 stats["dropped"] = dropped
81 if stats.get("alert") and cls.kind == "discovery": 82 if stats.get("alert") and cls.kind == "discovery":
82 print(f"[crea-ka] ⚠ ALERTE {sid} : {stats['alert']}") 83 print(f"[crea-ka] ⚠ ALERTE {sid} : {stats['alert']}")
83 print(f"[crea-ka] {stats}") 84 print(f"[crea-ka] {stats}")
84 results.append(stats) 85 results.append(stats)
86+ except SkipSource as exc:
87+ # passage sauté volontairement (ex. clés API absentes) : log clair,
88+ # PAS d'alerte de blocage ni de run compté à 0 dans sync_log
89+ print(f"[crea-ka] ↷ {sid} sauté : {exc}")
90+ results.append({"source": sid, "skipped": str(exc)})
85 except Exception as exc: # robustesse : une source ne bloque pas les autres 91 except Exception as exc: # robustesse : une source ne bloque pas les autres
86 db.log_failure(con, sid, f"{exc}") 92 db.log_failure(con, sid, f"{exc}")
87 traceback.print_exc() 93 traceback.print_exc()
88 results.append({"source": sid, "error": str(exc)}) 94 results.append({"source": sid, "error": str(exc)})
95+ if not sources: # passage COMPLET seulement : archivage des disparus (§13)
96+ db.archive_missing(con)
89 return results 97 return results
90 98
91 99
modified creaka/normalize.py +7 −3
@@ -24,8 +24,8 @@ NICHES = {
24 24
25 PLATFORMS = { 25 PLATFORMS = {
26 "instagram", "tiktok", "youtube", "twitch", "kick", "x", "facebook", 26 "instagram", "tiktok", "youtube", "twitch", "kick", "x", "facebook",
27− "snapchat", "substack", "patreon", "onlyfans", "linkedin", "threads", 27+ "snapchat", "substack", "patreon", "onlyfans", "mym", "fansly",
28− "podcast", "site-web", "autre", 28+ "linkedin", "threads", "podcast", "site-web", "autre",
29 } 29 }
30 30
31 CREATOR_TYPES = { 31 CREATOR_TYPES = {
@@ -57,6 +57,8 @@ _URL_TEMPLATES: dict[str, str | None] = {
57 "substack": "https://{h}.substack.com", 57 "substack": "https://{h}.substack.com",
58 "patreon": "https://www.patreon.com/{h}", 58 "patreon": "https://www.patreon.com/{h}",
59 "onlyfans": "https://onlyfans.com/{h}", 59 "onlyfans": "https://onlyfans.com/{h}",
60+ "mym": "https://mym.fans/{h}",
61+ "fansly": "https://fansly.com/{h}",
60 "linkedin": "https://www.linkedin.com/in/{h}", 62 "linkedin": "https://www.linkedin.com/in/{h}",
61 "threads": "https://www.threads.net/@{h}", 63 "threads": "https://www.threads.net/@{h}",
62 "podcast": None, 64 "podcast": None,
@@ -78,6 +80,8 @@ _URL_PLATFORM_RES: list[tuple[str, re.Pattern]] = [
78 ("substack", re.compile(r"https?://([\w\-]+)\.substack\.com", re.I)), 80 ("substack", re.compile(r"https?://([\w\-]+)\.substack\.com", re.I)),
79 ("patreon", re.compile(r"patreon\.com/([\w.\-]+)", re.I)), 81 ("patreon", re.compile(r"patreon\.com/([\w.\-]+)", re.I)),
80 ("onlyfans", re.compile(r"onlyfans\.com/([\w.\-]+)", re.I)), 82 ("onlyfans", re.compile(r"onlyfans\.com/([\w.\-]+)", re.I)),
83+ ("mym", re.compile(r"mym\.fans/([\w.\-]+)", re.I)),
84+ ("fansly", re.compile(r"fansly\.com/([\w.\-]+)", re.I)),
81 ("linkedin", re.compile(r"linkedin\.com/in/([\w.\-]+)", re.I)), 85 ("linkedin", re.compile(r"linkedin\.com/in/([\w.\-]+)", re.I)),
82 ("threads", re.compile(r"threads\.net/@?([\w.\-]+)", re.I)), 86 ("threads", re.compile(r"threads\.net/@?([\w.\-]+)", re.I)),
83 ] 87 ]
@@ -136,7 +140,7 @@ def normalize_platform(raw: str | None) -> str:
136 "facebook": "facebook", "fb": "facebook", 140 "facebook": "facebook", "fb": "facebook",
137 "snapchat": "snapchat", "snap": "snapchat", 141 "snapchat": "snapchat", "snap": "snapchat",
138 "substack": "substack", "patreon": "patreon", "onlyfans": "onlyfans", 142 "substack": "substack", "patreon": "patreon", "onlyfans": "onlyfans",
139− "of": "onlyfans", 143+ "of": "onlyfans", "mym": "mym", "fansly": "fansly",
140 "linkedin": "linkedin", "threads": "threads", 144 "linkedin": "linkedin", "threads": "threads",
141 "podcast": "podcast", "balado": "podcast", 145 "podcast": "podcast", "balado": "podcast",
142 "site": "site-web", "site-web": "site-web", "website": "site-web", 146 "site": "site-web", "site-web": "site-web", "website": "site-web",
modified creaka/web.py +20 −8
@@ -7,6 +7,7 @@
7 from __future__ import annotations 7 from __future__ import annotations
8 8
9 import json 9 import json
10+import threading
10 from pathlib import Path 11 from pathlib import Path
11 12
12 from fastapi import FastAPI, HTTPException, Query 13 from fastapi import FastAPI, HTTPException, Query
@@ -30,7 +31,18 @@ app.add_middleware(CORSMiddleware, allow_origins=["*"],
30 allow_methods=["*"], allow_headers=["*"]) 31 allow_methods=["*"], allow_headers=["*"])
31 app.add_middleware(GZipMiddleware, minimum_size=1000) 32 app.add_middleware(GZipMiddleware, minimum_size=1000)
32 33
33−_con = db.connect() 34+# Connexion SQLite PAR THREAD (§18) : FastAPI exécute les endpoints sync dans
35+# un pool de threads — une connexion partagée (check_same_thread=False) fait
36+# s'entrelacer les curseurs et produit des 500 intermittents (« 'NoneType'
37+# object is not subscriptable »). threading.local = une connexion par thread.
38+_local = threading.local()
39+
40+
41+def _db():
42+ con = getattr(_local, "con", None)
43+ if con is None:
44+ con = _local.con = db.connect()
45+ return con
34 46
35 # connexion « KA ID » (hub groupe-ka.com) — voir creaka/auth.py 47 # connexion « KA ID » (hub groupe-ka.com) — voir creaka/auth.py
36 app.include_router(auth.router) 48 app.include_router(auth.router)
@@ -43,7 +55,7 @@ def healthz():
43 55
44 @app.get("/api/health") 56 @app.get("/api/health")
45 def health(): 57 def health():
46− s = db.stats(_con) 58+ s = db.stats(_db())
47 return {"ok": True, "creators": s["creators"], "accounts": s["accounts"], 59 return {"ok": True, "creators": s["creators"], "accounts": s["accounts"],
48 "last_sync": s["last_sync"]} 60 "last_sync": s["last_sync"]}
49 61
@@ -53,14 +65,14 @@ def list_creators(q: str = "", niche: str = "", region: str = "",
53 langue: str = "", plateforme: str = "", tier: str = "", 65 langue: str = "", plateforme: str = "", tier: str = "",
54 sort: str = "reach", 66 sort: str = "reach",
55 limit: int = Query(60, le=200), offset: int = Query(0, ge=0)): 67 limit: int = Query(60, le=200), offset: int = Query(0, ge=0)):
56− return db.search(_con, q=q.strip(), niche=niche, region=region, 68+ return db.search(_db(), q=q.strip(), niche=niche, region=region,
57 langue=langue, plateforme=plateforme, tier=tier, 69 langue=langue, plateforme=plateforme, tier=tier,
58 sort=sort, limit=limit, offset=offset) 70 sort=sort, limit=limit, offset=offset)
59 71
60 72
61 @app.get("/api/creators/{cid}") 73 @app.get("/api/creators/{cid}")
62 def get_creator(cid: str): 74 def get_creator(cid: str):
63− creator = db.get_creator(_con, cid) 75+ creator = db.get_creator(_db(), cid)
64 if creator is None: 76 if creator is None:
65 raise HTTPException(404, "créateur introuvable") 77 raise HTTPException(404, "créateur introuvable")
66 return creator 78 return creator
@@ -68,7 +80,7 @@ def get_creator(cid: str):
68 80
69 @app.get("/api/stats") 81 @app.get("/api/stats")
70 def get_stats(): 82 def get_stats():
71− return db.stats(_con) 83+ return db.stats(_db())
72 84
73 85
74 _PERIODS_OK = {"auj", "7j", "30j", "3m", "6m", "12m", "annee", "tout"} 86 _PERIODS_OK = {"auj", "7j", "30j", "3m", "6m", "12m", "annee", "tout"}
@@ -82,7 +94,7 @@ def stats_dashboard(period: str = "30j",
82 if period not in _PERIODS_OK and not (date_from and date_to): 94 if period not in _PERIODS_OK and not (date_from and date_to):
83 raise HTTPException(400, "période inconnue " 95 raise HTTPException(400, "période inconnue "
84 "(auj|7j|30j|3m|6m|12m|annee|tout ou from/to)") 96 "(auj|7j|30j|3m|6m|12m|annee|tout ou from/to)")
85− return stats_mod.dashboard(_con, period=period, 97+ return stats_mod.dashboard(_db(), period=period,
86 date_from=date_from, date_to=date_to) 98 date_from=date_from, date_to=date_to)
87 99
88 100
@@ -95,7 +107,7 @@ def stats_report(period: str = "30j", mode: str = "complet",
95 raise HTTPException(400, "mode invalide (complet|synthese)") 107 raise HTTPException(400, "mode invalide (complet|synthese)")
96 if period not in _PERIODS_OK and not (date_from and date_to): 108 if period not in _PERIODS_OK and not (date_from and date_to):
97 raise HTTPException(400, "période inconnue") 109 raise HTTPException(400, "période inconnue")
98− dash = stats_mod.dashboard(_con, period=period, 110+ dash = stats_mod.dashboard(_db(), period=period,
99 date_from=date_from, date_to=date_to) 111 date_from=date_from, date_to=date_to)
100 pdf = kapdf.GroupeKAReport(site=stats_mod.site_info(), 112 pdf = kapdf.GroupeKAReport(site=stats_mod.site_info(),
101 dashboard=dash, mode=mode).build() 113 dashboard=dash, mode=mode).build()
@@ -142,7 +154,7 @@ def optout(req: OptOutRequest):
142 raise HTTPException(400, "préciser un nom ou un compte (plateforme:handle)") 154 raise HTTPException(400, "préciser un nom ou un compte (plateforme:handle)")
143 ethics.add_optout(name=name, account=acc, reason=req.reason, 155 ethics.add_optout(name=name, account=acc, reason=req.reason,
144 contact=req.contact) 156 contact=req.contact)
145− masked = db.apply_optout(_con, name=name, account=acc) 157+ masked = db.apply_optout(_db(), name=name, account=acc)
146 return {"ok": True, "masked": masked, 158 return {"ok": True, "masked": masked,
147 "message": "Fiche masquée. La demande sera vérifiée ; merci."} 159 "message": "Fiche masquée. La demande sera vérifiée ; merci."}
148 160
modified frontend/dist/index.html +2 −0
@@ -950,6 +950,8 @@ const PLAT={
950 substack:{label:"Substack",color:"#ff6719"}, 950 substack:{label:"Substack",color:"#ff6719"},
951 patreon:{label:"Patreon",color:"#f1465a"}, 951 patreon:{label:"Patreon",color:"#f1465a"},
952 onlyfans:{label:"OnlyFans",color:"#00aff0"}, 952 onlyfans:{label:"OnlyFans",color:"#00aff0"},
953+ mym:{label:"MYM",color:"#141814"},
954+ fansly:{label:"Fansly",color:"#2699f7"},
953 linkedin:{label:"LinkedIn",color:"#0a66c2"}, 955 linkedin:{label:"LinkedIn",color:"#0a66c2"},
954 threads:{label:"Threads",color:"#141814"}, 956 threads:{label:"Threads",color:"#141814"},
955 podcast:{label:"Balado",color:"#8940fa"}, 957 podcast:{label:"Balado",color:"#8940fa"},
modified frontend/src/index.template.html +2 −0
@@ -492,6 +492,8 @@ const PLAT={
492 substack:{label:"Substack",color:"#ff6719"}, 492 substack:{label:"Substack",color:"#ff6719"},
493 patreon:{label:"Patreon",color:"#f1465a"}, 493 patreon:{label:"Patreon",color:"#f1465a"},
494 onlyfans:{label:"OnlyFans",color:"#00aff0"}, 494 onlyfans:{label:"OnlyFans",color:"#00aff0"},
495+ mym:{label:"MYM",color:"#141814"},
496+ fansly:{label:"Fansly",color:"#2699f7"},
495 linkedin:{label:"LinkedIn",color:"#0a66c2"}, 497 linkedin:{label:"LinkedIn",color:"#0a66c2"},
496 threads:{label:"Threads",color:"#141814"}, 498 threads:{label:"Threads",color:"#141814"},
497 podcast:{label:"Balado",color:"#8940fa"}, 499 podcast:{label:"Balado",color:"#8940fa"},
498 500
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere