Fiches légendaires : upgrade majeur des 13 acteurs Apify (bannières, covers TikTok, miniatures YouTube/Twitch, media X, previews Snapchat, /about YouTube, socialMedias Twitch GQL, liens sociaux Patreon) + pipeline étendu (banner_url, metric_maps, cross-links), snapshots quotidiens d'audience, insights/Ka Score /100 par créateur (croissance 7j/30j, engagement pondéré, rythme), fiche frontend enrichie (bannière héro, Ka Score, sparkline, bloc live) — acteurs versionnés dans actors/, 8 poussés et validés par runs de test
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
98 changed files +4,340 −13
added
actors/ka-discord/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-discord) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-discord/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-discord", | |
| 4 | + "title": "KA Discord — serveurs via invitation", | |
| 5 | + "description": "Serveurs Discord via l'API publique d'invitation : membres approximatifs, en ligne, boosts, nom/description. Code d'invit sensible à la casse.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-discord/.actor/input_schema.json
+38 −0
@@ -0,0 +1,38 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-discord input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles / codes", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles (patreon) ou codes d'invitation (discord).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [] | |
| 12 | + }, | |
| 13 | + "requestDelay": { | |
| 14 | + "title": "Délai (s)", | |
| 15 | + "type": "integer", | |
| 16 | + "default": 1, | |
| 17 | + "minimum": 0, | |
| 18 | + "description": "Pause entre requêtes." | |
| 19 | + }, | |
| 20 | + "concurrency": { | |
| 21 | + "title": "Concurrence", | |
| 22 | + "type": "integer", | |
| 23 | + "default": 5, | |
| 24 | + "minimum": 1, | |
| 25 | + "maximum": 10, | |
| 26 | + "description": "Requêtes simultanées." | |
| 27 | + }, | |
| 28 | + "proxyConfiguration": { | |
| 29 | + "title": "Proxy", | |
| 30 | + "type": "object", | |
| 31 | + "editor": "proxy", | |
| 32 | + "description": "Proxy Apify.", | |
| 33 | + "prefill": { | |
| 34 | + "useApifyProxy": true | |
| 35 | + } | |
| 36 | + } | |
| 37 | + } | |
| 38 | +} | |
| \ No newline at end of file | ||
added
actors/ka-discord/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-discord) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-discord/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-discord) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-discord/src/main.py
+87 −0
@@ -0,0 +1,87 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-discord) | |
| 4 | +# Desc: Serveurs Discord via l'API PUBLIQUE d'invitation | |
| 5 | +# /api/v9/invites/{code}?with_counts=true : membres approximatifs, | |
| 6 | +# présents en ligne, nom/description/boosts du serveur. Le CODE d'invit | |
| 7 | +# est SENSIBLE À LA CASSE → il vient de l'URL, pas du handle minusculé. | |
| 8 | +# ============================================================================== | |
| 9 | +from __future__ import annotations | |
| 10 | + | |
| 11 | +import asyncio | |
| 12 | +import json | |
| 13 | + | |
| 14 | +from apify import Actor | |
| 15 | + | |
| 16 | +from .net import Fetcher | |
| 17 | + | |
| 18 | +INVITE_API = ("https://discord.com/api/v9/invites/{code}" | |
| 19 | + "?with_counts=true&with_expiration=true") | |
| 20 | + | |
| 21 | + | |
| 22 | +async def main() -> None: | |
| 23 | + async with Actor: | |
| 24 | + inp = await Actor.get_input() or {} | |
| 25 | + # ici « usernames » = codes d'invitation (casse préservée par l'appelant) | |
| 26 | + codes = [c.strip() for c in (inp.get("usernames") or []) if c.strip()] | |
| 27 | + proxy = await Actor.create_proxy_configuration( | |
| 28 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 29 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 30 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 6)) | |
| 31 | + | |
| 32 | + async def one(code: str) -> None: | |
| 33 | + async with sem: | |
| 34 | + miss = {"kind": "profile", "platform": "discord", | |
| 35 | + "username": code, "found": False} | |
| 36 | + try: | |
| 37 | + resp = await fetcher.get( | |
| 38 | + INVITE_API.format(code=code), session_id=code[:12], | |
| 39 | + headers={"Accept": "application/json"}) | |
| 40 | + except Exception as exc: | |
| 41 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 42 | + return | |
| 43 | + if resp.status_code == 404: | |
| 44 | + await Actor.push_data({**miss, "error": "invite_invalide"}) | |
| 45 | + return | |
| 46 | + if resp.status_code != 200: | |
| 47 | + await Actor.push_data( | |
| 48 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 49 | + return | |
| 50 | + try: | |
| 51 | + d = json.loads(resp.text) | |
| 52 | + except Exception: | |
| 53 | + await Actor.push_data({**miss, "error": "bad_json"}) | |
| 54 | + return | |
| 55 | + guild = d.get("guild") or {} | |
| 56 | + icon = guild.get("icon") | |
| 57 | + gid = guild.get("id") | |
| 58 | + banner_hash = guild.get("banner") | |
| 59 | + splash_hash = guild.get("splash") | |
| 60 | + await Actor.push_data({ | |
| 61 | + "kind": "profile", | |
| 62 | + "platform": "discord", | |
| 63 | + "found": True, | |
| 64 | + "username": code, | |
| 65 | + "guild_id": gid, | |
| 66 | + "full_name": guild.get("name"), | |
| 67 | + "biography": guild.get("description"), | |
| 68 | + "followers": d.get("approximate_member_count"), # membres | |
| 69 | + "members": d.get("approximate_member_count"), | |
| 70 | + "online": d.get("approximate_presence_count"), | |
| 71 | + "boosts": guild.get("premium_subscription_count"), | |
| 72 | + "verified": ("VERIFIED" in (guild.get("features") or [])), | |
| 73 | + "partnered": ("PARTNERED" in (guild.get("features") or [])), | |
| 74 | + "channel": (d.get("channel") or {}).get("name"), | |
| 75 | + "avatar": (f"https://cdn.discordapp.com/icons/{gid}/{icon}.png" | |
| 76 | + if gid and icon else None), | |
| 77 | + "banner": (f"https://cdn.discordapp.com/banners/{gid}/" | |
| 78 | + f"{banner_hash}.png?size=1024" | |
| 79 | + if gid and banner_hash else None), | |
| 80 | + "splash": (f"https://cdn.discordapp.com/splashes/{gid}/" | |
| 81 | + f"{splash_hash}.jpg?size=1024" | |
| 82 | + if gid and splash_hash else None), | |
| 83 | + "vanity_url_code": guild.get("vanity_url_code"), | |
| 84 | + }) | |
| 85 | + | |
| 86 | + await asyncio.gather(*[one(c) for c in codes]) | |
| 87 | + Actor.log.info(f"terminé : {len(codes)} invitations") | |
added
actors/ka-discord/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-facebook/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-facebook) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-facebook/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-facebook", | |
| 4 | + "title": "KA Facebook — pages publiques", | |
| 5 | + "description": "Métadonnées de pages Facebook publiques (nom, abonnés/mentions J'aime, catégorie, avatar) extraites du HTML public via proxy résidentiel + empreinte Chrome. Expérimental : Facebook mure agressivement.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
added
actors/ka-facebook/.actor/input_schema.json
+43 −0
@@ -0,0 +1,43 @@ | ||
| 1 | +{ | |
| 2 | + "title": "KA Facebook input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Pages à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Slugs de pages Facebook (facebook.com/<slug>) ou URLs complètes.", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "facebook" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "requestDelay": { | |
| 16 | + "title": "Délai entre requêtes (s)", | |
| 17 | + "type": "integer", | |
| 18 | + "default": 2, | |
| 19 | + "minimum": 0, | |
| 20 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 21 | + }, | |
| 22 | + "concurrency": { | |
| 23 | + "title": "Requêtes simultanées", | |
| 24 | + "type": "integer", | |
| 25 | + "default": 2, | |
| 26 | + "minimum": 1, | |
| 27 | + "maximum": 6, | |
| 28 | + "description": "Nombre de requêtes menées en parallèle." | |
| 29 | + }, | |
| 30 | + "proxyConfiguration": { | |
| 31 | + "title": "Proxy", | |
| 32 | + "type": "object", | |
| 33 | + "editor": "proxy", | |
| 34 | + "description": "Résidentiel obligatoire (Facebook rejette les IP datacenter).", | |
| 35 | + "prefill": { | |
| 36 | + "useApifyProxy": true, | |
| 37 | + "apifyProxyGroups": [ | |
| 38 | + "RESIDENTIAL" | |
| 39 | + ] | |
| 40 | + } | |
| 41 | + } | |
| 42 | + } | |
| 43 | +} | |
| \ No newline at end of file | ||
added
actors/ka-facebook/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-facebook) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-facebook/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-facebook) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-facebook/src/main.py
+121 −0
@@ -0,0 +1,121 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-facebook) | |
| 4 | +# Desc: Pages Facebook publiques — extraction best-effort du HTML servi aux | |
| 5 | +# navigateurs non connectés : compteur d'abonnés (JSON embarqué | |
| 6 | +# follower_count / texte « N followers/abonnés »), nom (og:title), | |
| 7 | +# avatar (og:image), catégorie. EXPÉRIMENTAL : FB mure agressivement ; | |
| 8 | +# un résultat found=False n'est pas une erreur du pipeline. | |
| 9 | +# ============================================================================== | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import asyncio | |
| 13 | +import html as htmllib | |
| 14 | +import re | |
| 15 | + | |
| 16 | +from apify import Actor | |
| 17 | + | |
| 18 | +from .net import Fetcher | |
| 19 | + | |
| 20 | +PAGE_URL = "https://www.facebook.com/{u}" | |
| 21 | + | |
| 22 | +_OG_RE = { | |
| 23 | + "title": re.compile(r'<meta property="og:title" content="([^"]*)"'), | |
| 24 | + "image": re.compile(r'<meta property="og:image" content="([^"]*)"'), | |
| 25 | + "description": re.compile( | |
| 26 | + r'<meta (?:property="og:description"|name="description") ' | |
| 27 | + r'content="([^"]*)"'), | |
| 28 | +} | |
| 29 | +_FOLLOWER_RES = ( | |
| 30 | + re.compile(r'"follower_count"\s*:\s*(\d+)'), | |
| 31 | + re.compile(r'"global_likers_count"\s*:\s*(\d+)'), | |
| 32 | +) | |
| 33 | +# texte « 1,2 M followers » / « 12 k abonnés » / « 4 016 971 mentions J’aime » | |
| 34 | +_TEXT_COUNT_RE = re.compile( | |
| 35 | + r'([\d][\d\s .,]*)\s*([KkMm]?)\s*' | |
| 36 | + r'(?:followers|abonnés|mentions\s+J’aime|J’aime|likes)', | |
| 37 | + re.I) | |
| 38 | +_CATEGORY_RE = re.compile(r'"category_name"\s*:\s*"([^"]+)"') | |
| 39 | +_VERIFIED_RE = re.compile(r'"is_verified"\s*:\s*(true|false)') | |
| 40 | + | |
| 41 | + | |
| 42 | +def parse_text_count(text: str) -> int | None: | |
| 43 | + m = _TEXT_COUNT_RE.search(text or "") | |
| 44 | + if not m: | |
| 45 | + return None | |
| 46 | + num = m.group(1).replace(" ", "").replace(" ", "").replace(",", ".") | |
| 47 | + try: | |
| 48 | + val = float(num) | |
| 49 | + except ValueError: | |
| 50 | + return None | |
| 51 | + unit = (m.group(2) or "").lower() | |
| 52 | + if unit == "k": | |
| 53 | + val *= 1_000 | |
| 54 | + elif unit == "m": | |
| 55 | + val *= 1_000_000 | |
| 56 | + return int(val) | |
| 57 | + | |
| 58 | + | |
| 59 | +async def main() -> None: | |
| 60 | + async with Actor: | |
| 61 | + inp = await Actor.get_input() or {} | |
| 62 | + usernames = [] | |
| 63 | + for raw in (inp.get("usernames") or []): | |
| 64 | + raw = (raw or "").strip() | |
| 65 | + if not raw: | |
| 66 | + continue | |
| 67 | + raw = re.sub(r"^https?://(www\.|m\.)?facebook\.com/", "", raw) | |
| 68 | + usernames.append(raw.strip("/").split("?")[0]) | |
| 69 | + proxy = await Actor.create_proxy_configuration( | |
| 70 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 71 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 2)) | |
| 72 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 2)) | |
| 73 | + | |
| 74 | + async def one(u: str) -> None: | |
| 75 | + async with sem: | |
| 76 | + miss = {"kind": "profile", "platform": "facebook", | |
| 77 | + "username": u, "found": False} | |
| 78 | + try: | |
| 79 | + resp = await fetcher.get(PAGE_URL.format(u=u), | |
| 80 | + session_id=u) | |
| 81 | + except Exception as exc: | |
| 82 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 83 | + return | |
| 84 | + text = resp.text or "" | |
| 85 | + if resp.status_code != 200 or len(text) < 5000: | |
| 86 | + await Actor.push_data( | |
| 87 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 88 | + return | |
| 89 | + followers = None | |
| 90 | + for rx in _FOLLOWER_RES: | |
| 91 | + m = rx.search(text) | |
| 92 | + if m: | |
| 93 | + followers = int(m.group(1)) | |
| 94 | + break | |
| 95 | + og = {k: htmllib.unescape(m.group(1)) if (m := rx.search(text)) | |
| 96 | + else None for k, rx in _OG_RE.items()} | |
| 97 | + if followers is None: | |
| 98 | + followers = parse_text_count(og.get("description") or "") | |
| 99 | + if followers is None: # texte « N followers/abonnés » du corps | |
| 100 | + followers = parse_text_count(text) | |
| 101 | + if followers is None and not og.get("title"): | |
| 102 | + await Actor.push_data({**miss, "error": "login_wall"}) | |
| 103 | + return | |
| 104 | + cat = _CATEGORY_RE.search(text) | |
| 105 | + ver = _VERIFIED_RE.search(text) | |
| 106 | + await Actor.push_data({ | |
| 107 | + "kind": "profile", | |
| 108 | + "platform": "facebook", | |
| 109 | + "found": True, | |
| 110 | + "username": u, | |
| 111 | + "full_name": og.get("title"), | |
| 112 | + "biography": og.get("description"), | |
| 113 | + "followers": followers, | |
| 114 | + "category": cat.group(1) if cat else None, | |
| 115 | + "is_verified": (ver.group(1) == "true") if ver else None, | |
| 116 | + "avatar": og.get("image"), | |
| 117 | + "og_description": og.get("description"), | |
| 118 | + }) | |
| 119 | + | |
| 120 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 121 | + Actor.log.info(f"terminé : {len(usernames)} pages") | |
added
actors/ka-facebook/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-fansly/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-fansly) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-fansly/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-fansly", | |
| 4 | + "title": "KA Fansly — profils", | |
| 5 | + "description": "Profils Fansly via l'API publique (abonnés, bio, badge, compteurs de médias, avatar) — lots de 20 handles.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-fansly/.actor/input_schema.json
+38 −0
@@ -0,0 +1,38 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-fansly input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles/slugs à traiter (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [] | |
| 12 | + }, | |
| 13 | + "requestDelay": { | |
| 14 | + "title": "Délai entre requêtes (s)", | |
| 15 | + "type": "integer", | |
| 16 | + "default": 1, | |
| 17 | + "minimum": 0, | |
| 18 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 19 | + }, | |
| 20 | + "concurrency": { | |
| 21 | + "title": "Requêtes simultanées", | |
| 22 | + "type": "integer", | |
| 23 | + "default": 3, | |
| 24 | + "minimum": 1, | |
| 25 | + "maximum": 10, | |
| 26 | + "description": "Nombre de requêtes menées en parallèle." | |
| 27 | + }, | |
| 28 | + "proxyConfiguration": { | |
| 29 | + "title": "Proxy", | |
| 30 | + "type": "object", | |
| 31 | + "editor": "proxy", | |
| 32 | + "description": "Configuration du proxy Apify.", | |
| 33 | + "prefill": { | |
| 34 | + "useApifyProxy": true | |
| 35 | + } | |
| 36 | + } | |
| 37 | + } | |
| 38 | +} | |
| \ No newline at end of file | ||
added
actors/ka-fansly/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-fansly) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-fansly/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-fansly) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-fansly/src/main.py
+93 −0
@@ -0,0 +1,93 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-fansly) | |
| 4 | +# Desc: Profils Fansly via l'API publique apiv3.fansly.com/api/v1/account | |
| 5 | +# (?usernames=) : abonnés (followCount), bio, badge, compteurs de | |
| 6 | +# médias (images/vidéos), avatar/bannière. Lots de 20 handles/requête. | |
| 7 | +# ============================================================================== | |
| 8 | +from __future__ import annotations | |
| 9 | + | |
| 10 | +import asyncio | |
| 11 | +import json | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | + | |
| 15 | +from .net import Fetcher | |
| 16 | + | |
| 17 | +API_URL = ("https://apiv3.fansly.com/api/v1/account" | |
| 18 | + "?usernames={us}&ngsw-bypass=true") | |
| 19 | + | |
| 20 | + | |
| 21 | +def _loc(media: dict | None) -> str | None: | |
| 22 | + """Objet média Fansly → URL de la première variante disponible.""" | |
| 23 | + if not isinstance(media, dict): | |
| 24 | + return None | |
| 25 | + for variant in ([media] + (media.get("variants") or [])): | |
| 26 | + for loc in (variant.get("locations") or []): | |
| 27 | + if loc.get("location"): | |
| 28 | + return loc["location"] | |
| 29 | + return None | |
| 30 | + | |
| 31 | + | |
| 32 | +async def main() -> None: | |
| 33 | + async with Actor: | |
| 34 | + inp = await Actor.get_input() or {} | |
| 35 | + usernames = [u.strip().lstrip("@").lower() | |
| 36 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 37 | + proxy = await Actor.create_proxy_configuration( | |
| 38 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 39 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 40 | + | |
| 41 | + found: set[str] = set() | |
| 42 | + for i in range(0, len(usernames), 20): | |
| 43 | + batch = usernames[i:i + 20] | |
| 44 | + try: | |
| 45 | + resp = await fetcher.get(API_URL.format(us=",".join(batch)), | |
| 46 | + session_id=f"b{i}", | |
| 47 | + headers={"Accept": | |
| 48 | + "application/json"}) | |
| 49 | + data = (json.loads(resp.text) | |
| 50 | + if resp.status_code == 200 else {}) | |
| 51 | + except Exception as exc: | |
| 52 | + for u in batch: | |
| 53 | + await Actor.push_data( | |
| 54 | + {"kind": "profile", "platform": "fansly", | |
| 55 | + "username": u, "found": False, | |
| 56 | + "error": str(exc)[:200]}) | |
| 57 | + continue | |
| 58 | + for acc in (data.get("response") or []): | |
| 59 | + u = str(acc.get("username") or "").lower() | |
| 60 | + if not u: | |
| 61 | + continue | |
| 62 | + found.add(u) | |
| 63 | + stats = acc.get("timelineStats") or {} | |
| 64 | + await Actor.push_data({ | |
| 65 | + "kind": "profile", | |
| 66 | + "platform": "fansly", | |
| 67 | + "found": True, | |
| 68 | + "id": acc.get("id"), | |
| 69 | + "username": u, | |
| 70 | + "full_name": acc.get("displayName") or u, | |
| 71 | + "biography": (acc.get("about") or "")[:1000], | |
| 72 | + "followers": acc.get("followCount"), | |
| 73 | + "following": acc.get("followingCount"), | |
| 74 | + "is_verified": bool((acc.get("statusInfo") or {}) | |
| 75 | + .get("verified") | |
| 76 | + or acc.get("verified")), | |
| 77 | + "posts_count": acc.get("postCount") | |
| 78 | + or stats.get("postCount"), | |
| 79 | + "images_count": stats.get("imageCount"), | |
| 80 | + "videos_count": stats.get("videoCount"), | |
| 81 | + "likes": acc.get("accountMediaLikes"), | |
| 82 | + "avatar": _loc(acc.get("avatar")), | |
| 83 | + "banner": _loc(acc.get("banner")), | |
| 84 | + "location": acc.get("location"), | |
| 85 | + }) | |
| 86 | + for u in batch: | |
| 87 | + if u not in found: | |
| 88 | + await Actor.push_data( | |
| 89 | + {"kind": "profile", "platform": "fansly", | |
| 90 | + "username": u, "found": False, | |
| 91 | + "error": f"not_found_{resp.status_code}"}) | |
| 92 | + Actor.log.info(f"terminé : {len(usernames)} profils, " | |
| 93 | + f"{len(found)} trouvés") | |
added
actors/ka-fansly/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-instagram/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-instagram) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-instagram/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-instagram", | |
| 4 | + "title": "KA Instagram — profils détaillés + recherche", | |
| 5 | + "description": "Profils Instagram publics ultra détaillés (abonnés, bio, liens, 12 derniers posts avec likes/commentaires/vues, engagement, profils reliés) + recherche d'utilisateurs par mot-clé. Endpoints web publics via proxy résidentiel.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
added
actors/ka-instagram/.actor/input_schema.json
+49 −0
@@ -0,0 +1,49 @@ | ||
| 1 | +{ | |
| 2 | + "title": "KA Instagram input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles Instagram (sans @) dont on veut le profil détaillé.", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "instagram" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "queries": { | |
| 16 | + "title": "Recherches (découverte)", | |
| 17 | + "type": "array", | |
| 18 | + "description": "Mots-clés de recherche d'utilisateurs (topsearch).", | |
| 19 | + "editor": "stringList" | |
| 20 | + }, | |
| 21 | + "requestDelay": { | |
| 22 | + "title": "Délai entre requêtes (s)", | |
| 23 | + "type": "integer", | |
| 24 | + "default": 1, | |
| 25 | + "minimum": 0, | |
| 26 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 27 | + }, | |
| 28 | + "concurrency": { | |
| 29 | + "title": "Requêtes simultanées", | |
| 30 | + "type": "integer", | |
| 31 | + "default": 3, | |
| 32 | + "minimum": 1, | |
| 33 | + "maximum": 10, | |
| 34 | + "description": "Nombre de requêtes menées en parallèle." | |
| 35 | + }, | |
| 36 | + "proxyConfiguration": { | |
| 37 | + "title": "Proxy", | |
| 38 | + "type": "object", | |
| 39 | + "editor": "proxy", | |
| 40 | + "description": "Résidentiel recommandé (Instagram bloque les IP datacenter).", | |
| 41 | + "prefill": { | |
| 42 | + "useApifyProxy": true, | |
| 43 | + "apifyProxyGroups": [ | |
| 44 | + "RESIDENTIAL" | |
| 45 | + ] | |
| 46 | + } | |
| 47 | + } | |
| 48 | + } | |
| 49 | +} | |
| \ No newline at end of file | ||
added
actors/ka-instagram/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-instagram) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-instagram/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-instagram) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-instagram/src/main.py
+243 −0
@@ -0,0 +1,243 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-instagram) | |
| 4 | +# Desc: Profils Instagram publics ULTRA DÉTAILLÉS via l'endpoint web officiel | |
| 5 | +# web_profile_info (JSON du web public) : identité complète, compteurs, | |
| 6 | +# 12 derniers posts (likes/commentaires/vues/légende/musique/lieu/ | |
| 7 | +# co-auteurs/épinglés), agrégats d'engagement et de cadence, profils | |
| 8 | +# reliés — + recherche d'utilisateurs (topsearch) en découverte. | |
| 9 | +# ============================================================================== | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import asyncio | |
| 13 | +import json | |
| 14 | +import urllib.parse | |
| 15 | +from datetime import datetime, timezone | |
| 16 | + | |
| 17 | +from apify import Actor | |
| 18 | + | |
| 19 | +from .net import Fetcher | |
| 20 | + | |
| 21 | +PROFILE_API = ("https://i.instagram.com/api/v1/users/web_profile_info/" | |
| 22 | + "?username={u}") | |
| 23 | +SEARCH_API = ("https://www.instagram.com/web/search/topsearch/" | |
| 24 | + "?context=blended&query={q}&count=30") | |
| 25 | +SEARCH_API_ALT = ("https://i.instagram.com/api/v1/web/search/topsearch/" | |
| 26 | + "?context=blended&query={q}&count=30") | |
| 27 | +HDRS = {"x-ig-app-id": "936619743392459", "Accept": "application/json"} | |
| 28 | + | |
| 29 | + | |
| 30 | +def _iso(ts) -> str | None: | |
| 31 | + try: | |
| 32 | + return datetime.fromtimestamp(int(ts), tz=timezone.utc) \ | |
| 33 | + .strftime("%Y-%m-%dT%H:%M:%SZ") | |
| 34 | + except (TypeError, ValueError, OSError): | |
| 35 | + return None | |
| 36 | + | |
| 37 | + | |
| 38 | +def _avg(vals: list) -> int | None: | |
| 39 | + vals = [v for v in vals if isinstance(v, (int, float))] | |
| 40 | + return round(sum(vals) / len(vals)) if vals else None | |
| 41 | + | |
| 42 | + | |
| 43 | +def parse_post(node: dict) -> dict: | |
| 44 | + """Nœud média du fil → post détaillé (métriques, musique, lieu, épinglé).""" | |
| 45 | + cap = (node.get("edge_media_to_caption") or {}).get("edges") or [] | |
| 46 | + music = node.get("clips_music_attribution_info") or {} | |
| 47 | + dims = node.get("dimensions") or {} | |
| 48 | + return { | |
| 49 | + "id": node.get("id"), | |
| 50 | + "shortcode": node.get("shortcode"), | |
| 51 | + "url": f"https://www.instagram.com/p/{node.get('shortcode')}/", | |
| 52 | + "type": ("reel" if node.get("product_type") == "clips" | |
| 53 | + else "video" if node.get("is_video") else | |
| 54 | + "carousel" if node.get("edge_sidecar_to_children") | |
| 55 | + else "image"), | |
| 56 | + "caption": ((cap[0].get("node") or {}).get("text", "") | |
| 57 | + if cap else "")[:500], | |
| 58 | + "likes": (node.get("edge_liked_by") or {}).get("count"), | |
| 59 | + "comments": (node.get("edge_media_to_comment") or {}).get("count"), | |
| 60 | + "video_views": node.get("video_view_count"), | |
| 61 | + "video_duration_s": node.get("video_duration"), | |
| 62 | + "timestamp": node.get("taken_at_timestamp"), | |
| 63 | + "posted_at": _iso(node.get("taken_at_timestamp")), | |
| 64 | + "display_url": node.get("display_url"), | |
| 65 | + "thumbnail_url": node.get("thumbnail_src"), | |
| 66 | + "width": dims.get("width"), | |
| 67 | + "height": dims.get("height"), | |
| 68 | + "is_pinned": bool(node.get("pinned_for_users")), | |
| 69 | + "comments_disabled": node.get("comments_disabled"), | |
| 70 | + "location": (node.get("location") or {}).get("name"), | |
| 71 | + "coauthors": [c.get("username") | |
| 72 | + for c in (node.get("coauthor_producers") or []) | |
| 73 | + if c.get("username")], | |
| 74 | + "music": ({"artist": music.get("artist_name"), | |
| 75 | + "song": music.get("song_name"), | |
| 76 | + "uses_original_audio": music.get("uses_original_audio")} | |
| 77 | + if music.get("song_name") or music.get("artist_name") | |
| 78 | + else None), | |
| 79 | + "accessibility_caption": (node.get("accessibility_caption") | |
| 80 | + or "")[:200] or None, | |
| 81 | + "tagged_users": [((t.get("node") or {}).get("user") or {}) | |
| 82 | + .get("username") | |
| 83 | + for t in ((node.get("edge_media_to_tagged_user") | |
| 84 | + or {}).get("edges") or [])][:10], | |
| 85 | + } | |
| 86 | + | |
| 87 | + | |
| 88 | +def parse_user(user: dict) -> dict: | |
| 89 | + """Objet user web_profile_info → enregistrement riche standardisé.""" | |
| 90 | + posts = [parse_post((e.get("node") or {})) | |
| 91 | + for e in ((user.get("edge_owner_to_timeline_media") or {}) | |
| 92 | + .get("edges") or [])] | |
| 93 | + followers = (user.get("edge_followed_by") or {}).get("count") | |
| 94 | + avg_likes = _avg([p["likes"] for p in posts]) | |
| 95 | + avg_comments = _avg([p["comments"] for p in posts]) | |
| 96 | + videos = [p for p in posts if p["type"] in ("video", "reel")] | |
| 97 | + avg_video_views = _avg([p["video_views"] for p in videos]) | |
| 98 | + engagement = None | |
| 99 | + if followers and avg_likes is not None: | |
| 100 | + engagement = round( | |
| 101 | + (avg_likes + (avg_comments or 0)) / followers * 100, 3) | |
| 102 | + timestamps = sorted(p["timestamp"] for p in posts | |
| 103 | + if isinstance(p["timestamp"], int)) | |
| 104 | + posts_per_week = None | |
| 105 | + if len(timestamps) >= 3 and timestamps[-1] > timestamps[0]: | |
| 106 | + span_weeks = (timestamps[-1] - timestamps[0]) / 604_800 | |
| 107 | + posts_per_week = round((len(timestamps) - 1) / span_weeks, 2) | |
| 108 | + top_post = max((p for p in posts if isinstance(p["likes"], int)), | |
| 109 | + key=lambda p: p["likes"], default=None) | |
| 110 | + related = [] | |
| 111 | + for edge in ((user.get("edge_related_profiles") or {}).get("edges") or []): | |
| 112 | + n = edge.get("node") or {} | |
| 113 | + if n.get("username"): | |
| 114 | + related.append({"username": n["username"], | |
| 115 | + "full_name": n.get("full_name"), | |
| 116 | + "is_verified": n.get("is_verified"), | |
| 117 | + "is_private": n.get("is_private")}) | |
| 118 | + return { | |
| 119 | + "kind": "profile", | |
| 120 | + "platform": "instagram", | |
| 121 | + "found": True, | |
| 122 | + "id": user.get("id"), | |
| 123 | + "username": user.get("username"), | |
| 124 | + "full_name": user.get("full_name"), | |
| 125 | + "biography": user.get("biography"), | |
| 126 | + "pronouns": user.get("pronouns") or None, | |
| 127 | + "external_url": user.get("external_url"), | |
| 128 | + "bio_links": [b.get("url") for b in (user.get("bio_links") or []) | |
| 129 | + if b.get("url")], | |
| 130 | + "followers": followers, | |
| 131 | + "following": (user.get("edge_follow") or {}).get("count"), | |
| 132 | + "posts_count": (user.get("edge_owner_to_timeline_media") | |
| 133 | + or {}).get("count"), | |
| 134 | + "highlight_reels": user.get("highlight_reel_count"), | |
| 135 | + "is_verified": user.get("is_verified"), | |
| 136 | + "is_private": user.get("is_private"), | |
| 137 | + "is_business": user.get("is_business_account"), | |
| 138 | + "is_professional": user.get("is_professional_account"), | |
| 139 | + "category": user.get("category_name"), | |
| 140 | + "business_category": user.get("business_category_name"), | |
| 141 | + "business_address": user.get("business_address_json"), | |
| 142 | + "has_clips": user.get("has_clips"), | |
| 143 | + "has_channel": user.get("has_channel"), | |
| 144 | + "has_ar_effects": user.get("has_ar_effects"), | |
| 145 | + "recently_joined": user.get("is_joined_recently"), | |
| 146 | + "avatar": user.get("profile_pic_url_hd") | |
| 147 | + or user.get("profile_pic_url"), | |
| 148 | + # agrégats calculés sur les 12 derniers posts | |
| 149 | + "avg_likes": avg_likes, | |
| 150 | + "avg_comments": avg_comments, | |
| 151 | + "avg_video_views": avg_video_views, | |
| 152 | + "engagement_rate_pct": engagement, | |
| 153 | + "posts_per_week": posts_per_week, | |
| 154 | + "last_post_at": _iso(timestamps[-1]) if timestamps else None, | |
| 155 | + "video_share_pct": (round(len(videos) / len(posts) * 100) | |
| 156 | + if posts else None), | |
| 157 | + "top_post": ({"url": top_post["url"], "likes": top_post["likes"], | |
| 158 | + "comments": top_post["comments"], | |
| 159 | + "type": top_post["type"]} if top_post else None), | |
| 160 | + "recent_posts": posts, | |
| 161 | + "related_profiles": related, | |
| 162 | + } | |
| 163 | + | |
| 164 | + | |
| 165 | +async def main() -> None: | |
| 166 | + async with Actor: | |
| 167 | + inp = await Actor.get_input() or {} | |
| 168 | + usernames = [u.strip().lstrip("@").lower() | |
| 169 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 170 | + queries = [q.strip() for q in (inp.get("queries") or []) | |
| 171 | + if q and q.strip()] | |
| 172 | + proxy = await Actor.create_proxy_configuration( | |
| 173 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 174 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 175 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 3)) | |
| 176 | + | |
| 177 | + async def one_profile(u: str) -> None: | |
| 178 | + async with sem: | |
| 179 | + miss = {"kind": "profile", "platform": "instagram", | |
| 180 | + "username": u, "found": False} | |
| 181 | + try: | |
| 182 | + resp = await fetcher.get(PROFILE_API.format(u=u), | |
| 183 | + headers=HDRS, session_id=u) | |
| 184 | + except Exception as exc: | |
| 185 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 186 | + return | |
| 187 | + if resp.status_code == 404: | |
| 188 | + await Actor.push_data({**miss, "error": "http_404"}) | |
| 189 | + return | |
| 190 | + user = None | |
| 191 | + if resp.status_code == 200: | |
| 192 | + try: | |
| 193 | + user = ((json.loads(resp.text).get("data") or {}) | |
| 194 | + .get("user")) | |
| 195 | + except Exception: | |
| 196 | + user = None | |
| 197 | + if not user: | |
| 198 | + await Actor.push_data( | |
| 199 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 200 | + return | |
| 201 | + await Actor.push_data(parse_user(user)) | |
| 202 | + | |
| 203 | + async def one_search(q: str) -> None: | |
| 204 | + async with sem: | |
| 205 | + data = {} | |
| 206 | + for endpoint in (SEARCH_API, SEARCH_API_ALT): | |
| 207 | + u2 = endpoint.format(q=urllib.parse.quote(q)) | |
| 208 | + try: | |
| 209 | + resp = await fetcher.get(u2, headers=HDRS, | |
| 210 | + session_id=f"srch{q[:4]}") | |
| 211 | + except Exception as exc: | |
| 212 | + Actor.log.warning(f"recherche '{q}' : {exc}") | |
| 213 | + continue | |
| 214 | + if resp.status_code == 200: | |
| 215 | + try: | |
| 216 | + data = json.loads(resp.text) | |
| 217 | + except Exception: | |
| 218 | + data = {} | |
| 219 | + if data.get("users"): | |
| 220 | + break | |
| 221 | + if not data.get("users"): | |
| 222 | + Actor.log.warning( | |
| 223 | + f"recherche '{q}' : 0 user " | |
| 224 | + f"(http {resp.status_code}) {resp.text[:200]!r}") | |
| 225 | + for item in data.get("users") or []: | |
| 226 | + u = item.get("user") or {} | |
| 227 | + if not u.get("username") or u.get("is_private"): | |
| 228 | + continue | |
| 229 | + await Actor.push_data({ | |
| 230 | + "kind": "search_user", | |
| 231 | + "platform": "instagram", | |
| 232 | + "query": q, | |
| 233 | + "username": u.get("username"), | |
| 234 | + "full_name": u.get("full_name"), | |
| 235 | + "is_verified": u.get("is_verified"), | |
| 236 | + "followers": u.get("follower_count"), | |
| 237 | + "avatar": u.get("profile_pic_url"), | |
| 238 | + }) | |
| 239 | + | |
| 240 | + await asyncio.gather(*[one_profile(u) for u in usernames], | |
| 241 | + *[one_search(q) for q in queries]) | |
| 242 | + Actor.log.info(f"terminé : {len(usernames)} profils, " | |
| 243 | + f"{len(queries)} recherches") | |
added
actors/ka-instagram/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-kick/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-kick) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-kick/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-kick", | |
| 4 | + "title": "KA Kick — chaînes détaillées", | |
| 5 | + "description": "Chaînes Kick via l'API v2 publique (abonnés, live, bio + liens sociaux auto-déclarés) — proxy résidentiel pour Cloudflare.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-kick/.actor/input_schema.json
+43 −0
@@ -0,0 +1,43 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-kick input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles/slugs à traiter (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "xqc" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "requestDelay": { | |
| 16 | + "title": "Délai entre requêtes (s)", | |
| 17 | + "type": "integer", | |
| 18 | + "default": 1, | |
| 19 | + "minimum": 0, | |
| 20 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 21 | + }, | |
| 22 | + "concurrency": { | |
| 23 | + "title": "Requêtes simultanées", | |
| 24 | + "type": "integer", | |
| 25 | + "default": 3, | |
| 26 | + "minimum": 1, | |
| 27 | + "maximum": 10, | |
| 28 | + "description": "Nombre de requêtes menées en parallèle." | |
| 29 | + }, | |
| 30 | + "proxyConfiguration": { | |
| 31 | + "title": "Proxy", | |
| 32 | + "type": "object", | |
| 33 | + "editor": "proxy", | |
| 34 | + "description": "Configuration du proxy Apify.", | |
| 35 | + "prefill": { | |
| 36 | + "useApifyProxy": true, | |
| 37 | + "apifyProxyGroups": [ | |
| 38 | + "RESIDENTIAL" | |
| 39 | + ] | |
| 40 | + } | |
| 41 | + } | |
| 42 | + } | |
| 43 | +} | |
| \ No newline at end of file | ||
added
actors/ka-kick/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-kick) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-kick/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-kick) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-kick/src/main.py
+103 −0
@@ -0,0 +1,103 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-kick) | |
| 4 | +# Desc: Chaînes Kick détaillées via l'API publique /api/v2/channels/{slug} | |
| 5 | +# (Cloudflare : proxy résidentiel + empreinte Chrome) : abonnés, badge, | |
| 6 | +# live en cours, bio + LIENS SOCIAUX AUTO-DÉCLARÉS (instagram/twitter/ | |
| 7 | +# youtube/tiktok/facebook/discord → cross_link 0.90 côté crea-ka). | |
| 8 | +# ============================================================================== | |
| 9 | +from __future__ import annotations | |
| 10 | + | |
| 11 | +import asyncio | |
| 12 | +import json | |
| 13 | + | |
| 14 | +from apify import Actor | |
| 15 | + | |
| 16 | +from .net import Fetcher | |
| 17 | + | |
| 18 | +API_URL = "https://kick.com/api/v2/channels/{u}" | |
| 19 | +SOCIAL_KEYS = ("instagram", "twitter", "youtube", "tiktok", "facebook", | |
| 20 | + "discord") | |
| 21 | + | |
| 22 | + | |
| 23 | +async def main() -> None: | |
| 24 | + async with Actor: | |
| 25 | + inp = await Actor.get_input() or {} | |
| 26 | + usernames = [u.strip().lstrip("@").lower() | |
| 27 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 28 | + proxy = await Actor.create_proxy_configuration( | |
| 29 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 30 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 31 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 3)) | |
| 32 | + | |
| 33 | + async def one(u: str) -> None: | |
| 34 | + async with sem: | |
| 35 | + miss = {"kind": "profile", "platform": "kick", | |
| 36 | + "username": u, "found": False} | |
| 37 | + resp = None | |
| 38 | + for attempt in range(3): # mur Cloudflare → nouvelle IP | |
| 39 | + try: | |
| 40 | + resp = await fetcher.get( | |
| 41 | + API_URL.format(u=u), session_id=f"{u}k{attempt}", | |
| 42 | + headers={"Accept": "application/json"}) | |
| 43 | + except Exception as exc: | |
| 44 | + await Actor.push_data({**miss, | |
| 45 | + "error": str(exc)[:200]}) | |
| 46 | + return | |
| 47 | + if resp.status_code == 404: | |
| 48 | + await Actor.push_data({**miss, "error": "http_404"}) | |
| 49 | + return | |
| 50 | + if resp.status_code == 200 and \ | |
| 51 | + (resp.text or "").startswith("{"): | |
| 52 | + break | |
| 53 | + if resp.status_code != 200 or \ | |
| 54 | + not (resp.text or "").startswith("{"): | |
| 55 | + await Actor.push_data( | |
| 56 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 57 | + return | |
| 58 | + try: | |
| 59 | + ch = json.loads(resp.text) | |
| 60 | + except Exception: | |
| 61 | + await Actor.push_data({**miss, "error": "bad_json"}) | |
| 62 | + return | |
| 63 | + user = ch.get("user") or {} | |
| 64 | + live = ch.get("livestream") or {} | |
| 65 | + cats = [((c.get("category") or {}).get("name")) | |
| 66 | + for c in (ch.get("recent_categories") or []) | |
| 67 | + if isinstance(c, dict)] | |
| 68 | + socials = {k: (user.get(k) or "").strip() | |
| 69 | + for k in SOCIAL_KEYS if (user.get(k) or "").strip()} | |
| 70 | + await Actor.push_data({ | |
| 71 | + "kind": "profile", | |
| 72 | + "platform": "kick", | |
| 73 | + "found": True, | |
| 74 | + "id": ch.get("id"), | |
| 75 | + "username": ch.get("slug"), | |
| 76 | + "full_name": user.get("username"), | |
| 77 | + "biography": user.get("bio"), | |
| 78 | + "followers": ch.get("followers_count") | |
| 79 | + or ch.get("followersCount"), | |
| 80 | + "is_verified": bool(ch.get("verified")), | |
| 81 | + "is_banned": bool(ch.get("is_banned")), | |
| 82 | + "subscription_enabled": ch.get("subscription_enabled"), | |
| 83 | + "is_live_now": bool(live), | |
| 84 | + "live_viewers": live.get("viewer_count"), | |
| 85 | + "live_title": (live.get("session_title") or "")[:200] | |
| 86 | + or None, | |
| 87 | + "live_thumbnail": ((live.get("thumbnail") or {}).get("url") | |
| 88 | + if isinstance(live.get("thumbnail"), | |
| 89 | + dict) else None), | |
| 90 | + "live_category": next( | |
| 91 | + (c.get("name") for c in (live.get("categories") or []) | |
| 92 | + if isinstance(c, dict) and c.get("name")), None), | |
| 93 | + "vod_enabled": ch.get("vod_enabled"), | |
| 94 | + "avatar": user.get("profile_pic"), | |
| 95 | + "banner": ((ch.get("banner_image") or {}).get("url") | |
| 96 | + if isinstance(ch.get("banner_image"), dict) | |
| 97 | + else None), | |
| 98 | + "recent_categories": [c for c in cats if c][:5], | |
| 99 | + "social_links": socials, | |
| 100 | + }) | |
| 101 | + | |
| 102 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 103 | + Actor.log.info(f"terminé : {len(usernames)} chaînes") | |
added
actors/ka-kick/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-onlyfans/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-onlyfans) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-onlyfans/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-onlyfans", | |
| 4 | + "title": "KA OnlyFans — profils publics", | |
| 5 | + "description": "Profils OnlyFans publics best-effort (posts, photos, vidéos, J'aime, prix d'abonnement, bio, avatar) — EXPÉRIMENTAL.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-onlyfans/.actor/input_schema.json
+41 −0
@@ -0,0 +1,41 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-onlyfans input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles/slugs à traiter (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [] | |
| 12 | + }, | |
| 13 | + "requestDelay": { | |
| 14 | + "title": "Délai entre requêtes (s)", | |
| 15 | + "type": "integer", | |
| 16 | + "default": 1, | |
| 17 | + "minimum": 0, | |
| 18 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 19 | + }, | |
| 20 | + "concurrency": { | |
| 21 | + "title": "Requêtes simultanées", | |
| 22 | + "type": "integer", | |
| 23 | + "default": 3, | |
| 24 | + "minimum": 1, | |
| 25 | + "maximum": 10, | |
| 26 | + "description": "Nombre de requêtes menées en parallèle." | |
| 27 | + }, | |
| 28 | + "proxyConfiguration": { | |
| 29 | + "title": "Proxy", | |
| 30 | + "type": "object", | |
| 31 | + "editor": "proxy", | |
| 32 | + "description": "Configuration du proxy Apify.", | |
| 33 | + "prefill": { | |
| 34 | + "useApifyProxy": true, | |
| 35 | + "apifyProxyGroups": [ | |
| 36 | + "RESIDENTIAL" | |
| 37 | + ] | |
| 38 | + } | |
| 39 | + } | |
| 40 | + } | |
| 41 | +} | |
| \ No newline at end of file | ||
added
actors/ka-onlyfans/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-onlyfans) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-onlyfans/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-onlyfans) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-onlyfans/src/main.py
+112 −0
@@ -0,0 +1,112 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-onlyfans) | |
| 4 | +# Desc: Profils OnlyFans publics (page de profil hors connexion) — extraction | |
| 5 | +# best-effort des compteurs embarqués : posts, photos, vidéos, J'aime | |
| 6 | +# (favoritedCount), prix d'abonnement, badge, bio, avatar/bannière. | |
| 7 | +# EXPÉRIMENTAL : OnlyFans change souvent son rendu ; found=False n'est | |
| 8 | +# pas une erreur du pipeline. | |
| 9 | +# ============================================================================== | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import asyncio | |
| 13 | +import html as htmllib | |
| 14 | +import json | |
| 15 | +import re | |
| 16 | + | |
| 17 | +from apify import Actor | |
| 18 | + | |
| 19 | +from .net import Fetcher | |
| 20 | + | |
| 21 | +PROFILE_URL = "https://onlyfans.com/{u}" | |
| 22 | + | |
| 23 | +_NUM_RES = { | |
| 24 | + "posts_count": re.compile(r'"postsCount"\s*:\s*(\d+)'), | |
| 25 | + "photos_count": re.compile(r'"photosCount"\s*:\s*(\d+)'), | |
| 26 | + "videos_count": re.compile(r'"videosCount"\s*:\s*(\d+)'), | |
| 27 | + "likes": re.compile(r'"favoritedCount"\s*:\s*(\d+)'), | |
| 28 | + "streams_count": re.compile(r'"finishedStreamsCount"\s*:\s*(\d+)'), | |
| 29 | +} | |
| 30 | +_PRICE_RE = re.compile(r'"subscribePrice"\s*:\s*([\d.]+)') | |
| 31 | +_STR_RES = { | |
| 32 | + "full_name": re.compile(r'"name"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 33 | + "biography": re.compile(r'"rawAbout"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 34 | + "location": re.compile(r'"location"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 35 | + "website": re.compile(r'"website"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 36 | + "avatar": re.compile(r'"avatar"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 37 | + "banner": re.compile(r'"header"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 38 | +} | |
| 39 | +_VERIFIED_RE = re.compile(r'"isVerified"\s*:\s*(true|false)') | |
| 40 | +_OG_TITLE_RE = re.compile(r'<meta property="og:title" content="([^"]*)"') | |
| 41 | + | |
| 42 | + | |
| 43 | +def _dec(raw: str) -> str: | |
| 44 | + try: | |
| 45 | + return htmllib.unescape(json.loads(f'"{raw}"')) | |
| 46 | + except Exception: | |
| 47 | + return raw | |
| 48 | + | |
| 49 | + | |
| 50 | +async def main() -> None: | |
| 51 | + async with Actor: | |
| 52 | + inp = await Actor.get_input() or {} | |
| 53 | + usernames = [u.strip().lstrip("@").lower() | |
| 54 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 55 | + proxy = await Actor.create_proxy_configuration( | |
| 56 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 57 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 2)) | |
| 58 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 2)) | |
| 59 | + | |
| 60 | + async def one(u: str) -> None: | |
| 61 | + async with sem: | |
| 62 | + miss = {"kind": "profile", "platform": "onlyfans", | |
| 63 | + "username": u, "found": False} | |
| 64 | + resp = None | |
| 65 | + for attempt in range(3): # mur Cloudflare → nouvelle IP | |
| 66 | + try: | |
| 67 | + resp = await fetcher.get(PROFILE_URL.format(u=u), | |
| 68 | + session_id=f"{u}o{attempt}") | |
| 69 | + except Exception as exc: | |
| 70 | + await Actor.push_data({**miss, | |
| 71 | + "error": str(exc)[:200]}) | |
| 72 | + return | |
| 73 | + if resp.status_code == 404: | |
| 74 | + await Actor.push_data({**miss, "error": "http_404"}) | |
| 75 | + return | |
| 76 | + if resp.status_code == 200 and \ | |
| 77 | + "postsCount" in (resp.text or ""): | |
| 78 | + break | |
| 79 | + text = resp.text or "" | |
| 80 | + if resp.status_code != 200 or "postsCount" not in text: | |
| 81 | + await Actor.push_data( | |
| 82 | + {**miss, "error": f"wall_{resp.status_code}"}) | |
| 83 | + return | |
| 84 | + rec: dict = {"kind": "profile", "platform": "onlyfans", | |
| 85 | + "found": True, "username": u} | |
| 86 | + for key, rx in _NUM_RES.items(): | |
| 87 | + m = rx.search(text) | |
| 88 | + rec[key] = int(m.group(1)) if m else None | |
| 89 | + for key, rx in _STR_RES.items(): | |
| 90 | + m = rx.search(text) | |
| 91 | + rec[key] = _dec(m.group(1)) if m else None | |
| 92 | + if rec.get("biography"): | |
| 93 | + rec["biography"] = re.sub(r"<[^>]+>", " ", | |
| 94 | + rec["biography"])[:1000].strip() | |
| 95 | + m = _PRICE_RE.search(text) | |
| 96 | + rec["subscribe_price_usd"] = float(m.group(1)) if m else None | |
| 97 | + rec["is_free"] = (rec["subscribe_price_usd"] == 0.0 | |
| 98 | + if rec["subscribe_price_usd"] is not None | |
| 99 | + else None) | |
| 100 | + m = _VERIFIED_RE.search(text) | |
| 101 | + rec["is_verified"] = (m.group(1) == "true") if m else None | |
| 102 | + if not rec.get("full_name"): | |
| 103 | + m = _OG_TITLE_RE.search(text) | |
| 104 | + if m: | |
| 105 | + rec["full_name"] = htmllib.unescape( | |
| 106 | + m.group(1)).split(" OnlyFans")[0].strip() | |
| 107 | + # « followers » au sens crea-ka : le compteur public = J'aime | |
| 108 | + rec["followers"] = None | |
| 109 | + await Actor.push_data(rec) | |
| 110 | + | |
| 111 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 112 | + Actor.log.info(f"terminé : {len(usernames)} profils") | |
added
actors/ka-onlyfans/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-patreon/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-patreon) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-patreon/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-patreon", | |
| 4 | + "title": "KA Patreon — pages publiques", | |
| 5 | + "description": "Pages Patreon publiques : patrons, publications, nom, avatar, résumé (données embarquées de la campagne).", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-patreon/.actor/input_schema.json
+41 −0
@@ -0,0 +1,41 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-patreon input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles / codes", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles (patreon) ou codes d'invitation (discord).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [] | |
| 12 | + }, | |
| 13 | + "requestDelay": { | |
| 14 | + "title": "Délai (s)", | |
| 15 | + "type": "integer", | |
| 16 | + "default": 1, | |
| 17 | + "minimum": 0, | |
| 18 | + "description": "Pause entre requêtes." | |
| 19 | + }, | |
| 20 | + "concurrency": { | |
| 21 | + "title": "Concurrence", | |
| 22 | + "type": "integer", | |
| 23 | + "default": 5, | |
| 24 | + "minimum": 1, | |
| 25 | + "maximum": 10, | |
| 26 | + "description": "Requêtes simultanées." | |
| 27 | + }, | |
| 28 | + "proxyConfiguration": { | |
| 29 | + "title": "Proxy", | |
| 30 | + "type": "object", | |
| 31 | + "editor": "proxy", | |
| 32 | + "description": "Proxy Apify.", | |
| 33 | + "prefill": { | |
| 34 | + "useApifyProxy": true, | |
| 35 | + "apifyProxyGroups": [ | |
| 36 | + "RESIDENTIAL" | |
| 37 | + ] | |
| 38 | + } | |
| 39 | + } | |
| 40 | + } | |
| 41 | +} | |
| \ No newline at end of file | ||
added
actors/ka-patreon/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-patreon) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-patreon/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-patreon) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-patreon/src/main.py
+114 −0
@@ -0,0 +1,114 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-patreon) | |
| 4 | +# Desc: Pages Patreon publiques — la page /<vanity> embarque l'objet campagne | |
| 5 | +# (patron_count, creation_count, is_monthly, pay_per_name, résumé) + | |
| 6 | +# og:title/og:image. Aucune clé. Proxy résidentiel recommandé. | |
| 7 | +# ============================================================================== | |
| 8 | +from __future__ import annotations | |
| 9 | + | |
| 10 | +import asyncio | |
| 11 | +import html as htmllib | |
| 12 | +import json | |
| 13 | +import re | |
| 14 | + | |
| 15 | +from apify import Actor | |
| 16 | + | |
| 17 | +from .net import Fetcher | |
| 18 | + | |
| 19 | +PAGE_URL = "https://www.patreon.com/{u}" | |
| 20 | +_NUM = { | |
| 21 | + "patrons": re.compile(r'"patron_count"\s*:\s*(\d+)'), | |
| 22 | + "posts": re.compile(r'"creation_count"\s*:\s*(\d+)'), | |
| 23 | + "paid_posts": re.compile(r'"paid_member_count"\s*:\s*(\d+)'), | |
| 24 | +} | |
| 25 | +_OG_TITLE = re.compile(r'<meta property="og:title" content="([^"]*)"') | |
| 26 | +_OG_IMG = re.compile(r'<meta property="og:image" content="([^"]*)"') | |
| 27 | +_OG_DESC = re.compile(r'<meta property="og:description" content="([^"]*)"') | |
| 28 | +_MONTHLY = re.compile(r'"is_monthly"\s*:\s*(true|false)') | |
| 29 | +_NSFW = re.compile(r'"is_nsfw"\s*:\s*(true|false)') | |
| 30 | +_COVER = re.compile(r'"cover_photo_url"\s*:\s*"((?:[^"\\]|\\.)*)"') | |
| 31 | +_CREATION = re.compile(r'"creation_name"\s*:\s*"((?:[^"\\]|\\.)*)"') | |
| 32 | +# liens sociaux auto-déclarés de la campagne (objets campaign de l'app Patreon) | |
| 33 | +_SOCIALS = { | |
| 34 | + "facebook": re.compile(r'"facebook"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'), | |
| 35 | + "twitter": re.compile(r'"twitter"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'), | |
| 36 | + "youtube": re.compile(r'"youtube"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'), | |
| 37 | +} | |
| 38 | + | |
| 39 | + | |
| 40 | +def _dec(raw: str) -> str: | |
| 41 | + try: | |
| 42 | + return htmllib.unescape(json.loads(f'"{raw}"')) | |
| 43 | + except Exception: | |
| 44 | + return raw | |
| 45 | + | |
| 46 | + | |
| 47 | +async def main() -> None: | |
| 48 | + async with Actor: | |
| 49 | + inp = await Actor.get_input() or {} | |
| 50 | + usernames = [u.strip().lstrip("@").lower() | |
| 51 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 52 | + proxy = await Actor.create_proxy_configuration( | |
| 53 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 54 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 55 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 4)) | |
| 56 | + | |
| 57 | + async def one(u: str) -> None: | |
| 58 | + async with sem: | |
| 59 | + miss = {"kind": "profile", "platform": "patreon", | |
| 60 | + "username": u, "found": False} | |
| 61 | + try: | |
| 62 | + resp = await fetcher.get(PAGE_URL.format(u=u), | |
| 63 | + session_id=u) | |
| 64 | + except Exception as exc: | |
| 65 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 66 | + return | |
| 67 | + text = resp.text or "" | |
| 68 | + if resp.status_code == 404: | |
| 69 | + await Actor.push_data({**miss, "error": "http_404"}) | |
| 70 | + return | |
| 71 | + m = _NUM["patrons"].search(text) | |
| 72 | + if resp.status_code != 200 or not m: | |
| 73 | + await Actor.push_data( | |
| 74 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 75 | + return | |
| 76 | + | |
| 77 | + def num(key): | |
| 78 | + mm = _NUM[key].search(text) | |
| 79 | + return int(mm.group(1)) if mm else None | |
| 80 | + title = _OG_TITLE.search(text) | |
| 81 | + img = _OG_IMG.search(text) | |
| 82 | + desc = _OG_DESC.search(text) | |
| 83 | + monthly = _MONTHLY.search(text) | |
| 84 | + nsfw = _NSFW.search(text) | |
| 85 | + cover = _COVER.search(text) | |
| 86 | + creation = _CREATION.search(text) | |
| 87 | + socials = {k: _dec(m.group(1)) | |
| 88 | + for k, rx in _SOCIALS.items() | |
| 89 | + if (m := rx.search(text))} | |
| 90 | + name = htmllib.unescape(title.group(1)) if title else u | |
| 91 | + name = re.sub(r"\s*\|\s*Patreon\s*$", "", name).strip() | |
| 92 | + await Actor.push_data({ | |
| 93 | + "kind": "profile", | |
| 94 | + "platform": "patreon", | |
| 95 | + "found": True, | |
| 96 | + "username": u, | |
| 97 | + "full_name": name, | |
| 98 | + "biography": (htmllib.unescape(desc.group(1)) | |
| 99 | + if desc else None), | |
| 100 | + "followers": num("patrons"), # patrons = « abonnés » KA | |
| 101 | + "patrons": num("patrons"), | |
| 102 | + "posts_count": num("posts"), | |
| 103 | + "is_monthly": (monthly.group(1) == "true" | |
| 104 | + if monthly else None), | |
| 105 | + "is_nsfw": nsfw.group(1) == "true" if nsfw else None, | |
| 106 | + "creation_name": (_dec(creation.group(1))[:200] | |
| 107 | + if creation else None), | |
| 108 | + "avatar": htmllib.unescape(img.group(1)) if img else None, | |
| 109 | + "banner": _dec(cover.group(1)) if cover else None, | |
| 110 | + "social_links": socials or None, | |
| 111 | + }) | |
| 112 | + | |
| 113 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 114 | + Actor.log.info(f"terminé : {len(usernames)} pages") | |
added
actors/ka-patreon/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-snapchat/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-snapchat) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-snapchat/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-snapchat", | |
| 4 | + "title": "KA Snapchat — profils publics", | |
| 5 | + "description": "Profils Snapchat publics (abonnés, bio, badge, catégorie, avatar, présence story/spotlight, comptes reliés) via le JSON __NEXT_DATA__ de snapchat.com/add/<user>.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
added
actors/ka-snapchat/.actor/input_schema.json
+43 −0
@@ -0,0 +1,43 @@ | ||
| 1 | +{ | |
| 2 | + "title": "KA Snapchat input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles Snapchat (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "team.snapchat" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "requestDelay": { | |
| 16 | + "title": "Délai entre requêtes (s)", | |
| 17 | + "type": "integer", | |
| 18 | + "default": 1, | |
| 19 | + "minimum": 0, | |
| 20 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 21 | + }, | |
| 22 | + "concurrency": { | |
| 23 | + "title": "Requêtes simultanées", | |
| 24 | + "type": "integer", | |
| 25 | + "default": 3, | |
| 26 | + "minimum": 1, | |
| 27 | + "maximum": 10, | |
| 28 | + "description": "Nombre de requêtes menées en parallèle." | |
| 29 | + }, | |
| 30 | + "proxyConfiguration": { | |
| 31 | + "title": "Proxy", | |
| 32 | + "type": "object", | |
| 33 | + "editor": "proxy", | |
| 34 | + "prefill": { | |
| 35 | + "useApifyProxy": true, | |
| 36 | + "apifyProxyGroups": [ | |
| 37 | + "RESIDENTIAL" | |
| 38 | + ] | |
| 39 | + }, | |
| 40 | + "description": "Configuration du proxy Apify." | |
| 41 | + } | |
| 42 | + } | |
| 43 | +} | |
| \ No newline at end of file | ||
added
actors/ka-snapchat/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-snapchat) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-snapchat/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-snapchat) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-snapchat/src/main.py
+121 −0
@@ -0,0 +1,121 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-snapchat) | |
| 4 | +# Desc: Profils Snapchat publics via le JSON __NEXT_DATA__ de la page | |
| 5 | +# snapchat.com/add/<user> : publicProfileInfo (abonnés, bio, badge, | |
| 6 | +# catégorie, avatar) + présence story/spotlight + comptes reliés. | |
| 7 | +# Structure validée le 2026-08-18 (pageProps.userProfile.$case). | |
| 8 | +# ============================================================================== | |
| 9 | +from __future__ import annotations | |
| 10 | + | |
| 11 | +import asyncio | |
| 12 | +import json | |
| 13 | +import re | |
| 14 | + | |
| 15 | +from apify import Actor | |
| 16 | + | |
| 17 | +from .net import Fetcher | |
| 18 | + | |
| 19 | +PROFILE_URL = "https://www.snapchat.com/add/{u}" | |
| 20 | +_NEXT_DATA_RE = re.compile( | |
| 21 | + r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', re.S) | |
| 22 | + | |
| 23 | + | |
| 24 | +def _num(v): | |
| 25 | + try: | |
| 26 | + return int(v) | |
| 27 | + except (TypeError, ValueError): | |
| 28 | + return None | |
| 29 | + | |
| 30 | + | |
| 31 | +async def main() -> None: | |
| 32 | + async with Actor: | |
| 33 | + inp = await Actor.get_input() or {} | |
| 34 | + usernames = [u.strip().lstrip("@").lower() | |
| 35 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 36 | + proxy = await Actor.create_proxy_configuration( | |
| 37 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 38 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 39 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 3)) | |
| 40 | + | |
| 41 | + async def one(u: str) -> None: | |
| 42 | + async with sem: | |
| 43 | + miss = {"kind": "profile", "platform": "snapchat", | |
| 44 | + "username": u, "found": False} | |
| 45 | + try: | |
| 46 | + resp = await fetcher.get(PROFILE_URL.format(u=u), | |
| 47 | + session_id=u) | |
| 48 | + except Exception as exc: | |
| 49 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 50 | + return | |
| 51 | + m = _NEXT_DATA_RE.search(resp.text or "") | |
| 52 | + if resp.status_code != 200 or not m: | |
| 53 | + await Actor.push_data( | |
| 54 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 55 | + return | |
| 56 | + try: | |
| 57 | + props = (json.loads(m.group(1)).get("props") or {}) \ | |
| 58 | + .get("pageProps") or {} | |
| 59 | + except Exception: | |
| 60 | + await Actor.push_data({**miss, "error": "bad_json"}) | |
| 61 | + return | |
| 62 | + up = props.get("userProfile") or {} | |
| 63 | + info = up.get("publicProfileInfo") or {} | |
| 64 | + story = (props.get("story") or {}) | |
| 65 | + snaps = story.get("snapList") or [] | |
| 66 | + lenses = props.get("lenses") or [] | |
| 67 | + spotlight = props.get("spotlightHighlights") or [] | |
| 68 | + if not info.get("username"): | |
| 69 | + await Actor.push_data({**miss, "error": "no_profile"}) | |
| 70 | + return | |
| 71 | + related = [ | |
| 72 | + {"username": r.get("username"), "title": r.get("title")} | |
| 73 | + for r in ((info.get("relatedAccountsInfo") or []) | |
| 74 | + if isinstance(info.get("relatedAccountsInfo"), | |
| 75 | + list) else []) | |
| 76 | + if isinstance(r, dict) and r.get("username")] | |
| 77 | + story_previews = [] | |
| 78 | + for s in snaps[:6]: | |
| 79 | + if not isinstance(s, dict): | |
| 80 | + continue | |
| 81 | + urls = s.get("snapUrls") or {} | |
| 82 | + prev = (urls.get("mediaPreviewUrl") or {}) \ | |
| 83 | + if isinstance(urls.get("mediaPreviewUrl"), dict) \ | |
| 84 | + else {"value": urls.get("mediaPreviewUrl")} | |
| 85 | + url = prev.get("value") or urls.get("mediaUrl") | |
| 86 | + if url: | |
| 87 | + story_previews.append(url) | |
| 88 | + spotlight_previews = [ | |
| 89 | + h.get("thumbnailUrl") | |
| 90 | + for h in spotlight[:6] | |
| 91 | + if isinstance(h, dict) and h.get("thumbnailUrl")] | |
| 92 | + subs = _num(info.get("subscriberCount")) | |
| 93 | + await Actor.push_data({ | |
| 94 | + "kind": "profile", | |
| 95 | + "platform": "snapchat", | |
| 96 | + "found": True, | |
| 97 | + "username": info.get("username"), | |
| 98 | + "full_name": info.get("title"), | |
| 99 | + "biography": info.get("bio"), | |
| 100 | + "followers": subs if subs else None, # 0 = masqué | |
| 101 | + "is_verified": str(info.get("badge")) == "1", | |
| 102 | + "category": info.get("categoryStringId"), | |
| 103 | + "website": info.get("websiteUrl"), | |
| 104 | + "address": info.get("address"), | |
| 105 | + "avatar": info.get("profilePictureUrl"), | |
| 106 | + "has_story": bool(info.get("hasStory")), | |
| 107 | + "has_spotlight": bool(info.get("hasSpotlightHighlights")), | |
| 108 | + "has_curated_highlights": bool( | |
| 109 | + info.get("hasCuratedHighlights")), | |
| 110 | + "story_snaps_count": len(snaps) or None, | |
| 111 | + "story_previews": story_previews or None, | |
| 112 | + "lenses_count": len(lenses) or None, | |
| 113 | + "spotlight_highlights_count": len(spotlight) or None, | |
| 114 | + "spotlight_previews": spotlight_previews or None, | |
| 115 | + "publisher_type": info.get("publisherType") or None, | |
| 116 | + "subcategory": info.get("subcategoryStringId") or None, | |
| 117 | + "related_accounts": related, | |
| 118 | + }) | |
| 119 | + | |
| 120 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 121 | + Actor.log.info(f"terminé : {len(usernames)} profils") | |
added
actors/ka-snapchat/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-threads/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-threads) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-threads/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-threads", | |
| 4 | + "title": "KA Threads — profils", | |
| 5 | + "description": "Profils Meta Threads publics (abonnés, bio, badge, avatar + posts récents si embarqués) extraits du JSON de la page publique via proxy résidentiel.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
added
actors/ka-threads/.actor/input_schema.json
+43 −0
@@ -0,0 +1,43 @@ | ||
| 1 | +{ | |
| 2 | + "title": "KA Threads input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles Threads (sans @, généralement = handle Instagram).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "zuck" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "requestDelay": { | |
| 16 | + "title": "Délai entre requêtes (s)", | |
| 17 | + "type": "integer", | |
| 18 | + "default": 2, | |
| 19 | + "minimum": 0, | |
| 20 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 21 | + }, | |
| 22 | + "concurrency": { | |
| 23 | + "title": "Requêtes simultanées", | |
| 24 | + "type": "integer", | |
| 25 | + "default": 2, | |
| 26 | + "minimum": 1, | |
| 27 | + "maximum": 6, | |
| 28 | + "description": "Nombre de requêtes menées en parallèle." | |
| 29 | + }, | |
| 30 | + "proxyConfiguration": { | |
| 31 | + "title": "Proxy", | |
| 32 | + "type": "object", | |
| 33 | + "editor": "proxy", | |
| 34 | + "description": "Résidentiel recommandé (Meta sert une coquille aux IP datacenter).", | |
| 35 | + "prefill": { | |
| 36 | + "useApifyProxy": true, | |
| 37 | + "apifyProxyGroups": [ | |
| 38 | + "RESIDENTIAL" | |
| 39 | + ] | |
| 40 | + } | |
| 41 | + } | |
| 42 | + } | |
| 43 | +} | |
| \ No newline at end of file | ||
added
actors/ka-threads/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-threads) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-threads/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-threads) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-threads/src/main.py
+99 −0
@@ -0,0 +1,99 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-threads) | |
| 4 | +# Desc: Profils Meta Threads publics : la page @handle embarque le profil | |
| 5 | +# dans des scripts JSON Relay (follower_count, biography, badge, | |
| 6 | +# avatar) + les posts récents (caption/like_count) quand servis. | |
| 7 | +# Extraction par regex ciblées — robuste aux réorganisations du JSON. | |
| 8 | +# ============================================================================== | |
| 9 | +from __future__ import annotations | |
| 10 | + | |
| 11 | +import asyncio | |
| 12 | +import html as htmllib | |
| 13 | +import json | |
| 14 | +import re | |
| 15 | + | |
| 16 | +from apify import Actor | |
| 17 | + | |
| 18 | +from .net import Fetcher | |
| 19 | + | |
| 20 | +PROFILE_URL = "https://www.threads.com/@{u}" | |
| 21 | + | |
| 22 | +_FOLLOWERS_RE = re.compile(r'"follower_count"\s*:\s*(\d+)') | |
| 23 | +_BIO_RE = re.compile(r'"biography"\s*:\s*"((?:[^"\\]|\\.)*)"') | |
| 24 | +_NAME_RE = re.compile(r'"full_name"\s*:\s*"((?:[^"\\]|\\.)*)"') | |
| 25 | +_VERIFIED_RE = re.compile(r'"is_verified"\s*:\s*(true|false)') | |
| 26 | +_PIC_RE = re.compile(r'"profile_pic_url"\s*:\s*"((?:[^"\\]|\\.)*)"') | |
| 27 | +# posts : paires texte + like_count dans les payloads thread_items | |
| 28 | +_POST_RE = re.compile( | |
| 29 | + r'"caption"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"[^{}]*?\}' | |
| 30 | + r'.{0,600}?"like_count"\s*:\s*(\d+)', re.S) | |
| 31 | + | |
| 32 | + | |
| 33 | +async def main() -> None: | |
| 34 | + async with Actor: | |
| 35 | + inp = await Actor.get_input() or {} | |
| 36 | + usernames = [u.strip().lstrip("@").lower() | |
| 37 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 38 | + proxy = await Actor.create_proxy_configuration( | |
| 39 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 40 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 2)) | |
| 41 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 2)) | |
| 42 | + | |
| 43 | + async def one(u: str) -> None: | |
| 44 | + async with sem: | |
| 45 | + miss = {"kind": "profile", "platform": "threads", | |
| 46 | + "username": u, "found": False} | |
| 47 | + try: | |
| 48 | + resp = await fetcher.get(PROFILE_URL.format(u=u), | |
| 49 | + session_id=u) | |
| 50 | + except Exception as exc: | |
| 51 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 52 | + return | |
| 53 | + text = resp.text or "" | |
| 54 | + fol = _FOLLOWERS_RE.search(text) | |
| 55 | + if resp.status_code != 200 or not fol: | |
| 56 | + await Actor.push_data( | |
| 57 | + {**miss, | |
| 58 | + "error": f"shell_or_{resp.status_code}"}) | |
| 59 | + return | |
| 60 | + posts = [] | |
| 61 | + for m in _POST_RE.finditer(text): | |
| 62 | + cap = _dec(m.group(1))[:500] | |
| 63 | + if cap and all(p["caption"] != cap for p in posts): | |
| 64 | + posts.append({"caption": cap, | |
| 65 | + "likes": int(m.group(2))}) | |
| 66 | + if len(posts) >= 15: | |
| 67 | + break | |
| 68 | + bio = _BIO_RE.search(text) | |
| 69 | + name = _NAME_RE.search(text) | |
| 70 | + ver = _VERIFIED_RE.search(text) | |
| 71 | + pic = _PIC_RE.search(text) | |
| 72 | + likes = [p["likes"] for p in posts] | |
| 73 | + top = max(posts, key=lambda p: p["likes"], default=None) | |
| 74 | + await Actor.push_data({ | |
| 75 | + "kind": "profile", | |
| 76 | + "platform": "threads", | |
| 77 | + "found": True, | |
| 78 | + "username": u, | |
| 79 | + "full_name": _dec(name.group(1)) if name else None, | |
| 80 | + "biography": _dec(bio.group(1)) if bio else None, | |
| 81 | + "followers": int(fol.group(1)), | |
| 82 | + "is_verified": (ver.group(1) == "true") if ver else None, | |
| 83 | + "avatar": _dec(pic.group(1)) if pic else None, | |
| 84 | + "avg_likes": round(sum(likes) / len(likes)) | |
| 85 | + if likes else None, | |
| 86 | + "top_post": top, | |
| 87 | + "recent_posts": posts, | |
| 88 | + }) | |
| 89 | + | |
| 90 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 91 | + Actor.log.info(f"terminé : {len(usernames)} profils") | |
| 92 | + | |
| 93 | + | |
| 94 | +def _dec(raw: str) -> str: | |
| 95 | + """Décode une chaîne échappée JSON (\\uXXXX, \\/ …).""" | |
| 96 | + try: | |
| 97 | + return htmllib.unescape(json.loads(f'"{raw}"')) | |
| 98 | + except Exception: | |
| 99 | + return raw | |
added
actors/ka-threads/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-tiktok/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-tiktok) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-tiktok/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-tiktok", | |
| 4 | + "title": "KA TikTok — profils détaillés", | |
| 5 | + "description": "Profils TikTok publics détaillés (abonnés, cœurs, vidéos, bio, lien de bio, badge, vidéos récentes si présentes dans le SSR) via la page publique + proxy résidentiel.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
added
actors/ka-tiktok/.actor/input_schema.json
+43 −0
@@ -0,0 +1,43 @@ | ||
| 1 | +{ | |
| 2 | + "title": "KA TikTok input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles TikTok (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "tiktok" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "requestDelay": { | |
| 16 | + "title": "Délai entre requêtes (s)", | |
| 17 | + "type": "integer", | |
| 18 | + "default": 1, | |
| 19 | + "minimum": 0, | |
| 20 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 21 | + }, | |
| 22 | + "concurrency": { | |
| 23 | + "title": "Requêtes simultanées", | |
| 24 | + "type": "integer", | |
| 25 | + "default": 3, | |
| 26 | + "minimum": 1, | |
| 27 | + "maximum": 10, | |
| 28 | + "description": "Nombre de requêtes menées en parallèle." | |
| 29 | + }, | |
| 30 | + "proxyConfiguration": { | |
| 31 | + "title": "Proxy", | |
| 32 | + "type": "object", | |
| 33 | + "editor": "proxy", | |
| 34 | + "description": "Résidentiel recommandé (TikTok sert une coquille vide aux IP datacenter).", | |
| 35 | + "prefill": { | |
| 36 | + "useApifyProxy": true, | |
| 37 | + "apifyProxyGroups": [ | |
| 38 | + "RESIDENTIAL" | |
| 39 | + ] | |
| 40 | + } | |
| 41 | + } | |
| 42 | + } | |
| 43 | +} | |
| \ No newline at end of file | ||
added
actors/ka-tiktok/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-tiktok) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-tiktok/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-tiktok) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-tiktok/src/main.py
+228 −0
@@ -0,0 +1,228 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-tiktok) | |
| 4 | +# Desc: Profils TikTok publics ULTRA DÉTAILLÉS via la page @handle : le JSON | |
| 5 | +# SSR __UNIVERSAL_DATA_FOR_REHYDRATION__ contient user + stats complètes | |
| 6 | +# (abonnés, cœurs, vidéos) ; vidéos récentes avec COVERS (miniatures), | |
| 7 | +# durée, hashtags, musique, épinglés + cadence (videos_per_week) et | |
| 8 | +# top hashtags quand TikTok les inclut dans le SSR (ItemModule/itemList). | |
| 9 | +# ============================================================================== | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import asyncio | |
| 13 | +import json | |
| 14 | +import re | |
| 15 | + | |
| 16 | +from apify import Actor | |
| 17 | + | |
| 18 | +from .net import Fetcher | |
| 19 | + | |
| 20 | +PROFILE_URL = "https://www.tiktok.com/@{u}" | |
| 21 | +_STATE_RE = re.compile( | |
| 22 | + r'<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)</script>', | |
| 23 | + re.S) | |
| 24 | +_ITEM_RE = re.compile(r'"ItemModule"\s*:\s*(\{.*?\})\s*,\s*"[A-Z]', re.S) | |
| 25 | + | |
| 26 | + | |
| 27 | +def parse_videos(html: str, data: dict) -> list[dict]: | |
| 28 | + """Vidéos récentes si le SSR les inclut (souvent absent — best effort).""" | |
| 29 | + items: list[dict] = [] | |
| 30 | + raw: dict = {} | |
| 31 | + scope = data.get("__DEFAULT_SCOPE__") or {} | |
| 32 | + detail = scope.get("webapp.user-detail") or {} | |
| 33 | + for it in (detail.get("itemList") or []): | |
| 34 | + raw[it.get("id", str(len(raw)))] = it | |
| 35 | + if not raw: | |
| 36 | + m = _ITEM_RE.search(html) | |
| 37 | + if m: | |
| 38 | + try: | |
| 39 | + raw = json.loads(m.group(1)) | |
| 40 | + except Exception: | |
| 41 | + raw = {} | |
| 42 | + for vid in list(raw.values())[:20]: | |
| 43 | + stats = vid.get("stats") or vid.get("statsV2") or {} | |
| 44 | + author = vid.get("author") | |
| 45 | + handle = author if isinstance(author, str) else \ | |
| 46 | + (author or {}).get("uniqueId", "") | |
| 47 | + video = vid.get("video") or {} | |
| 48 | + music = vid.get("music") or {} | |
| 49 | + items.append({ | |
| 50 | + "id": vid.get("id"), | |
| 51 | + "url": f"https://www.tiktok.com/@{handle}/video/{vid.get('id')}", | |
| 52 | + "caption": (vid.get("desc") or "")[:500], | |
| 53 | + "views": _num(stats.get("playCount")), | |
| 54 | + "likes": _num(stats.get("diggCount")), | |
| 55 | + "comments": _num(stats.get("commentCount")), | |
| 56 | + "shares": _num(stats.get("shareCount")), | |
| 57 | + "saves": _num(stats.get("collectCount")), | |
| 58 | + "timestamp": vid.get("createTime"), | |
| 59 | + "cover": (video.get("cover") or video.get("dynamicCover") | |
| 60 | + or video.get("originCover")), | |
| 61 | + "duration_s": _num(video.get("duration")), | |
| 62 | + "width": _num(video.get("width")), | |
| 63 | + "height": _num(video.get("height")), | |
| 64 | + "is_pinned": bool(vid.get("isPinnedItem")), | |
| 65 | + "hashtags": [c.get("hashtagName") | |
| 66 | + for c in (vid.get("textExtra") or []) | |
| 67 | + if c.get("hashtagName")][:8], | |
| 68 | + "music": ({"title": music.get("title"), | |
| 69 | + "author": music.get("authorName"), | |
| 70 | + "original": music.get("original")} | |
| 71 | + if music.get("title") else None), | |
| 72 | + }) | |
| 73 | + return items | |
| 74 | + | |
| 75 | + | |
| 76 | +def _num(v): | |
| 77 | + try: | |
| 78 | + return int(v) | |
| 79 | + except (TypeError, ValueError): | |
| 80 | + return None | |
| 81 | + | |
| 82 | + | |
| 83 | +async def main() -> None: | |
| 84 | + async with Actor: | |
| 85 | + inp = await Actor.get_input() or {} | |
| 86 | + usernames = [u.strip().lstrip("@").lower() | |
| 87 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 88 | + proxy = await Actor.create_proxy_configuration( | |
| 89 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 90 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 91 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 3)) | |
| 92 | + | |
| 93 | + async def one(u: str) -> None: | |
| 94 | + async with sem: | |
| 95 | + miss = {"kind": "profile", "platform": "tiktok", | |
| 96 | + "username": u, "found": False} | |
| 97 | + resp = None | |
| 98 | + m = None | |
| 99 | + for attempt in range(3): # coquille sans SSR → nouvelle IP | |
| 100 | + try: | |
| 101 | + resp = await fetcher.get(PROFILE_URL.format(u=u), | |
| 102 | + session_id=f"{u}t{attempt}") | |
| 103 | + except Exception as exc: | |
| 104 | + await Actor.push_data({**miss, | |
| 105 | + "error": str(exc)[:200]}) | |
| 106 | + return | |
| 107 | + if resp.status_code == 404: | |
| 108 | + await Actor.push_data({**miss, "error": "http_404"}) | |
| 109 | + return | |
| 110 | + m = _STATE_RE.search(resp.text or "") | |
| 111 | + if m and "webapp.user-detail" in (resp.text or ""): | |
| 112 | + break | |
| 113 | + info = None | |
| 114 | + data: dict = {} | |
| 115 | + if m: | |
| 116 | + try: | |
| 117 | + data = json.loads(m.group(1)) | |
| 118 | + info = ((data.get("__DEFAULT_SCOPE__") or {}) | |
| 119 | + .get("webapp.user-detail") or {}).get("userInfo") | |
| 120 | + except Exception: | |
| 121 | + info = None | |
| 122 | + if not info or not (info.get("user") or {}).get("uniqueId"): | |
| 123 | + await Actor.push_data( | |
| 124 | + {**miss, "error": f"shell_page_{resp.status_code}"}) | |
| 125 | + return | |
| 126 | + user = info.get("user") or {} | |
| 127 | + stats = info.get("statsV2") or info.get("stats") or {} | |
| 128 | + if user.get("privateAccount"): | |
| 129 | + await Actor.push_data({**miss, "error": "private"}) | |
| 130 | + return | |
| 131 | + followers = _num(stats.get("followerCount")) | |
| 132 | + videos = parse_videos(resp.text, data) | |
| 133 | + views = [v["views"] for v in videos | |
| 134 | + if isinstance(v["views"], int)] | |
| 135 | + likes = [v["likes"] for v in videos | |
| 136 | + if isinstance(v["likes"], int)] | |
| 137 | + comments = [v["comments"] for v in videos | |
| 138 | + if isinstance(v["comments"], int)] | |
| 139 | + shares = [v["shares"] for v in videos | |
| 140 | + if isinstance(v["shares"], int)] | |
| 141 | + avg_views = round(sum(views) / len(views)) if views else None | |
| 142 | + avg_likes = round(sum(likes) / len(likes)) if likes else None | |
| 143 | + engagement = None | |
| 144 | + if followers and avg_likes is not None: | |
| 145 | + engagement = round( | |
| 146 | + (avg_likes | |
| 147 | + + (round(sum(comments) / len(comments)) | |
| 148 | + if comments else 0) | |
| 149 | + + (round(sum(shares) / len(shares)) | |
| 150 | + if shares else 0)) / followers * 100, 3) | |
| 151 | + top_video = max( | |
| 152 | + (v for v in videos if isinstance(v["views"], int)), | |
| 153 | + key=lambda v: v["views"], default=None) | |
| 154 | + stamps = sorted(_num(v["timestamp"]) for v in videos | |
| 155 | + if _num(v["timestamp"])) | |
| 156 | + videos_per_week = None | |
| 157 | + if len(stamps) >= 3 and stamps[-1] > stamps[0]: | |
| 158 | + videos_per_week = round( | |
| 159 | + (len(stamps) - 1) | |
| 160 | + / ((stamps[-1] - stamps[0]) / 604_800), 2) | |
| 161 | + last_video_at = None | |
| 162 | + if stamps: | |
| 163 | + from datetime import datetime, timezone | |
| 164 | + last_video_at = datetime.fromtimestamp( | |
| 165 | + stamps[-1], tz=timezone.utc) \ | |
| 166 | + .strftime("%Y-%m-%dT%H:%M:%SZ") | |
| 167 | + tag_counts: dict[str, int] = {} | |
| 168 | + for v in videos: | |
| 169 | + for h in v.get("hashtags") or []: | |
| 170 | + tag_counts[h.lower()] = tag_counts.get(h.lower(), 0) + 1 | |
| 171 | + commerce = user.get("commerceUserInfo") or {} | |
| 172 | + bio_link = ((user.get("bioLink") or {}).get("link") | |
| 173 | + or "").strip() | |
| 174 | + total_likes = _num(stats.get("heartCount") | |
| 175 | + or stats.get("heart")) | |
| 176 | + videos_count = _num(stats.get("videoCount")) | |
| 177 | + await Actor.push_data({ | |
| 178 | + "kind": "profile", | |
| 179 | + "platform": "tiktok", | |
| 180 | + "found": True, | |
| 181 | + "id": user.get("id"), | |
| 182 | + "sec_uid": user.get("secUid"), | |
| 183 | + "username": user.get("uniqueId"), | |
| 184 | + "full_name": user.get("nickname"), | |
| 185 | + "biography": user.get("signature"), | |
| 186 | + "bio_link": bio_link, | |
| 187 | + "followers": followers, | |
| 188 | + "following": _num(stats.get("followingCount")), | |
| 189 | + "friends": _num(stats.get("friendCount")), | |
| 190 | + "total_likes": total_likes, | |
| 191 | + "videos_count": videos_count, | |
| 192 | + "avg_likes_per_video_lifetime": ( | |
| 193 | + round(total_likes / videos_count) | |
| 194 | + if total_likes and videos_count else None), | |
| 195 | + "is_verified": bool(user.get("verified")), | |
| 196 | + "is_organization": bool(user.get("isOrganization")), | |
| 197 | + "is_seller": bool(user.get("ttSeller")), | |
| 198 | + "is_live_now": bool(user.get("roomId")), | |
| 199 | + "commerce_category": (commerce.get("category") or None | |
| 200 | + if commerce.get("commerceUser") | |
| 201 | + else None), | |
| 202 | + "account_created_at": user.get("createTime"), | |
| 203 | + "region": user.get("region"), | |
| 204 | + "language": user.get("language"), | |
| 205 | + "avatar": user.get("avatarLarger") | |
| 206 | + or user.get("avatarMedium"), | |
| 207 | + # agrégats sur les vidéos présentes dans le SSR | |
| 208 | + "avg_views": avg_views, | |
| 209 | + "avg_likes": avg_likes, | |
| 210 | + "avg_comments": (round(sum(comments) / len(comments)) | |
| 211 | + if comments else None), | |
| 212 | + "avg_shares": (round(sum(shares) / len(shares)) | |
| 213 | + if shares else None), | |
| 214 | + "engagement_rate_pct": engagement, | |
| 215 | + "videos_per_week": videos_per_week, | |
| 216 | + "last_video_at": last_video_at, | |
| 217 | + "top_hashtags": sorted(tag_counts, key=tag_counts.get, | |
| 218 | + reverse=True)[:8], | |
| 219 | + "top_video": ({"url": top_video["url"], | |
| 220 | + "views": top_video["views"], | |
| 221 | + "likes": top_video["likes"], | |
| 222 | + "cover": top_video.get("cover")} | |
| 223 | + if top_video else None), | |
| 224 | + "recent_videos": videos, | |
| 225 | + }) | |
| 226 | + | |
| 227 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 228 | + Actor.log.info(f"terminé : {len(usernames)} profils") | |
added
actors/ka-tiktok/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-twitch/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-twitch) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-twitch/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-twitch", | |
| 4 | + "title": "KA Twitch — profils détaillés", | |
| 5 | + "description": "Profils Twitch détaillés via le GQL public (abonnés, partenaire/affilié, live en cours, dernière diffusion, jeux et vidéos récentes).", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-twitch/.actor/input_schema.json
+40 −0
@@ -0,0 +1,40 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-twitch input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles/slugs à traiter (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "xqc" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "requestDelay": { | |
| 16 | + "title": "Délai entre requêtes (s)", | |
| 17 | + "type": "integer", | |
| 18 | + "default": 1, | |
| 19 | + "minimum": 0, | |
| 20 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 21 | + }, | |
| 22 | + "concurrency": { | |
| 23 | + "title": "Requêtes simultanées", | |
| 24 | + "type": "integer", | |
| 25 | + "default": 3, | |
| 26 | + "minimum": 1, | |
| 27 | + "maximum": 10, | |
| 28 | + "description": "Nombre de requêtes menées en parallèle." | |
| 29 | + }, | |
| 30 | + "proxyConfiguration": { | |
| 31 | + "title": "Proxy", | |
| 32 | + "type": "object", | |
| 33 | + "editor": "proxy", | |
| 34 | + "description": "Configuration du proxy Apify.", | |
| 35 | + "prefill": { | |
| 36 | + "useApifyProxy": true | |
| 37 | + } | |
| 38 | + } | |
| 39 | + } | |
| 40 | +} | |
| \ No newline at end of file | ||
added
actors/ka-twitch/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-twitch) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-twitch/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-twitch) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-twitch/src/main.py
+158 −0
@@ -0,0 +1,158 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-twitch) | |
| 4 | +# Desc: Profils Twitch ULTRA DÉTAILLÉS via le GQL PUBLIC du web (Client-ID | |
| 5 | +# public kimne78kx3ncx6brgo4mv6wki5h1ko, celui du site twitch.tv) : | |
| 6 | +# abonnés, partenaire/affilié, live en cours (titre + MINIATURE), | |
| 7 | +# dernière diffusion, jeux récents, vidéos récentes avec MINIATURES et | |
| 8 | +# vues + LIENS SOCIAUX auto-déclarés du panneau « À propos ». | |
| 9 | +# Pas de clé requise. | |
| 10 | +# ============================================================================== | |
| 11 | +from __future__ import annotations | |
| 12 | + | |
| 13 | +import asyncio | |
| 14 | +import json | |
| 15 | + | |
| 16 | +from apify import Actor | |
| 17 | + | |
| 18 | +from .net import Fetcher, UA | |
| 19 | + | |
| 20 | +GQL_URL = "https://gql.twitch.tv/gql" | |
| 21 | +CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko" | |
| 22 | + | |
| 23 | +QUERY = """ | |
| 24 | +query($login: String!) { | |
| 25 | + user(login: $login) { | |
| 26 | + id | |
| 27 | + login | |
| 28 | + displayName | |
| 29 | + description | |
| 30 | + createdAt | |
| 31 | + profileImageURL(width: 300) | |
| 32 | + bannerImageURL | |
| 33 | + followers { totalCount } | |
| 34 | + roles { isPartner isAffiliate } | |
| 35 | + primaryTeam { displayName } | |
| 36 | + channel { socialMedias { name title url } } | |
| 37 | + lastBroadcast { startedAt title game { displayName } } | |
| 38 | + stream { viewersCount createdAt title | |
| 39 | + previewImageURL(width: 640, height: 360) | |
| 40 | + game { displayName } } | |
| 41 | + videos(first: 10, sort: TIME) { | |
| 42 | + edges { node { | |
| 43 | + id title viewCount lengthSeconds publishedAt | |
| 44 | + previewThumbnailURL(width: 320, height: 180) | |
| 45 | + game { displayName } | |
| 46 | + } } | |
| 47 | + } | |
| 48 | + } | |
| 49 | +} | |
| 50 | +""" | |
| 51 | + | |
| 52 | + | |
| 53 | +async def main() -> None: | |
| 54 | + async with Actor: | |
| 55 | + inp = await Actor.get_input() or {} | |
| 56 | + usernames = [u.strip().lstrip("@").lower() | |
| 57 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 58 | + proxy = await Actor.create_proxy_configuration( | |
| 59 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 60 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 61 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 4)) | |
| 62 | + | |
| 63 | + import random | |
| 64 | + from curl_cffi import requests as cffi | |
| 65 | + | |
| 66 | + async def gql(login: str) -> dict | None: | |
| 67 | + body = json.dumps({"query": QUERY, | |
| 68 | + "variables": {"login": login}}) | |
| 69 | + p = None | |
| 70 | + if fetcher.proxy_configuration: | |
| 71 | + p = await fetcher.proxy_configuration.new_url( | |
| 72 | + session_id=f"tw{random.randint(1, 999_999)}") | |
| 73 | + await fetcher._throttle() | |
| 74 | + resp = await asyncio.to_thread( | |
| 75 | + cffi.post, GQL_URL, data=body, | |
| 76 | + headers={"Client-ID": CLIENT_ID, "User-Agent": UA, | |
| 77 | + "Content-Type": "application/json"}, | |
| 78 | + impersonate="chrome", timeout=45, | |
| 79 | + proxies={"http": p, "https": p} if p else None) | |
| 80 | + if resp.status_code != 200: | |
| 81 | + raise RuntimeError(f"gql http_{resp.status_code}") | |
| 82 | + return (resp.json().get("data") or {}).get("user") | |
| 83 | + | |
| 84 | + async def one(u: str) -> None: | |
| 85 | + async with sem: | |
| 86 | + miss = {"kind": "profile", "platform": "twitch", | |
| 87 | + "username": u, "found": False} | |
| 88 | + try: | |
| 89 | + user = await gql(u) | |
| 90 | + except Exception as exc: | |
| 91 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 92 | + return | |
| 93 | + if not user: | |
| 94 | + await Actor.push_data({**miss, "error": "not_found"}) | |
| 95 | + return | |
| 96 | + vids = [] | |
| 97 | + for e in ((user.get("videos") or {}).get("edges") or []): | |
| 98 | + n = e.get("node") or {} | |
| 99 | + vids.append({ | |
| 100 | + "id": n.get("id"), | |
| 101 | + "url": f"https://www.twitch.tv/videos/{n.get('id')}", | |
| 102 | + "title": (n.get("title") or "")[:200], | |
| 103 | + "views": n.get("viewCount"), | |
| 104 | + "duration_s": n.get("lengthSeconds"), | |
| 105 | + "published_at": n.get("publishedAt"), | |
| 106 | + "thumbnail": n.get("previewThumbnailURL"), | |
| 107 | + "game": (n.get("game") or {}).get("displayName"), | |
| 108 | + }) | |
| 109 | + views = [v["views"] for v in vids | |
| 110 | + if isinstance(v["views"], int)] | |
| 111 | + games = [v["game"] for v in vids if v["game"]] | |
| 112 | + stream = user.get("stream") or {} | |
| 113 | + last = user.get("lastBroadcast") or {} | |
| 114 | + roles = user.get("roles") or {} | |
| 115 | + await Actor.push_data({ | |
| 116 | + "kind": "profile", | |
| 117 | + "platform": "twitch", | |
| 118 | + "found": True, | |
| 119 | + "id": user.get("id"), | |
| 120 | + "username": user.get("login"), | |
| 121 | + "full_name": user.get("displayName"), | |
| 122 | + "biography": user.get("description"), | |
| 123 | + "followers": ((user.get("followers") or {}) | |
| 124 | + .get("totalCount")), | |
| 125 | + "is_partner": roles.get("isPartner"), | |
| 126 | + "is_affiliate": roles.get("isAffiliate"), | |
| 127 | + "is_verified": bool(roles.get("isPartner")), | |
| 128 | + "team": ((user.get("primaryTeam") or {}) | |
| 129 | + .get("displayName")), | |
| 130 | + "created_at": user.get("createdAt"), | |
| 131 | + "avatar": user.get("profileImageURL"), | |
| 132 | + "banner": user.get("bannerImageURL"), | |
| 133 | + "is_live_now": bool(stream), | |
| 134 | + "live_viewers": stream.get("viewersCount"), | |
| 135 | + "live_title": (stream.get("title") or "")[:200] or None, | |
| 136 | + "live_thumbnail": stream.get("previewImageURL"), | |
| 137 | + "live_game": (stream.get("game") or {}).get("displayName"), | |
| 138 | + "last_broadcast_at": last.get("startedAt"), | |
| 139 | + "last_broadcast_title": (last.get("title") or "")[:200] | |
| 140 | + or None, | |
| 141 | + "last_broadcast_game": ((last.get("game") or {}) | |
| 142 | + .get("displayName")), | |
| 143 | + "avg_video_views": (round(sum(views) / len(views)) | |
| 144 | + if views else None), | |
| 145 | + "recent_games": list(dict.fromkeys(games))[:5], | |
| 146 | + "recent_videos": vids, | |
| 147 | + # liens sociaux AUTO-DÉCLARÉS (panneau « À propos ») → | |
| 148 | + # cross_link fort côté crea-ka (§12.1) | |
| 149 | + "social_links": [ | |
| 150 | + {"name": s.get("name"), "title": s.get("title"), | |
| 151 | + "url": s.get("url")} | |
| 152 | + for s in (((user.get("channel") or {}) | |
| 153 | + .get("socialMedias")) or []) | |
| 154 | + if s.get("url")][:10], | |
| 155 | + }) | |
| 156 | + | |
| 157 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 158 | + Actor.log.info(f"terminé : {len(usernames)} profils") | |
added
actors/ka-twitch/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-x/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-x) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-x/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-x", | |
| 4 | + "title": "KA X (Twitter) — profils + tweets récents", | |
| 5 | + "description": "Profils X (Twitter) détaillés SANS clé API via le service officiel de syndication des widgets : abonnés, bio, badge, avatar + tweets récents avec métriques (favoris, RT, réponses) et engagement moyen.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
added
actors/ka-x/.actor/input_schema.json
+48 −0
@@ -0,0 +1,48 @@ | ||
| 1 | +{ | |
| 2 | + "title": "KA X input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles X/Twitter (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "x" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "maxTweets": { | |
| 16 | + "title": "Tweets récents conservés par profil", | |
| 17 | + "type": "integer", | |
| 18 | + "default": 20, | |
| 19 | + "minimum": 0, | |
| 20 | + "maximum": 100, | |
| 21 | + "description": "Nombre maximal de tweets récents conservés par profil." | |
| 22 | + }, | |
| 23 | + "requestDelay": { | |
| 24 | + "title": "Délai entre requêtes (s)", | |
| 25 | + "type": "integer", | |
| 26 | + "default": 1, | |
| 27 | + "minimum": 0, | |
| 28 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 29 | + }, | |
| 30 | + "concurrency": { | |
| 31 | + "title": "Requêtes simultanées", | |
| 32 | + "type": "integer", | |
| 33 | + "default": 4, | |
| 34 | + "minimum": 1, | |
| 35 | + "maximum": 10, | |
| 36 | + "description": "Nombre de requêtes menées en parallèle." | |
| 37 | + }, | |
| 38 | + "proxyConfiguration": { | |
| 39 | + "title": "Proxy", | |
| 40 | + "type": "object", | |
| 41 | + "editor": "proxy", | |
| 42 | + "description": "Datacenter suffit généralement (service de widgets ouvert).", | |
| 43 | + "prefill": { | |
| 44 | + "useApifyProxy": true | |
| 45 | + } | |
| 46 | + } | |
| 47 | + } | |
| 48 | +} | |
| \ No newline at end of file | ||
added
actors/ka-x/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-x) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-x/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-x) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-x/src/main.py
+217 −0
@@ -0,0 +1,217 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-x) | |
| 4 | +# Desc: Profils X (Twitter) SANS clé via le service OFFICIEL de syndication | |
| 5 | +# des widgets (syndication.twitter.com/srv/timeline-profile) : objet | |
| 6 | +# user complet + timeline récente avec métriques par tweet. Conçu pour | |
| 7 | +# être appelé sans authentification ; l'empreinte TLS Chrome de | |
| 8 | +# curl_cffi évite les 429 réservés aux clients python-requests. | |
| 9 | +# ============================================================================== | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import asyncio | |
| 13 | +import json | |
| 14 | +import re | |
| 15 | +import time | |
| 16 | + | |
| 17 | +from apify import Actor | |
| 18 | + | |
| 19 | +from .net import Fetcher | |
| 20 | + | |
| 21 | +SYNDICATION_URL = ("https://syndication.twitter.com/srv/timeline-profile/" | |
| 22 | + "screen-name/{u}") | |
| 23 | +_NEXT_DATA_RE = re.compile( | |
| 24 | + r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', re.S) | |
| 25 | + | |
| 26 | + | |
| 27 | +def _find_user(node, handle: str): | |
| 28 | + """Parcourt le JSON Next.js : l'objet user dont screen_name == handle.""" | |
| 29 | + if isinstance(node, dict): | |
| 30 | + if str(node.get("screen_name", "")).lower() == handle and \ | |
| 31 | + "followers_count" in node: | |
| 32 | + return node | |
| 33 | + for v in node.values(): | |
| 34 | + hit = _find_user(v, handle) | |
| 35 | + if hit is not None: | |
| 36 | + return hit | |
| 37 | + elif isinstance(node, list): | |
| 38 | + for v in node: | |
| 39 | + hit = _find_user(v, handle) | |
| 40 | + if hit is not None: | |
| 41 | + return hit | |
| 42 | + return None | |
| 43 | + | |
| 44 | + | |
| 45 | +def parse_tweets(data: dict, handle: str, cap: int) -> list[dict]: | |
| 46 | + entries = (((data.get("props") or {}).get("pageProps") or {}) | |
| 47 | + .get("timeline") or {}).get("entries") or [] | |
| 48 | + tweets: list[dict] = [] | |
| 49 | + for entry in entries: | |
| 50 | + t = (entry.get("content") or {}).get("tweet") or {} | |
| 51 | + if not t.get("id_str"): | |
| 52 | + continue | |
| 53 | + u = (t.get("user") or {}) | |
| 54 | + ent = t.get("entities") or {} | |
| 55 | + media = (t.get("extended_entities") or ent).get("media") or [] | |
| 56 | + tweets.append({ | |
| 57 | + "id": t.get("id_str"), | |
| 58 | + "url": f"https://x.com/{u.get('screen_name', handle)}" | |
| 59 | + f"/status/{t.get('id_str')}", | |
| 60 | + "text": (t.get("full_text") or t.get("text") or "")[:500], | |
| 61 | + "likes": t.get("favorite_count"), | |
| 62 | + "retweets": t.get("retweet_count"), | |
| 63 | + "replies": t.get("reply_count"), | |
| 64 | + "quotes": t.get("quote_count"), | |
| 65 | + "created_at": t.get("created_at"), | |
| 66 | + "is_retweet": bool(t.get("retweeted_status")), | |
| 67 | + "is_reply": bool(t.get("in_reply_to_status_id_str")), | |
| 68 | + "is_quote": bool(t.get("is_quote_status")), | |
| 69 | + "lang": t.get("lang"), | |
| 70 | + "hashtags": [h.get("text") for h in (ent.get("hashtags") or []) | |
| 71 | + if h.get("text")][:8], | |
| 72 | + "mentions": [m.get("screen_name") | |
| 73 | + for m in (ent.get("user_mentions") or []) | |
| 74 | + if m.get("screen_name")][:8], | |
| 75 | + "media_type": (media[0].get("type") if media else None), | |
| 76 | + "media_count": len(media) or None, | |
| 77 | + "media_urls": [x.get("media_url_https") for x in media | |
| 78 | + if x.get("media_url_https")][:4], | |
| 79 | + "links": [x.get("expanded_url") for x in (ent.get("urls") or []) | |
| 80 | + if x.get("expanded_url")][:4], | |
| 81 | + }) | |
| 82 | + if len(tweets) >= cap: | |
| 83 | + break | |
| 84 | + return tweets | |
| 85 | + | |
| 86 | + | |
| 87 | +def _ts(created_at: str) -> float | None: | |
| 88 | + """« Tue Sep 12 23:56:24 +0000 2023 » → epoch (None si illisible).""" | |
| 89 | + try: | |
| 90 | + from datetime import datetime | |
| 91 | + return datetime.strptime(created_at, | |
| 92 | + "%a %b %d %H:%M:%S %z %Y").timestamp() | |
| 93 | + except (TypeError, ValueError): | |
| 94 | + return None | |
| 95 | + | |
| 96 | + | |
| 97 | +async def main() -> None: | |
| 98 | + async with Actor: | |
| 99 | + inp = await Actor.get_input() or {} | |
| 100 | + usernames = [u.strip().lstrip("@").lower() | |
| 101 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 102 | + max_tweets = int(inp.get("maxTweets", 20)) | |
| 103 | + proxy = await Actor.create_proxy_configuration( | |
| 104 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 105 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 106 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 4)) | |
| 107 | + | |
| 108 | + async def one(u: str) -> None: | |
| 109 | + async with sem: | |
| 110 | + miss = {"kind": "profile", "platform": "x", | |
| 111 | + "username": u, "found": False} | |
| 112 | + try: | |
| 113 | + resp = await fetcher.get(SYNDICATION_URL.format(u=u), | |
| 114 | + session_id=u, impersonate=None) | |
| 115 | + if resp.status_code == 429: | |
| 116 | + resp = await fetcher.get(SYNDICATION_URL.format(u=u), | |
| 117 | + session_id=f"{u}b", | |
| 118 | + impersonate="chrome") | |
| 119 | + except Exception as exc: | |
| 120 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 121 | + return | |
| 122 | + m = _NEXT_DATA_RE.search(resp.text or "") | |
| 123 | + if resp.status_code != 200 or not m: | |
| 124 | + await Actor.push_data( | |
| 125 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 126 | + return | |
| 127 | + try: | |
| 128 | + data = json.loads(m.group(1)) | |
| 129 | + except Exception: | |
| 130 | + await Actor.push_data({**miss, "error": "bad_json"}) | |
| 131 | + return | |
| 132 | + user = _find_user(data, u) | |
| 133 | + if not user: | |
| 134 | + await Actor.push_data({**miss, "error": "user_not_found"}) | |
| 135 | + return | |
| 136 | + tweets = parse_tweets(data, u, max_tweets) | |
| 137 | + own = [t for t in tweets if not t["is_retweet"]] | |
| 138 | + likes = [t["likes"] for t in own if isinstance(t["likes"], int)] | |
| 139 | + rts = [t["retweets"] for t in own | |
| 140 | + if isinstance(t["retweets"], int)] | |
| 141 | + reps = [t["replies"] for t in own | |
| 142 | + if isinstance(t["replies"], int)] | |
| 143 | + followers = user.get("followers_count") | |
| 144 | + avg_likes = round(sum(likes) / len(likes)) if likes else None | |
| 145 | + avg_rts = round(sum(rts) / len(rts)) if rts else None | |
| 146 | + engagement = None | |
| 147 | + if followers and avg_likes is not None: | |
| 148 | + engagement = round((avg_likes + (avg_rts or 0)) | |
| 149 | + / followers * 100, 3) | |
| 150 | + stamps = sorted(s for s in (_ts(t["created_at"]) | |
| 151 | + for t in own) if s) | |
| 152 | + tweets_per_week = None | |
| 153 | + if len(stamps) >= 3 and stamps[-1] > stamps[0]: | |
| 154 | + tweets_per_week = round( | |
| 155 | + (len(stamps) - 1) / ((stamps[-1] - stamps[0]) | |
| 156 | + / 604_800), 2) | |
| 157 | + top = max((t for t in own if isinstance(t["likes"], int)), | |
| 158 | + key=lambda t: t["likes"], default=None) | |
| 159 | + hashtags: dict[str, int] = {} | |
| 160 | + for t in own: | |
| 161 | + for h in t["hashtags"]: | |
| 162 | + hashtags[h.lower()] = hashtags.get(h.lower(), 0) + 1 | |
| 163 | + created_ts = _ts(user.get("created_at")) | |
| 164 | + await Actor.push_data({ | |
| 165 | + "kind": "profile", | |
| 166 | + "platform": "x", | |
| 167 | + "found": True, | |
| 168 | + "id": user.get("id_str"), | |
| 169 | + "username": user.get("screen_name"), | |
| 170 | + "full_name": user.get("name"), | |
| 171 | + "biography": user.get("description"), | |
| 172 | + "location": user.get("location"), | |
| 173 | + "website": ((user.get("entities") or {}).get("url") or {}) | |
| 174 | + .get("urls", [{}])[0].get("expanded_url") | |
| 175 | + if (user.get("entities") or {}).get("url") else None, | |
| 176 | + "followers": followers, | |
| 177 | + "following": user.get("friends_count"), | |
| 178 | + "tweets_count": user.get("statuses_count"), | |
| 179 | + "listed_count": user.get("listed_count"), | |
| 180 | + "likes_given": user.get("favourites_count"), | |
| 181 | + "is_verified": bool(user.get("verified") | |
| 182 | + or user.get("is_blue_verified")), | |
| 183 | + "is_blue_verified": user.get("is_blue_verified"), | |
| 184 | + "is_protected": user.get("protected"), | |
| 185 | + "created_at": user.get("created_at"), | |
| 186 | + "account_age_days": (round((time.time() - created_ts) | |
| 187 | + / 86_400) | |
| 188 | + if created_ts else None), | |
| 189 | + "avatar": (user.get("profile_image_url_https") or "") | |
| 190 | + .replace("_normal.", "_400x400."), | |
| 191 | + "banner": user.get("profile_banner_url"), | |
| 192 | + # agrégats sur les tweets récents (hors retweets) | |
| 193 | + "avg_likes": avg_likes, | |
| 194 | + "avg_retweets": avg_rts, | |
| 195 | + "avg_replies": (round(sum(reps) / len(reps)) | |
| 196 | + if reps else None), | |
| 197 | + "engagement_rate_pct": engagement, | |
| 198 | + "tweets_per_week": tweets_per_week, | |
| 199 | + "last_tweet_at": own[0]["created_at"] if own else None, | |
| 200 | + "reply_share_pct": (round(sum(t["is_reply"] | |
| 201 | + for t in own) | |
| 202 | + / len(own) * 100) | |
| 203 | + if own else None), | |
| 204 | + "media_share_pct": (round(sum(bool(t["media_count"]) | |
| 205 | + for t in own) | |
| 206 | + / len(own) * 100) | |
| 207 | + if own else None), | |
| 208 | + "top_hashtags": sorted(hashtags, key=hashtags.get, | |
| 209 | + reverse=True)[:8], | |
| 210 | + "top_tweet": ({"url": top["url"], "likes": top["likes"], | |
| 211 | + "retweets": top["retweets"]} | |
| 212 | + if top else None), | |
| 213 | + "recent_tweets": tweets, | |
| 214 | + }) | |
| 215 | + | |
| 216 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 217 | + Actor.log.info(f"terminé : {len(usernames)} profils") | |
added
actors/ka-x/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-youtube/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-youtube) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-youtube/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-youtube", | |
| 4 | + "title": "KA YouTube — chaînes détaillées", | |
| 5 | + "description": "Chaînes YouTube publiques détaillées SANS clé (abonnés, vidéos, description, pays, avatar + vidéos récentes avec vues) via ytInitialData.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-youtube/.actor/input_schema.json
+46 −0
@@ -0,0 +1,46 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-youtube input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles/slugs à traiter (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "@mrbeast" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "fetchAbout": { | |
| 16 | + "title": "Récupérer la page À propos", | |
| 17 | + "type": "boolean", | |
| 18 | + "default": true, | |
| 19 | + "description": "2e requête /about par chaîne : vues totales, date de création, pays, liens externes auto-déclarés." | |
| 20 | + }, | |
| 21 | + "requestDelay": { | |
| 22 | + "title": "Délai entre requêtes (s)", | |
| 23 | + "type": "integer", | |
| 24 | + "default": 1, | |
| 25 | + "minimum": 0, | |
| 26 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 27 | + }, | |
| 28 | + "concurrency": { | |
| 29 | + "title": "Requêtes simultanées", | |
| 30 | + "type": "integer", | |
| 31 | + "default": 3, | |
| 32 | + "minimum": 1, | |
| 33 | + "maximum": 10, | |
| 34 | + "description": "Nombre de requêtes menées en parallèle." | |
| 35 | + }, | |
| 36 | + "proxyConfiguration": { | |
| 37 | + "title": "Proxy", | |
| 38 | + "type": "object", | |
| 39 | + "editor": "proxy", | |
| 40 | + "description": "Configuration du proxy Apify.", | |
| 41 | + "prefill": { | |
| 42 | + "useApifyProxy": true | |
| 43 | + } | |
| 44 | + } | |
| 45 | + } | |
| 46 | +} | |
| \ No newline at end of file | ||
added
actors/ka-youtube/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-youtube) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-youtube/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-youtube) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-youtube/src/main.py
+303 −0
@@ -0,0 +1,303 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-youtube) | |
| 4 | +# Desc: Chaînes YouTube ULTRA DÉTAILLÉES SANS clé API via le JSON ytInitialData | |
| 5 | +# de la page /videos : abonnés, nb de vidéos, description, avatar, | |
| 6 | +# BANNIÈRE, badge vérifié + vidéos récentes avec MINIATURES (vues, durée, | |
| 7 | +# date relative) et agrégats — plus un 2e passage /about (vues totales de | |
| 8 | +# la chaîne, date de création, pays, LIENS EXTERNES auto-déclarés). | |
| 9 | +# Accepte @handle, handle nu, ou id de chaîne (UC…). | |
| 10 | +# ============================================================================== | |
| 11 | +from __future__ import annotations | |
| 12 | + | |
| 13 | +import asyncio | |
| 14 | +import json | |
| 15 | +import re | |
| 16 | +import urllib.parse | |
| 17 | + | |
| 18 | +from apify import Actor | |
| 19 | + | |
| 20 | +from .net import Fetcher | |
| 21 | + | |
| 22 | +_INITIAL_RE = re.compile(r"var ytInitialData\s*=\s*(\{.*?\});</script>", re.S) | |
| 23 | +_COUNT_RE = re.compile(r"([\d][\d\s .,]*)\s*([KkMB]?)") | |
| 24 | + | |
| 25 | + | |
| 26 | +def _url_for(handle: str) -> str: | |
| 27 | + h = urllib.parse.unquote(handle).strip() | |
| 28 | + if h.startswith("UC") and len(h) == 24: | |
| 29 | + return f"https://www.youtube.com/channel/{h}/videos" | |
| 30 | + if not h.startswith("@"): | |
| 31 | + h = "@" + h | |
| 32 | + return f"https://www.youtube.com/{urllib.parse.quote(h)}/videos" | |
| 33 | + | |
| 34 | + | |
| 35 | +def parse_text_count(text: str) -> int | None: | |
| 36 | + """« 1.23M subscribers », « 12 345 vues », « 4,5 k » → entier.""" | |
| 37 | + if not text: | |
| 38 | + return None | |
| 39 | + m = _COUNT_RE.search(str(text)) | |
| 40 | + if not m: | |
| 41 | + return None | |
| 42 | + num = re.sub(r"[\s ]", "", m.group(1)) | |
| 43 | + unit = (m.group(2) or "").upper() | |
| 44 | + if unit: | |
| 45 | + num = num.replace(",", ".") | |
| 46 | + if num.count(".") > 1: | |
| 47 | + num = num.replace(".", "", num.count(".") - 1) | |
| 48 | + try: | |
| 49 | + val = float(num) | |
| 50 | + except ValueError: | |
| 51 | + return None | |
| 52 | + return int(val * {"K": 1e3, "M": 1e6, "B": 1e9}[unit]) | |
| 53 | + num = re.sub(r"[.,]", "", num) | |
| 54 | + return int(num) if num.isdigit() else None | |
| 55 | + | |
| 56 | + | |
| 57 | +def _walk(node, key: str): | |
| 58 | + """Premier objet portant `key` dans l'arbre ytInitialData.""" | |
| 59 | + if isinstance(node, dict): | |
| 60 | + if key in node: | |
| 61 | + return node[key] | |
| 62 | + for v in node.values(): | |
| 63 | + hit = _walk(v, key) | |
| 64 | + if hit is not None: | |
| 65 | + return hit | |
| 66 | + elif isinstance(node, list): | |
| 67 | + for v in node: | |
| 68 | + hit = _walk(v, key) | |
| 69 | + if hit is not None: | |
| 70 | + return hit | |
| 71 | + return None | |
| 72 | + | |
| 73 | + | |
| 74 | +def _texts(node) -> list[str]: | |
| 75 | + """Toutes les chaînes `content`/`simpleText`/`text` sous un nœud.""" | |
| 76 | + out: list[str] = [] | |
| 77 | + | |
| 78 | + def rec(n): | |
| 79 | + if isinstance(n, dict): | |
| 80 | + for k in ("content", "simpleText", "text"): | |
| 81 | + if isinstance(n.get(k), str): | |
| 82 | + out.append(n[k]) | |
| 83 | + for v in n.values(): | |
| 84 | + rec(v) | |
| 85 | + elif isinstance(n, list): | |
| 86 | + for v in n: | |
| 87 | + rec(v) | |
| 88 | + rec(node) | |
| 89 | + return out | |
| 90 | + | |
| 91 | + | |
| 92 | +def _lockup_video(lv: dict) -> dict | None: | |
| 93 | + """Carte vidéo moderne (lockupViewModel) → dict vidéo standardisé.""" | |
| 94 | + if lv.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO": | |
| 95 | + return None | |
| 96 | + vid = lv.get("contentId") | |
| 97 | + if not vid: | |
| 98 | + return None | |
| 99 | + md = lv.get("metadata") or {} | |
| 100 | + title = ((md.get("lockupMetadataViewModel") or {}).get("title") or {}) \ | |
| 101 | + .get("content") | |
| 102 | + strings = _texts(md) | |
| 103 | + views = next((parse_text_count(s) for s in strings | |
| 104 | + if "view" in s.lower() or "vue" in s.lower()), None) | |
| 105 | + published = next((s for s in strings | |
| 106 | + if "ago" in s.lower() or "il y a" in s.lower()), None) | |
| 107 | + overlays = _texts(lv.get("contentImage") or {}) | |
| 108 | + duration = next((s for s in overlays | |
| 109 | + if re.fullmatch(r"\d?\d:\d\d(:\d\d)?", s.strip())), None) | |
| 110 | + return { | |
| 111 | + "id": vid, | |
| 112 | + "url": f"https://www.youtube.com/watch?v={vid}", | |
| 113 | + "title": (title or "")[:200], | |
| 114 | + "views": views, | |
| 115 | + "published": published, | |
| 116 | + "duration": duration, | |
| 117 | + "thumbnail": f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg", | |
| 118 | + } | |
| 119 | + | |
| 120 | + | |
| 121 | +def _collect_videos(data: dict, cap: int = 15) -> list[dict]: | |
| 122 | + out: list[dict] = [] | |
| 123 | + | |
| 124 | + def rec(node): | |
| 125 | + if len(out) >= cap: | |
| 126 | + return | |
| 127 | + if isinstance(node, dict): | |
| 128 | + lv = node.get("lockupViewModel") | |
| 129 | + if isinstance(lv, dict): | |
| 130 | + v = _lockup_video(lv) | |
| 131 | + if v: | |
| 132 | + out.append(v) | |
| 133 | + return | |
| 134 | + vr = node.get("videoRenderer") or node.get("gridVideoRenderer") | |
| 135 | + if vr and vr.get("videoId"): | |
| 136 | + out.append({ | |
| 137 | + "id": vr["videoId"], | |
| 138 | + "url": f"https://www.youtube.com/watch?v={vr['videoId']}", | |
| 139 | + "title": "".join( | |
| 140 | + r.get("text", "") for r in | |
| 141 | + (vr.get("title") or {}).get("runs") or [])[:200], | |
| 142 | + "views": parse_text_count( | |
| 143 | + (vr.get("viewCountText") or {}).get("simpleText")), | |
| 144 | + "published": ((vr.get("publishedTimeText") or {}) | |
| 145 | + .get("simpleText")), | |
| 146 | + "duration": ((vr.get("lengthText") or {}) | |
| 147 | + .get("simpleText")), | |
| 148 | + "thumbnail": (f"https://i.ytimg.com/vi/{vr['videoId']}" | |
| 149 | + f"/hqdefault.jpg"), | |
| 150 | + }) | |
| 151 | + return | |
| 152 | + for v in node.values(): | |
| 153 | + rec(v) | |
| 154 | + elif isinstance(node, list): | |
| 155 | + for v in node: | |
| 156 | + rec(v) | |
| 157 | + | |
| 158 | + rec(data) | |
| 159 | + return out | |
| 160 | + | |
| 161 | + | |
| 162 | +def _s(v): | |
| 163 | + """ViewModel : champ tantôt chaîne nue, tantôt {content: "..."}.""" | |
| 164 | + return v.get("content") if isinstance(v, dict) else v | |
| 165 | + | |
| 166 | + | |
| 167 | +def _banner_of(data: dict) -> str | None: | |
| 168 | + """Bannière de chaîne : nouvel en-tête (imageBannerViewModel) ou ancien.""" | |
| 169 | + b = _walk(data, "imageBannerViewModel") | |
| 170 | + if isinstance(b, dict): | |
| 171 | + srcs = (b.get("image") or {}).get("sources") or [] | |
| 172 | + if srcs and srcs[-1].get("url"): | |
| 173 | + return srcs[-1]["url"] | |
| 174 | + thumbs = (((_walk(data, "c4TabbedHeaderRenderer") or {}).get("banner") | |
| 175 | + or {}).get("thumbnails")) or [] | |
| 176 | + return thumbs[-1].get("url") if thumbs else None | |
| 177 | + | |
| 178 | + | |
| 179 | +def _about_of(data: dict) -> dict: | |
| 180 | + """Panneau « À propos » (/about) → vues totales, date, pays, liens.""" | |
| 181 | + acv = _walk(data, "aboutChannelViewModel") or {} | |
| 182 | + if not acv: | |
| 183 | + return {} | |
| 184 | + links = [] | |
| 185 | + for entry in (acv.get("links") or []): | |
| 186 | + lv = entry.get("channelExternalLinkViewModel") or {} | |
| 187 | + url = _s(lv.get("link")) | |
| 188 | + if url: | |
| 189 | + links.append({"title": _s(lv.get("title")), "url": url}) | |
| 190 | + return { | |
| 191 | + "total_views": parse_text_count(_s(acv.get("viewCountText"))), | |
| 192 | + "joined_date": _s(acv.get("joinedDateText")), | |
| 193 | + "country": _s(acv.get("country")), | |
| 194 | + "external_links": links[:10], | |
| 195 | + } | |
| 196 | + | |
| 197 | + | |
| 198 | +async def main() -> None: | |
| 199 | + async with Actor: | |
| 200 | + inp = await Actor.get_input() or {} | |
| 201 | + fetch_about = bool(inp.get("fetchAbout", True)) | |
| 202 | + usernames = [u.strip() for u in (inp.get("usernames") or []) | |
| 203 | + if u and u.strip()] | |
| 204 | + proxy = await Actor.create_proxy_configuration( | |
| 205 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 206 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 207 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 3)) | |
| 208 | + | |
| 209 | + async def one(u: str) -> None: | |
| 210 | + async with sem: | |
| 211 | + miss = {"kind": "profile", "platform": "youtube", | |
| 212 | + "username": u, "found": False} | |
| 213 | + try: | |
| 214 | + resp = await fetcher.get( | |
| 215 | + _url_for(u), session_id=u, | |
| 216 | + headers={"Accept-Language": "en-US,en;q=0.9"}) | |
| 217 | + except Exception as exc: | |
| 218 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 219 | + return | |
| 220 | + m = _INITIAL_RE.search(resp.text or "") | |
| 221 | + if resp.status_code != 200 or not m: | |
| 222 | + await Actor.push_data( | |
| 223 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 224 | + return | |
| 225 | + try: | |
| 226 | + data = json.loads(m.group(1)) | |
| 227 | + except Exception: | |
| 228 | + await Actor.push_data({**miss, "error": "bad_json"}) | |
| 229 | + return | |
| 230 | + meta = _walk(data, "channelMetadataRenderer") or {} | |
| 231 | + if not meta.get("externalId"): | |
| 232 | + await Actor.push_data({**miss, "error": "no_channel"}) | |
| 233 | + return | |
| 234 | + # compteurs : le nouvel en-tête n'a plus subscriberCountText ; | |
| 235 | + # on lit les chaînes « N subscribers » / « N videos » du header | |
| 236 | + header = _walk(data, "pageHeaderViewModel") or {} | |
| 237 | + header_txt = _texts(header) or _texts( | |
| 238 | + _walk(data, "c4TabbedHeaderRenderer") or {}) | |
| 239 | + sub_text = next((t for t in header_txt | |
| 240 | + if "subscriber" in t.lower() | |
| 241 | + or "abonné" in t.lower()), "") | |
| 242 | + vid_text = next((t for t in header_txt | |
| 243 | + if ("video" in t.lower() | |
| 244 | + or "vidéo" in t.lower()) | |
| 245 | + and any(c.isdigit() for c in t)), "") | |
| 246 | + if not sub_text: # ancien en-tête | |
| 247 | + sub_text = ((_walk(data, "subscriberCountText") or {}) | |
| 248 | + .get("simpleText") or "") | |
| 249 | + subs = parse_text_count(sub_text) | |
| 250 | + videos_count = parse_text_count(vid_text) | |
| 251 | + vids = _collect_videos(data) | |
| 252 | + views = [v["views"] for v in vids | |
| 253 | + if isinstance(v["views"], int)] | |
| 254 | + text = resp.text or "" | |
| 255 | + is_verified = True if ( | |
| 256 | + '"BADGE_STYLE_TYPE_VERIFIED"' in text | |
| 257 | + or '"BADGE_STYLE_TYPE_VERIFIED_ARTIST"' in text | |
| 258 | + or '"OFFICIAL_ARTIST_BADGE"' in text) else None | |
| 259 | + about: dict = {} | |
| 260 | + if fetch_about: | |
| 261 | + try: | |
| 262 | + resp_a = await fetcher.get( | |
| 263 | + _url_for(u).replace("/videos", "/about"), | |
| 264 | + session_id=f"{u}a", | |
| 265 | + headers={"Accept-Language": "en-US,en;q=0.9"}) | |
| 266 | + ma = _INITIAL_RE.search(resp_a.text or "") | |
| 267 | + if ma: | |
| 268 | + about = _about_of(json.loads(ma.group(1))) | |
| 269 | + except Exception as exc: | |
| 270 | + Actor.log.warning(f"/about {u} : {exc}") | |
| 271 | + await Actor.push_data({ | |
| 272 | + "kind": "profile", | |
| 273 | + "platform": "youtube", | |
| 274 | + "found": True, | |
| 275 | + "username": u, | |
| 276 | + "channel_id": meta.get("externalId"), | |
| 277 | + "full_name": meta.get("title"), | |
| 278 | + "biography": (meta.get("description") or "")[:1000], | |
| 279 | + "followers": subs, | |
| 280 | + "videos_count": videos_count, | |
| 281 | + "is_family_safe": meta.get("isFamilySafe"), | |
| 282 | + "is_verified": is_verified, | |
| 283 | + "keywords": (meta.get("keywords") or "")[:300] or None, | |
| 284 | + "country": about.get("country") or _walk(data, "country"), | |
| 285 | + "canonical_url": meta.get("vanityChannelUrl"), | |
| 286 | + "avatar": (((meta.get("avatar") or {}).get("thumbnails") | |
| 287 | + or [{}])[-1].get("url")), | |
| 288 | + "banner": _banner_of(data), | |
| 289 | + "total_views": about.get("total_views"), | |
| 290 | + "joined_date": about.get("joined_date"), | |
| 291 | + "external_links": about.get("external_links") or [], | |
| 292 | + "avg_views": (round(sum(views) / len(views)) | |
| 293 | + if views else None), | |
| 294 | + "top_video": (max( | |
| 295 | + (v for v in vids if isinstance(v["views"], int)), | |
| 296 | + key=lambda v: v["views"], default=None)), | |
| 297 | + "last_video_published": (vids[0]["published"] | |
| 298 | + if vids else None), | |
| 299 | + "recent_videos": vids, | |
| 300 | + }) | |
| 301 | + | |
| 302 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 303 | + Actor.log.info(f"terminé : {len(usernames)} chaînes") | |
added
actors/ka-youtube/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
modified
creaka/connectors/apify_social.py
+53 −3
@@ -246,10 +246,22 @@ class _ApifySocialEnrich(BaseConnector): | ||
| 246 | 246 | if it.get(src) is not None} |
| 247 | 247 | if self.content_key and it.get(self.content_key): |
| 248 | 248 | metrics[self.content_key] = it[self.content_key][:12] |
| 249 | + # images du profil : avatar/bannière conservés PAR COMPTE (affichage | |
| 250 | + # par plateforme + chaîne de repli côté frontend) | |
| 251 | + for img_key in ("avatar", "banner"): | |
| 252 | + if it.get(img_key): | |
| 253 | + metrics[img_key] = it[img_key] | |
| 249 | 254 | metrics[self._stamp_key] = now_iso() # tampon d'enrichissement propre |
| 250 | 255 | acc.metrics.update(metrics) |
| 251 | − if not cr.avatar_url and it.get("avatar"): | |
| 256 | + # avatar/bannière de la FICHE : rafraîchis à chaque passage depuis la | |
| 257 | + # plateforme principale (les URLs CDN signées expirent — ex. Instagram) ; | |
| 258 | + # sinon on remplit seulement les manquants | |
| 259 | + if it.get("avatar") and (not cr.avatar_url | |
| 260 | + or acc.platform == cr.primary_platform): | |
| 252 | 261 | cr.avatar_url = it["avatar"] |
| 262 | + if it.get("banner") and (not getattr(cr, "banner_url", None) | |
| 263 | + or acc.platform == cr.primary_platform): | |
| 264 | + cr.banner_url = it["banner"] | |
| 253 | 265 | if not cr.bio and it.get("biography"): |
| 254 | 266 | cr.bio = it["biography"] |
| 255 | 267 | self.cross_links(cr, it) |
@@ -285,6 +297,7 @@ class ApifyInstagram(_ApifySocialEnrich): | ||
| 285 | 297 | "category": "category", "is_business": "is_business", |
| 286 | 298 | "business_category": "business_category", |
| 287 | 299 | "highlight_reels": "highlight_reels", |
| 300 | + "pronouns": "pronouns", "bio_links": "bio_links", | |
| 288 | 301 | "avg_likes": "avg_likes", "avg_comments": "avg_comments", |
| 289 | 302 | "avg_video_views": "avg_video_views", |
| 290 | 303 | "engagement_rate_pct": "engagement_rate_pct", |
@@ -319,6 +332,9 @@ class ApifyTikTok(_ApifySocialEnrich): | ||
| 319 | 332 | "avg_views": "avg_views", "avg_likes": "avg_likes", |
| 320 | 333 | "avg_comments": "avg_comments", "avg_shares": "avg_shares", |
| 321 | 334 | "engagement_rate_pct": "engagement_rate_pct", |
| 335 | + "videos_per_week": "videos_per_week", | |
| 336 | + "last_video_at": "last_video_at", | |
| 337 | + "top_hashtags": "top_hashtags", "language": "language", | |
| 322 | 338 | "top_video": "top_video"} |
| 323 | 339 | |
| 324 | 340 | def bio_urls(self, it: dict) -> list[str]: |
@@ -340,6 +356,9 @@ class ApifyX(_ApifySocialEnrich): | ||
| 340 | 356 | "tweets_per_week": "tweets_per_week", |
| 341 | 357 | "last_tweet_at": "last_tweet_at", |
| 342 | 358 | "media_share_pct": "media_share_pct", |
| 359 | + "reply_share_pct": "reply_share_pct", | |
| 360 | + "is_blue_verified": "is_blue_verified", | |
| 361 | + "likes_given": "likes_given", "website": "website", | |
| 343 | 362 | "top_hashtags": "top_hashtags", "top_tweet": "top_tweet"} |
| 344 | 363 | |
| 345 | 364 | def extra_input(self) -> dict: |
@@ -380,6 +399,9 @@ class ApifySnapchat(_ApifySocialEnrich): | ||
| 380 | 399 | "publisher_type": "publisher_type", |
| 381 | 400 | "story_snaps_count": "story_snaps", |
| 382 | 401 | "lenses_count": "lenses", |
| 402 | + "story_previews": "story_previews", | |
| 403 | + "spotlight_previews": "spotlight_previews", | |
| 404 | + "website": "website", | |
| 383 | 405 | "spotlight_highlights_count": "spotlight_highlights"} |
| 384 | 406 | |
| 385 | 407 | def bio_urls(self, it: dict) -> list[str]: |
@@ -395,8 +417,15 @@ class ApifyYouTube(_ApifySocialEnrich): | ||
| 395 | 417 | metric_map = {"videos_count": "videos", "country": "country", |
| 396 | 418 | "keywords": "keywords", "channel_id": "channel_id", |
| 397 | 419 | "avg_views": "avg_views", "top_video": "top_video", |
| 420 | + "total_views": "total_views", "joined_date": "joined_date", | |
| 421 | + "external_links": "external_links", | |
| 398 | 422 | "last_video_published": "last_video_published"} |
| 399 | 423 | |
| 424 | + def bio_urls(self, it: dict) -> list[str]: | |
| 425 | + # liens externes AUTO-DÉCLARÉS de la page À propos → cross_link (§12.1) | |
| 426 | + return [ln.get("url") or "" for ln in (it.get("external_links") or []) | |
| 427 | + if isinstance(ln, dict)] | |
| 428 | + | |
| 400 | 429 | |
| 401 | 430 | class ApifyTwitch(_ApifySocialEnrich): |
| 402 | 431 | source_id = "twitch-apify" |
@@ -410,9 +439,18 @@ class ApifyTwitch(_ApifySocialEnrich): | ||
| 410 | 439 | "live_game": "live_game", |
| 411 | 440 | "last_broadcast_at": "last_broadcast_at", |
| 412 | 441 | "last_broadcast_game": "last_broadcast_game", |
| 442 | + "last_broadcast_title": "last_broadcast_title", | |
| 443 | + "live_title": "live_title", | |
| 444 | + "live_thumbnail": "live_thumbnail", | |
| 413 | 445 | "avg_video_views": "avg_video_views", |
| 446 | + "social_links": "social_links", | |
| 414 | 447 | "recent_games": "recent_games"} |
| 415 | 448 | |
| 449 | + def bio_urls(self, it: dict) -> list[str]: | |
| 450 | + # panneau « À propos » Twitch : liens sociaux AUTO-DÉCLARÉS (§12.1) | |
| 451 | + return [ln.get("url") or "" for ln in (it.get("social_links") or []) | |
| 452 | + if isinstance(ln, dict)] | |
| 453 | + | |
| 416 | 454 | |
| 417 | 455 | class ApifyKick(_ApifySocialEnrich): |
| 418 | 456 | source_id = "kick-apify" |
@@ -421,6 +459,10 @@ class ApifyKick(_ApifySocialEnrich): | ||
| 421 | 459 | cap = 600 |
| 422 | 460 | metric_map = {"is_live_now": "is_live_now", "live_viewers": "live_viewers", |
| 423 | 461 | "subscription_enabled": "subscription_enabled", |
| 462 | + "live_title": "live_title", | |
| 463 | + "live_thumbnail": "live_thumbnail", | |
| 464 | + "live_category": "live_category", | |
| 465 | + "vod_enabled": "vod_enabled", | |
| 424 | 466 | "recent_categories": "recent_categories"} |
| 425 | 467 | |
| 426 | 468 | def cross_links(self, cr: Creator, it: dict) -> None: |
@@ -479,7 +521,13 @@ class ApifyPatreon(_ApifySocialEnrich): | ||
| 479 | 521 | cap = 400 |
| 480 | 522 | revisit_days = 7 |
| 481 | 523 | metric_map = {"patrons": "patrons", "posts_count": "posts", |
| 482 | − "is_monthly": "is_monthly"} | |
| 524 | + "is_monthly": "is_monthly", "is_nsfw": "is_nsfw", | |
| 525 | + "creation_name": "creation_name", | |
| 526 | + "social_links": "social_links"} | |
| 527 | + | |
| 528 | + def bio_urls(self, it: dict) -> list[str]: | |
| 529 | + # liens sociaux AUTO-DÉCLARÉS de la campagne Patreon → cross_link | |
| 530 | + return list((it.get("social_links") or {}).values()) | |
| 483 | 531 | |
| 484 | 532 | |
| 485 | 533 | class ApifyDiscord(_ApifySocialEnrich): |
@@ -489,7 +537,9 @@ class ApifyDiscord(_ApifySocialEnrich): | ||
| 489 | 537 | cap = 400 |
| 490 | 538 | revisit_days = 7 |
| 491 | 539 | metric_map = {"members": "members", "online": "online", "boosts": "boosts", |
| 492 | − "partnered": "partnered", "guild_id": "guild_id"} | |
| 540 | + "partnered": "partnered", "guild_id": "guild_id", | |
| 541 | + "channel": "channel", "splash": "splash", | |
| 542 | + "vanity_url_code": "vanity_url_code"} | |
| 493 | 543 | |
| 494 | 544 | def target_of(self, acc) -> str: |
| 495 | 545 | """Code d'invitation SENSIBLE À LA CASSE, repris de l'URL (pas du |
modified
creaka/db.py
+56 −0
@@ -66,6 +66,16 @@ CREATE TABLE IF NOT EXISTS accounts ( | ||
| 66 | 66 | CREATE INDEX IF NOT EXISTS idx_accounts_creator ON accounts(creator_id); |
| 67 | 67 | CREATE INDEX IF NOT EXISTS idx_creators_status ON creators(status); |
| 68 | 68 | CREATE INDEX IF NOT EXISTS idx_creators_tier ON creators(audience_tier); |
| 69 | +CREATE TABLE IF NOT EXISTS snapshots ( | |
| 70 | + day TEXT NOT NULL, | |
| 71 | + platform TEXT NOT NULL, | |
| 72 | + handle TEXT NOT NULL, | |
| 73 | + creator_id TEXT NOT NULL, | |
| 74 | + followers INTEGER, | |
| 75 | + engagement REAL, | |
| 76 | + PRIMARY KEY (day, platform, handle) | |
| 77 | +); | |
| 78 | +CREATE INDEX IF NOT EXISTS idx_snapshots_creator ON snapshots(creator_id, day); | |
| 69 | 79 | CREATE TABLE IF NOT EXISTS sync_log ( |
| 70 | 80 | id INTEGER PRIMARY KEY AUTOINCREMENT, |
| 71 | 81 | ts TEXT NOT NULL, |
@@ -180,6 +190,20 @@ def _write_creator(con: sqlite3.Connection, cid: str, cr: Creator, | ||
| 180 | 190 | None if acc.verified is None else int(acc.verified), |
| 181 | 191 | acc.confidence, acc.signal, int(needs_review(acc)), acc.last_checked, |
| 182 | 192 | json.dumps(acc.metrics, ensure_ascii=False) if acc.metrics else None)) |
| 193 | + # historisation (§13-14) : un point par jour et par compte — matière | |
| 194 | + # première des courbes de croissance et des insights (7 j / 30 j) | |
| 195 | + if acc.followers: | |
| 196 | + con.execute( | |
| 197 | + """INSERT INTO snapshots (day, platform, handle, creator_id, | |
| 198 | + followers, engagement) | |
| 199 | + VALUES (?,?,?,?,?,?) | |
| 200 | + ON CONFLICT(day, platform, handle) DO UPDATE SET | |
| 201 | + creator_id=excluded.creator_id, | |
| 202 | + followers=excluded.followers, | |
| 203 | + engagement=COALESCE(excluded.engagement, | |
| 204 | + snapshots.engagement)""", | |
| 205 | + (ts[:10], acc.platform, acc.handle, cid, acc.followers, | |
| 206 | + (acc.metrics or {}).get("engagement_rate_pct"))) | |
| 183 | 207 | |
| 184 | 208 | |
| 185 | 209 | def _find_existing(con: sqlite3.Connection, cr: Creator) -> str | None: |
@@ -373,6 +397,38 @@ def search(con: sqlite3.Connection, *, q: str = "", niche: str = "", | ||
| 373 | 397 | return {"total": total, "count": len(items), "offset": offset, "items": items} |
| 374 | 398 | |
| 375 | 399 | |
| 400 | +def follower_history(con: sqlite3.Connection, cid: str, | |
| 401 | + days: int = 90) -> dict: | |
| 402 | + """Historique d'audience d'une fiche (snapshots quotidiens, §13-14). | |
| 403 | + | |
| 404 | + Retourne les séries par plateforme (jour → abonnés) + la série totale | |
| 405 | + (somme des plateformes connues ce jour-là), prêtes pour une sparkline. | |
| 406 | + """ | |
| 407 | + cutoff = (datetime.now(timezone.utc) - timedelta(days=days)) \ | |
| 408 | + .strftime("%Y-%m-%d") | |
| 409 | + rows = con.execute( | |
| 410 | + "SELECT day, platform, handle, followers FROM snapshots " | |
| 411 | + "WHERE creator_id=? AND day>=? ORDER BY day", (cid, cutoff)).fetchall() | |
| 412 | + by_platform: dict[str, dict[str, int]] = {} | |
| 413 | + for r in rows: | |
| 414 | + by_platform.setdefault(r["platform"], {})[r["day"]] = r["followers"] | |
| 415 | + total: dict[str, int] = {} | |
| 416 | + for series in by_platform.values(): | |
| 417 | + # report de la dernière valeur connue pour que la somme quotidienne | |
| 418 | + # ne s'effondre pas quand une plateforme n'a pas de point ce jour-là | |
| 419 | + last = None | |
| 420 | + for day in sorted({d for s in by_platform.values() for d in s}): | |
| 421 | + last = series.get(day, last) | |
| 422 | + if last is not None: | |
| 423 | + total[day] = total.get(day, 0) + last | |
| 424 | + return { | |
| 425 | + "platforms": {p: [{"day": d, "followers": f} | |
| 426 | + for d, f in sorted(s.items())] | |
| 427 | + for p, s in by_platform.items()}, | |
| 428 | + "total": [{"day": d, "followers": f} for d, f in sorted(total.items())], | |
| 429 | + } | |
| 430 | + | |
| 431 | + | |
| 376 | 432 | def get_creator(con: sqlite3.Connection, cid: str) -> dict | None: |
| 377 | 433 | row = con.execute( |
| 378 | 434 | "SELECT * FROM creators WHERE id=? AND status='active' AND is_minor=0", |
modified
creaka/schema.py
+1 −0
@@ -100,6 +100,7 @@ class Creator: | ||
| 100 | 100 | platforms: list[PlatformAccount] = field(default_factory=list) |
| 101 | 101 | link_in_bio_url: str | None = None |
| 102 | 102 | avatar_url: str | None = None # photo de profil PUBLIQUE (URL source) |
| 103 | + banner_url: str | None = None # bannière/cover PUBLIQUE (URL source) | |
| 103 | 104 | total_reach: int | None = None |
| 104 | 105 | audience_tier: str = "" |
| 105 | 106 | business_contact: str | None = None # courriel PRO affiché publiquement (§15) |
modified
creaka/stats.py
+90 −0
@@ -628,3 +628,93 @@ def dashboard(con: sqlite3.Connection, period: str = "30j", | ||
| 628 | 628 | _CACHE.clear() |
| 629 | 629 | _CACHE[key] = (now, data) |
| 630 | 630 | return data |
| 631 | + | |
| 632 | + | |
| 633 | +# --- insights par créateur (fiche « légendaire ») ------------------------------ | |
| 634 | + | |
| 635 | +def creator_insights(con: sqlite3.Connection, doc: dict) -> dict: | |
| 636 | + """Insights d'une fiche créateur : croissance (snapshots §13-14), | |
| 637 | + engagement pondéré, rythme de publication et Ka Score composite /100. | |
| 638 | + | |
| 639 | + AUCUNE stat inventée : tout provient des comptes rattachés (accounts.metrics | |
| 640 | + remplis par les acteurs Apify) et des snapshots quotidiens. Un champ absent | |
| 641 | + reste absent — pas d'estimation. | |
| 642 | + """ | |
| 643 | + import math | |
| 644 | + | |
| 645 | + from .db import follower_history | |
| 646 | + | |
| 647 | + platforms = doc.get("platforms") or [] | |
| 648 | + reach = doc.get("total_reach") or sum( | |
| 649 | + p.get("followers") or 0 for p in platforms) or 0 | |
| 650 | + | |
| 651 | + # croissance : série totale quotidienne (report dernière valeur connue) | |
| 652 | + hist = follower_history(con, doc["id"], days=95) | |
| 653 | + total = hist["total"] | |
| 654 | + | |
| 655 | + def growth(days: int) -> dict | None: | |
| 656 | + if len(total) < 2: | |
| 657 | + return None | |
| 658 | + last = total[-1] | |
| 659 | + cutoff = (date.today() - timedelta(days=days)).isoformat() | |
| 660 | + base = next((p for p in total if p["day"] >= cutoff), None) | |
| 661 | + if base is None or base["day"] == last["day"] or not base["followers"]: | |
| 662 | + return None | |
| 663 | + delta = last["followers"] - base["followers"] | |
| 664 | + return {"since": base["day"], "delta": delta, | |
| 665 | + "pct": round(100 * delta / base["followers"], 2)} | |
| 666 | + | |
| 667 | + # engagement moyen pondéré par l'audience de chaque plateforme | |
| 668 | + weighted = [(p["metrics"].get("engagement_rate_pct"), p.get("followers") or 1) | |
| 669 | + for p in platforms | |
| 670 | + if isinstance(p.get("metrics"), dict) | |
| 671 | + and p["metrics"].get("engagement_rate_pct") is not None] | |
| 672 | + engagement = (round(sum(e * w for e, w in weighted) | |
| 673 | + / sum(w for _, w in weighted), 2) | |
| 674 | + if weighted else None) | |
| 675 | + | |
| 676 | + # rythme de publication : somme des cadences hebdo déclarées par plateforme | |
| 677 | + rates = [p["metrics"].get(k) for p in platforms | |
| 678 | + if isinstance(p.get("metrics"), dict) | |
| 679 | + for k in ("posts_per_week", "videos_per_week", "tweets_per_week") | |
| 680 | + if isinstance(p["metrics"].get(k), (int, float))] | |
| 681 | + pubs_week = round(sum(rates), 1) if rates else None | |
| 682 | + | |
| 683 | + # dernière activité publique connue, toutes plateformes confondues | |
| 684 | + last_dates = [str(p["metrics"].get(k)) for p in platforms | |
| 685 | + if isinstance(p.get("metrics"), dict) | |
| 686 | + for k in ("last_post_at", "last_video_at", "last_tweet_at", | |
| 687 | + "last_broadcast_at", "last_video_published") | |
| 688 | + if p["metrics"].get(k)] | |
| 689 | + last_activity = max(last_dates) if last_dates else None | |
| 690 | + | |
| 691 | + is_verified = any(p.get("verified") for p in platforms) | |
| 692 | + live_now = any(isinstance(p.get("metrics"), dict) | |
| 693 | + and p["metrics"].get("is_live_now") for p in platforms) | |
| 694 | + | |
| 695 | + # Ka Score /100 : audience 40 (log), engagement 25, présence 20, rythme 10, | |
| 696 | + # vérification 5 — comparable d'un créateur à l'autre, jamais inventé : | |
| 697 | + # une composante inconnue vaut simplement 0. | |
| 698 | + parts = { | |
| 699 | + "audience": round(40 * min(1.0, math.log10(max(reach, 1)) / 7), 1), | |
| 700 | + "engagement": round(25 * min(1.0, (engagement or 0) / 10), 1), | |
| 701 | + "presence": round(20 * min(1.0, len(platforms) / 5), 1), | |
| 702 | + "rythme": round(10 * min(1.0, (pubs_week or 0) / 3), 1), | |
| 703 | + "verification": 5.0 if is_verified else 0.0, | |
| 704 | + } | |
| 705 | + top = max(platforms, key=lambda p: p.get("followers") or 0, default=None) | |
| 706 | + return { | |
| 707 | + "ka_score": round(sum(parts.values()), 1), | |
| 708 | + "ka_score_parts": parts, | |
| 709 | + "total_reach": reach or None, | |
| 710 | + "growth_7d": growth(7), | |
| 711 | + "growth_30d": growth(30), | |
| 712 | + "avg_engagement_pct": engagement, | |
| 713 | + "publications_per_week": pubs_week, | |
| 714 | + "last_activity": last_activity, | |
| 715 | + "platforms_count": len(platforms), | |
| 716 | + "is_verified_somewhere": is_verified, | |
| 717 | + "is_live_now": live_now, | |
| 718 | + "top_platform": (top or {}).get("platform"), | |
| 719 | + "history": hist, | |
| 720 | + } | |
modified
creaka/web.py
+3 −0
@@ -75,6 +75,9 @@ def get_creator(cid: str): | ||
| 75 | 75 | creator = db.get_creator(_db(), cid) |
| 76 | 76 | if creator is None: |
| 77 | 77 | raise HTTPException(404, "créateur introuvable") |
| 78 | + # insights « légendaires » : croissance (snapshots), engagement pondéré, | |
| 79 | + # rythme de publication, Ka Score /100 + historique pour la sparkline | |
| 80 | + creator["insights"] = stats_mod.creator_insights(_db(), creator) | |
| 78 | 81 | return creator |
| 79 | 82 | |
| 80 | 83 | |
modified
frontend/dist/index.html
+102 −5
@@ -997,6 +997,50 @@ select.f.active{background-color:var(--accent-soft);color:var(--ink);border-colo | ||
| 997 | 997 | .linkinbio svg{width:16px;height:16px;fill:currentColor} |
| 998 | 998 | .cpage-foot{margin-top:38px;font-size:12px;color:var(--ink-3); |
| 999 | 999 | border-top:1.5px solid var(--ink);padding-top:16px} |
| 1000 | +/* bannière héro (banner_url capté par les acteurs Apify) */ | |
| 1001 | +.cbanner{position:relative;border:1.5px solid var(--ink);border-bottom:none; | |
| 1002 | + border-radius:var(--r-card) var(--r-card) 0 0;overflow:hidden;height:clamp(120px,22vw,240px); | |
| 1003 | + background:var(--ink)} | |
| 1004 | +.cbanner img{width:100%;height:100%;object-fit:cover;display:block} | |
| 1005 | +.cbanner+.cpage-head{border-radius:0 0 var(--r-card) var(--r-card)} | |
| 1006 | +/* Ka Score */ | |
| 1007 | +.kascore{display:flex;gap:22px;align-items:center;flex-wrap:wrap;margin:6px 0 2px} | |
| 1008 | +.kascore-num{flex:none;width:112px;height:112px;border-radius:50%;display:flex; | |
| 1009 | + flex-direction:column;align-items:center;justify-content:center; | |
| 1010 | + border:2px solid var(--ink);background:var(--accent);color:var(--on-accent); | |
| 1011 | + box-shadow:var(--shadow-off-soft)} | |
| 1012 | +.kascore-num b{font-family:var(--font-display);font-size:34px;font-weight:800;line-height:1} | |
| 1013 | +.kascore-num span{font-family:var(--font-mono);font-size:9px;font-weight:700; | |
| 1014 | + text-transform:uppercase;letter-spacing:.06em;margin-top:3px} | |
| 1015 | +.kascore-parts{flex:1;min-width:230px;display:flex;flex-direction:column;gap:7px} | |
| 1016 | +.kspart{display:flex;align-items:center;gap:10px} | |
| 1017 | +.kspart-lab{width:110px;flex:none;font-family:var(--font-mono);font-size:10px; | |
| 1018 | + font-weight:700;text-transform:uppercase;letter-spacing:.04em;color:var(--ink-3)} | |
| 1019 | +.kspart-track{flex:1;height:9px;background:var(--surface-2);border:1px solid var(--line); | |
| 1020 | + border-radius:5px;overflow:hidden} | |
| 1021 | +.kspart-fill{display:block;height:100%;background:var(--green);border-radius:5px 0 0 5px;min-width:2px} | |
| 1022 | +.kspart-val{width:52px;flex:none;text-align:right;font-family:var(--font-mono); | |
| 1023 | + font-size:11px;font-weight:700;color:var(--ink)} | |
| 1024 | +@media(max-width:640px){.kspart-lab{width:88px;font-size:9px}} | |
| 1025 | +/* sparkline évolution d'audience */ | |
| 1026 | +.spark-wrap{margin-top:8px} | |
| 1027 | +.spark-svg{width:100%;height:88px;display:block} | |
| 1028 | +.spark-line{fill:none;stroke:var(--green);stroke-width:2.5;stroke-linejoin:round;stroke-linecap:round} | |
| 1029 | +.spark-fill{fill:var(--lime-soft);opacity:.7;stroke:none} | |
| 1030 | +.spark-meta{display:flex;justify-content:space-between;font-family:var(--font-mono); | |
| 1031 | + font-size:10px;color:var(--ink-3);margin-top:4px} | |
| 1032 | +.growth-up{color:var(--green-deep);font-weight:700} | |
| 1033 | +.growth-down{color:#b0201a;font-weight:700} | |
| 1034 | +/* bloc EN DIRECT (miniature live Twitch/Kick) */ | |
| 1035 | +.live-card{display:flex;gap:14px;align-items:stretch;margin:14px 0 4px; | |
| 1036 | + border:1.5px solid #b0201a;border-radius:var(--r-card);overflow:hidden; | |
| 1037 | + background:var(--surface);text-decoration:none;color:inherit} | |
| 1038 | +.live-card img{width:200px;min-height:100%;object-fit:cover;flex:none;display:block} | |
| 1039 | +.live-card-body{padding:12px 14px;display:flex;flex-direction:column;gap:5px;justify-content:center} | |
| 1040 | +.live-card-tag{font-family:var(--font-mono);font-size:10px;font-weight:700;color:#b0201a} | |
| 1041 | +.live-card-title{font-size:13.5px;font-weight:700;color:var(--ink);line-height:1.3} | |
| 1042 | +.live-card-meta{font-family:var(--font-mono);font-size:10.5px;color:var(--ink-3)} | |
| 1043 | +@media(max-width:560px){.live-card{flex-direction:column}.live-card img{width:100%;height:150px}} | |
| 1000 | 1044 | .cpage-foot a{color:var(--green);font-weight:600} |
| 1001 | 1045 | |
| 1002 | 1046 | /* ===== page stats ===== */ |
@@ -1455,7 +1499,11 @@ const _relTimeC=ts=>_relTime(ts); | ||
| 1455 | 1499 | window.thumbFail=el=>{const w=el.parentElement;if(w){w.classList.add("noimg"); |
| 1456 | 1500 | w.innerHTML=w.dataset.icon||"";}}; |
| 1457 | 1501 | function _mediaItems(a){const m=a.metrics||{}; |
| 1458 | − return m.recent_posts||m.recent_videos||m.recent_tweets||[];} | |
| 1502 | + const base=m.recent_posts||m.recent_videos||m.recent_tweets||[]; | |
| 1503 | + if(base.length)return base; | |
| 1504 | + // Snapchat : les previews de story/spotlight sont des listes d'URLs d'images | |
| 1505 | + const prev=[...(m.story_previews||[]),...(m.spotlight_previews||[])]; | |
| 1506 | + return prev.map(u=>({display_url:u,type:"story"}));} | |
| 1459 | 1507 | |
| 1460 | 1508 | /* grille de contenu récent, images/vidéos bien affichées */ |
| 1461 | 1509 | function contentGrid(a){ |
@@ -1463,7 +1511,7 @@ function contentGrid(a){ | ||
| 1463 | 1511 | if(!items.length)return `<div class="cg-empty">${icon(a.platform)} Aucun contenu récent capté pour ce compte.</div>`; |
| 1464 | 1512 | const ico=icon(a.platform); |
| 1465 | 1513 | const cards=items.slice(0,12).map(it=>{ |
| 1466 | − const img=it.display_url||it.thumbnail_url||""; | |
| 1514 | + const img=it.display_url||it.thumbnail_url||it.thumbnail||it.cover||(it.media_urls||[])[0]||""; | |
| 1467 | 1515 | const isVid=it.type==="video"||it.type==="reel"||!!it.duration||it.video_views!=null; |
| 1468 | 1516 | const cap=esc((it.caption||it.title||it.text||"").slice(0,140)); |
| 1469 | 1517 | const when=_relTimeC(it.timestamp||it.published_at||it.published||it.created_at); |
@@ -1497,7 +1545,8 @@ function platformKPIs(a){ | ||
| 1497 | 1545 | push(m.avg_views!=null?m.avg_views:m.avg_video_views,"vues / pub (moy.)"); |
| 1498 | 1546 | push(m.avg_comments,"comm. / pub (moy.)"); |
| 1499 | 1547 | push(m.avg_retweets,"RT / pub (moy.)"); |
| 1500 | − push(m.posts_per_week!=null?m.posts_per_week:m.tweets_per_week,"pubs / semaine"); | |
| 1548 | + push(m.posts_per_week!=null?m.posts_per_week:(m.tweets_per_week!=null?m.tweets_per_week:m.videos_per_week),"pubs / semaine"); | |
| 1549 | + push(m.total_views,"vues totales"); | |
| 1501 | 1550 | push(m.posts,"publications"); |
| 1502 | 1551 | push(m.videos,"vidéos"); |
| 1503 | 1552 | push(m.likes,a.platform==="tiktok"?"j'aime cumulés":"j'aime"); |
@@ -1521,6 +1570,10 @@ function platformBadges(a){ | ||
| 1521 | 1570 | m.is_monthly?'<span class="abadge">Mensuel</span>':"", |
| 1522 | 1571 | m.has_story?'<span class="abadge">Story active</span>':"", |
| 1523 | 1572 | m.has_spotlight?'<span class="abadge">Spotlight</span>':"", |
| 1573 | + m.is_nsfw?'<span class="abadge live">18+</span>':"", | |
| 1574 | + m.creation_name?`<span class="abadge">${esc(String(m.creation_name).slice(0,60))}</span>`:"", | |
| 1575 | + m.joined_date?`<span class="abadge">${esc(m.joined_date)}</span>`:"", | |
| 1576 | + m.is_blue_verified?'<span class="abadge ok">✓ Bleu</span>':"", | |
| 1524 | 1577 | m.country?`<span class="abadge">${esc(m.country)}</span>`:"", |
| 1525 | 1578 | (m.recent_games||[]).slice(0,2).map(g=>`<span class="abadge">🎮 ${esc(g)}</span>`).join(""), |
| 1526 | 1579 | (m.top_hashtags||[]).slice(0,3).map(h=>`<span class="abadge">#${esc(h)}</span>`).join(""), |
@@ -1544,6 +1597,13 @@ function platformPanel(a,idx,active){ | ||
| 1544 | 1597 | <span class="conf ${confCls}" title="Score de rattachement">${conf}%</span> |
| 1545 | 1598 | </div> |
| 1546 | 1599 | ${badges?`<div class="abadge-row">${badges}</div>`:""} |
| 1600 | + ${(a.metrics||{}).is_live_now&&(a.metrics||{}).live_thumbnail?` | |
| 1601 | + <a class="live-card" href="${esc(a.url)}" target="_blank" rel="noopener"> | |
| 1602 | + <img loading="lazy" src="${esc((a.metrics||{}).live_thumbnail)}" alt="" referrerpolicy="no-referrer" onerror="this.remove()"> | |
| 1603 | + <span class="live-card-body"><span class="live-card-tag">● EN DIRECT</span> | |
| 1604 | + ${(a.metrics||{}).live_title?`<span class="live-card-title">${esc((a.metrics||{}).live_title)}</span>`:""} | |
| 1605 | + <span class="live-card-meta">${(a.metrics||{}).live_viewers!=null?fmt.format((a.metrics||{}).live_viewers)+" spectateurs":""}${(a.metrics||{}).live_game?" · "+esc((a.metrics||{}).live_game):((a.metrics||{}).live_category?" · "+esc((a.metrics||{}).live_category):"")}</span> | |
| 1606 | + </span></a>`:""} | |
| 1547 | 1607 | <div class="kpi-grid">${kpis||'<div class="kpi"><b>—</b><span>pas encore de métriques</span></div>'}</div> |
| 1548 | 1608 | ${top&&top.url?`<a class="ppanel-top" href="${esc(top.url)}" target="_blank" rel="noopener">★ Top contenu — ${top.views!=null?fmt.format(top.views)+" vues":""}${top.likes!=null?" · "+fmt.format(top.likes)+" ❤":""} ↗</a>`:""} |
| 1549 | 1609 | <h3 class="cg-title">Contenu récent</h3> |
@@ -1571,6 +1631,21 @@ function computeAgg(c){ | ||
| 1571 | 1631 | nPlat:P.length,nVerif:P.filter(a=>a.verified).length,main}; |
| 1572 | 1632 | } |
| 1573 | 1633 | |
| 1634 | +/* sparkline SVG (série quotidienne des snapshots — insights.history.total) */ | |
| 1635 | +function sparkSvg(pts){ | |
| 1636 | + if(!pts||pts.length<2)return""; | |
| 1637 | + const w=600,h=88,pad=4; | |
| 1638 | + const vals=pts.map(p=>p.followers); | |
| 1639 | + const min=Math.min(...vals),max=Math.max(...vals),span=Math.max(1,max-min); | |
| 1640 | + const x=i=>pad+i*(w-2*pad)/(pts.length-1); | |
| 1641 | + const y=v=>h-pad-((v-min)/span)*(h-2*pad); | |
| 1642 | + const line=pts.map((p,i)=>`${x(i).toFixed(1)},${y(p.followers).toFixed(1)}`).join(" "); | |
| 1643 | + const area=`${pad},${h-pad} ${line} ${(w-pad).toFixed(1)},${h-pad}`; | |
| 1644 | + return `<div class="spark-wrap"><svg class="spark-svg" viewBox="0 0 ${w} ${h}" preserveAspectRatio="none" aria-hidden="true"> | |
| 1645 | + <polygon class="spark-fill" points="${area}"/><polyline class="spark-line" points="${line}"/></svg> | |
| 1646 | + <div class="spark-meta"><span>${esc(pts[0].day)}</span><span>${fmt.format(min)} → ${fmt.format(max)}</span><span>${esc(pts[pts.length-1].day)}</span></div></div>`; | |
| 1647 | +} | |
| 1648 | + | |
| 1574 | 1649 | async function renderCreator(id){ |
| 1575 | 1650 | app.innerHTML=header()+`<div class="container"><div class="spin"></div></div>`+footer()+tabbar("");bindNav(); |
| 1576 | 1651 | let c;try{c=await api("/api/creators/"+encodeURIComponent(id))} |
@@ -1581,16 +1656,35 @@ async function renderCreator(id){ | ||
| 1581 | 1656 | const niches=(c.niches||[]).map(n=>`<span class="mchip">${NICHE_LBL[n]||n}</span>`).join(""); |
| 1582 | 1657 | const P=(c.platforms||[]).slice().sort((x,y)=>(y.followers||0)-(x.followers||0)); |
| 1583 | 1658 | const g=computeAgg(c); |
| 1659 | + const ins=c.insights||{}; | |
| 1660 | + const banner=c.banner_url||P.map(a=>(a.metrics||{}).banner).find(Boolean); | |
| 1584 | 1661 | // bandeau de métriques CALCULÉES (multi-plateforme) |
| 1585 | 1662 | const kpi=(v,l)=>v==null?"":`<div class="hkpi"><b>${typeof v==="number"?fmt.format(v):esc(v)}</b><span>${l}</span></div>`; |
| 1663 | + const gkpi=(d,l)=>!d||!d.delta?"":`<div class="hkpi"><b class="${d.delta>=0?"growth-up":"growth-down"}">${d.delta>=0?"+":""}${fmt.format(d.delta)}</b><span>${l}</span></div>`; | |
| 1586 | 1664 | const heroKpis=[ |
| 1587 | 1665 | kpi(g.reach,"abonnés cumulés"), |
| 1666 | + gkpi(ins.growth_7d,"abonnés / 7 j"), | |
| 1667 | + gkpi(ins.growth_30d,"abonnés / 30 j"), | |
| 1588 | 1668 | kpi(g.nPlat,"plateformes"), |
| 1589 | − kpi(g.avgEng!=null?Math.round(g.avgEng*100)/100+" %":null,"engagement moyen"), | |
| 1590 | − kpi(g.maxCad!=null?g.maxCad:null,"pubs / semaine"), | |
| 1669 | + kpi(ins.avg_engagement_pct!=null?ins.avg_engagement_pct+" %":(g.avgEng!=null?Math.round(g.avgEng*100)/100+" %":null),"engagement moyen"), | |
| 1670 | + kpi(ins.publications_per_week!=null?ins.publications_per_week:g.maxCad,"pubs / semaine"), | |
| 1591 | 1671 | kpi(g.content||null,"contenus analysés"), |
| 1592 | 1672 | kpi(g.main?(PLAT[g.main.platform]||PLAT.autre).label:null,"plateforme nº 1"), |
| 1593 | 1673 | ].filter(Boolean).join(""); |
| 1674 | + // Ka Score /100 (insights serveur) + sparkline d'audience (snapshots) | |
| 1675 | + const KS_MAX={audience:40,engagement:25,presence:20,rythme:10,verification:5}; | |
| 1676 | + const KS_LBL={audience:"Audience",engagement:"Engagement",presence:"Multi-plateforme", | |
| 1677 | + rythme:"Rythme de pub.",verification:"Vérification"}; | |
| 1678 | + const ksParts=ins.ka_score_parts||{}; | |
| 1679 | + const ksHtml=ins.ka_score!=null?`<h3 class="cg-title">Ka Score</h3><div class="kascore"> | |
| 1680 | + <div class="kascore-num"><b>${ins.ka_score}</b><span>/ 100</span></div> | |
| 1681 | + <div class="kascore-parts">${Object.keys(KS_MAX).map(k=>`<div class="kspart"> | |
| 1682 | + <span class="kspart-lab">${KS_LBL[k]}</span> | |
| 1683 | + <span class="kspart-track"><span class="kspart-fill" style="width:${Math.min(100,Math.round((ksParts[k]||0)/KS_MAX[k]*100))}%"></span></span> | |
| 1684 | + <span class="kspart-val">${ksParts[k]!=null?ksParts[k]:0}/${KS_MAX[k]}</span></div>`).join("")}</div> | |
| 1685 | + </div>`:""; | |
| 1686 | + const hist=((ins.history||{}).total)||[]; | |
| 1687 | + const sparkHtml=hist.length>1?`<h3 class="cg-title">Évolution de l'audience</h3>${sparkSvg(hist)}`:""; | |
| 1594 | 1688 | // barres comparatives de portée par plateforme (vue d'ensemble) |
| 1595 | 1689 | const maxF=Math.max(1,...P.map(a=>a.followers||0)); |
| 1596 | 1690 | const bars=P.map(a=>{const p=PLAT[a.platform]||PLAT.autre;const w=Math.round((a.followers||0)/maxF*100); |
@@ -1605,8 +1699,10 @@ async function renderCreator(id){ | ||
| 1605 | 1699 | <span class="ptab-ico" style="color:${p.color}">${icon(a.platform)}</span>${p.label} |
| 1606 | 1700 | ${a.followers!=null?`<span class="ptab-n">${fmt.format(a.followers)}</span>`:""}</button>`;}).join(""); |
| 1607 | 1701 | const ovPanel=`<section class="ppanel on" id="ov-panel" data-panel="ov"> |
| 1702 | + ${ksHtml} | |
| 1608 | 1703 | <h3 class="cg-title">Portée par plateforme</h3> |
| 1609 | 1704 | <div class="obars">${bars||"<p class=klabel>Aucune plateforme.</p>"}</div> |
| 1705 | + ${sparkHtml} | |
| 1610 | 1706 | ${c.link_in_bio_url?`<a class="linkinbio" href="${esc(c.link_in_bio_url)}" target="_blank" rel="noopener">${icon("linktree")} Sa page de liens officielle ↗</a>`:""} |
| 1611 | 1707 | </section>`; |
| 1612 | 1708 | const panels=ovPanel+P.map((a,i)=>platformPanel(a,i,false)).join(""); |
@@ -1614,6 +1710,7 @@ async function renderCreator(id){ | ||
| 1614 | 1710 | app.innerHTML=header()+` |
| 1615 | 1711 | <div class="container cpage"> |
| 1616 | 1712 | <a class="backlink" href="/" data-nav>← Tous les créateurs</a> |
| 1713 | + ${banner?`<div class="cbanner"><img src="${esc(banner)}" alt="" referrerpolicy="no-referrer" onerror="this.parentElement.remove()"></div>`:""} | |
| 1617 | 1714 | <div class="cpage-head"> |
| 1618 | 1715 | <div class="cavatar">${esc(initials(c.display_name))}${photoImg(c)}</div> |
| 1619 | 1716 | <div style="flex:1;min-width:200px"><h1>${esc(c.display_name)}${verified?checkSvg:""}</h1> |
modified
frontend/src/index.template.html
+102 −5
@@ -359,6 +359,50 @@ select.f.active{background-color:var(--accent-soft);color:var(--ink);border-colo | ||
| 359 | 359 | .linkinbio svg{width:16px;height:16px;fill:currentColor} |
| 360 | 360 | .cpage-foot{margin-top:38px;font-size:12px;color:var(--ink-3); |
| 361 | 361 | border-top:1.5px solid var(--ink);padding-top:16px} |
| 362 | +/* bannière héro (banner_url capté par les acteurs Apify) */ | |
| 363 | +.cbanner{position:relative;border:1.5px solid var(--ink);border-bottom:none; | |
| 364 | + border-radius:var(--r-card) var(--r-card) 0 0;overflow:hidden;height:clamp(120px,22vw,240px); | |
| 365 | + background:var(--ink)} | |
| 366 | +.cbanner img{width:100%;height:100%;object-fit:cover;display:block} | |
| 367 | +.cbanner+.cpage-head{border-radius:0 0 var(--r-card) var(--r-card)} | |
| 368 | +/* Ka Score */ | |
| 369 | +.kascore{display:flex;gap:22px;align-items:center;flex-wrap:wrap;margin:6px 0 2px} | |
| 370 | +.kascore-num{flex:none;width:112px;height:112px;border-radius:50%;display:flex; | |
| 371 | + flex-direction:column;align-items:center;justify-content:center; | |
| 372 | + border:2px solid var(--ink);background:var(--accent);color:var(--on-accent); | |
| 373 | + box-shadow:var(--shadow-off-soft)} | |
| 374 | +.kascore-num b{font-family:var(--font-display);font-size:34px;font-weight:800;line-height:1} | |
| 375 | +.kascore-num span{font-family:var(--font-mono);font-size:9px;font-weight:700; | |
| 376 | + text-transform:uppercase;letter-spacing:.06em;margin-top:3px} | |
| 377 | +.kascore-parts{flex:1;min-width:230px;display:flex;flex-direction:column;gap:7px} | |
| 378 | +.kspart{display:flex;align-items:center;gap:10px} | |
| 379 | +.kspart-lab{width:110px;flex:none;font-family:var(--font-mono);font-size:10px; | |
| 380 | + font-weight:700;text-transform:uppercase;letter-spacing:.04em;color:var(--ink-3)} | |
| 381 | +.kspart-track{flex:1;height:9px;background:var(--surface-2);border:1px solid var(--line); | |
| 382 | + border-radius:5px;overflow:hidden} | |
| 383 | +.kspart-fill{display:block;height:100%;background:var(--green);border-radius:5px 0 0 5px;min-width:2px} | |
| 384 | +.kspart-val{width:52px;flex:none;text-align:right;font-family:var(--font-mono); | |
| 385 | + font-size:11px;font-weight:700;color:var(--ink)} | |
| 386 | +@media(max-width:640px){.kspart-lab{width:88px;font-size:9px}} | |
| 387 | +/* sparkline évolution d'audience */ | |
| 388 | +.spark-wrap{margin-top:8px} | |
| 389 | +.spark-svg{width:100%;height:88px;display:block} | |
| 390 | +.spark-line{fill:none;stroke:var(--green);stroke-width:2.5;stroke-linejoin:round;stroke-linecap:round} | |
| 391 | +.spark-fill{fill:var(--lime-soft);opacity:.7;stroke:none} | |
| 392 | +.spark-meta{display:flex;justify-content:space-between;font-family:var(--font-mono); | |
| 393 | + font-size:10px;color:var(--ink-3);margin-top:4px} | |
| 394 | +.growth-up{color:var(--green-deep);font-weight:700} | |
| 395 | +.growth-down{color:#b0201a;font-weight:700} | |
| 396 | +/* bloc EN DIRECT (miniature live Twitch/Kick) */ | |
| 397 | +.live-card{display:flex;gap:14px;align-items:stretch;margin:14px 0 4px; | |
| 398 | + border:1.5px solid #b0201a;border-radius:var(--r-card);overflow:hidden; | |
| 399 | + background:var(--surface);text-decoration:none;color:inherit} | |
| 400 | +.live-card img{width:200px;min-height:100%;object-fit:cover;flex:none;display:block} | |
| 401 | +.live-card-body{padding:12px 14px;display:flex;flex-direction:column;gap:5px;justify-content:center} | |
| 402 | +.live-card-tag{font-family:var(--font-mono);font-size:10px;font-weight:700;color:#b0201a} | |
| 403 | +.live-card-title{font-size:13.5px;font-weight:700;color:var(--ink);line-height:1.3} | |
| 404 | +.live-card-meta{font-family:var(--font-mono);font-size:10.5px;color:var(--ink-3)} | |
| 405 | +@media(max-width:560px){.live-card{flex-direction:column}.live-card img{width:100%;height:150px}} | |
| 362 | 406 | .cpage-foot a{color:var(--green);font-weight:600} |
| 363 | 407 | |
| 364 | 408 | /* ===== page stats ===== */ |
@@ -817,7 +861,11 @@ const _relTimeC=ts=>_relTime(ts); | ||
| 817 | 861 | window.thumbFail=el=>{const w=el.parentElement;if(w){w.classList.add("noimg"); |
| 818 | 862 | w.innerHTML=w.dataset.icon||"";}}; |
| 819 | 863 | function _mediaItems(a){const m=a.metrics||{}; |
| 820 | − return m.recent_posts||m.recent_videos||m.recent_tweets||[];} | |
| 864 | + const base=m.recent_posts||m.recent_videos||m.recent_tweets||[]; | |
| 865 | + if(base.length)return base; | |
| 866 | + // Snapchat : les previews de story/spotlight sont des listes d'URLs d'images | |
| 867 | + const prev=[...(m.story_previews||[]),...(m.spotlight_previews||[])]; | |
| 868 | + return prev.map(u=>({display_url:u,type:"story"}));} | |
| 821 | 869 | |
| 822 | 870 | /* grille de contenu récent, images/vidéos bien affichées */ |
| 823 | 871 | function contentGrid(a){ |
@@ -825,7 +873,7 @@ function contentGrid(a){ | ||
| 825 | 873 | if(!items.length)return `<div class="cg-empty">${icon(a.platform)} Aucun contenu récent capté pour ce compte.</div>`; |
| 826 | 874 | const ico=icon(a.platform); |
| 827 | 875 | const cards=items.slice(0,12).map(it=>{ |
| 828 | − const img=it.display_url||it.thumbnail_url||""; | |
| 876 | + const img=it.display_url||it.thumbnail_url||it.thumbnail||it.cover||(it.media_urls||[])[0]||""; | |
| 829 | 877 | const isVid=it.type==="video"||it.type==="reel"||!!it.duration||it.video_views!=null; |
| 830 | 878 | const cap=esc((it.caption||it.title||it.text||"").slice(0,140)); |
| 831 | 879 | const when=_relTimeC(it.timestamp||it.published_at||it.published||it.created_at); |
@@ -859,7 +907,8 @@ function platformKPIs(a){ | ||
| 859 | 907 | push(m.avg_views!=null?m.avg_views:m.avg_video_views,"vues / pub (moy.)"); |
| 860 | 908 | push(m.avg_comments,"comm. / pub (moy.)"); |
| 861 | 909 | push(m.avg_retweets,"RT / pub (moy.)"); |
| 862 | − push(m.posts_per_week!=null?m.posts_per_week:m.tweets_per_week,"pubs / semaine"); | |
| 910 | + push(m.posts_per_week!=null?m.posts_per_week:(m.tweets_per_week!=null?m.tweets_per_week:m.videos_per_week),"pubs / semaine"); | |
| 911 | + push(m.total_views,"vues totales"); | |
| 863 | 912 | push(m.posts,"publications"); |
| 864 | 913 | push(m.videos,"vidéos"); |
| 865 | 914 | push(m.likes,a.platform==="tiktok"?"j'aime cumulés":"j'aime"); |
@@ -883,6 +932,10 @@ function platformBadges(a){ | ||
| 883 | 932 | m.is_monthly?'<span class="abadge">Mensuel</span>':"", |
| 884 | 933 | m.has_story?'<span class="abadge">Story active</span>':"", |
| 885 | 934 | m.has_spotlight?'<span class="abadge">Spotlight</span>':"", |
| 935 | + m.is_nsfw?'<span class="abadge live">18+</span>':"", | |
| 936 | + m.creation_name?`<span class="abadge">${esc(String(m.creation_name).slice(0,60))}</span>`:"", | |
| 937 | + m.joined_date?`<span class="abadge">${esc(m.joined_date)}</span>`:"", | |
| 938 | + m.is_blue_verified?'<span class="abadge ok">✓ Bleu</span>':"", | |
| 886 | 939 | m.country?`<span class="abadge">${esc(m.country)}</span>`:"", |
| 887 | 940 | (m.recent_games||[]).slice(0,2).map(g=>`<span class="abadge">🎮 ${esc(g)}</span>`).join(""), |
| 888 | 941 | (m.top_hashtags||[]).slice(0,3).map(h=>`<span class="abadge">#${esc(h)}</span>`).join(""), |
@@ -906,6 +959,13 @@ function platformPanel(a,idx,active){ | ||
| 906 | 959 | <span class="conf ${confCls}" title="Score de rattachement">${conf}%</span> |
| 907 | 960 | </div> |
| 908 | 961 | ${badges?`<div class="abadge-row">${badges}</div>`:""} |
| 962 | + ${(a.metrics||{}).is_live_now&&(a.metrics||{}).live_thumbnail?` | |
| 963 | + <a class="live-card" href="${esc(a.url)}" target="_blank" rel="noopener"> | |
| 964 | + <img loading="lazy" src="${esc((a.metrics||{}).live_thumbnail)}" alt="" referrerpolicy="no-referrer" onerror="this.remove()"> | |
| 965 | + <span class="live-card-body"><span class="live-card-tag">● EN DIRECT</span> | |
| 966 | + ${(a.metrics||{}).live_title?`<span class="live-card-title">${esc((a.metrics||{}).live_title)}</span>`:""} | |
| 967 | + <span class="live-card-meta">${(a.metrics||{}).live_viewers!=null?fmt.format((a.metrics||{}).live_viewers)+" spectateurs":""}${(a.metrics||{}).live_game?" · "+esc((a.metrics||{}).live_game):((a.metrics||{}).live_category?" · "+esc((a.metrics||{}).live_category):"")}</span> | |
| 968 | + </span></a>`:""} | |
| 909 | 969 | <div class="kpi-grid">${kpis||'<div class="kpi"><b>—</b><span>pas encore de métriques</span></div>'}</div> |
| 910 | 970 | ${top&&top.url?`<a class="ppanel-top" href="${esc(top.url)}" target="_blank" rel="noopener">★ Top contenu — ${top.views!=null?fmt.format(top.views)+" vues":""}${top.likes!=null?" · "+fmt.format(top.likes)+" ❤":""} ↗</a>`:""} |
| 911 | 971 | <h3 class="cg-title">Contenu récent</h3> |
@@ -933,6 +993,21 @@ function computeAgg(c){ | ||
| 933 | 993 | nPlat:P.length,nVerif:P.filter(a=>a.verified).length,main}; |
| 934 | 994 | } |
| 935 | 995 | |
| 996 | +/* sparkline SVG (série quotidienne des snapshots — insights.history.total) */ | |
| 997 | +function sparkSvg(pts){ | |
| 998 | + if(!pts||pts.length<2)return""; | |
| 999 | + const w=600,h=88,pad=4; | |
| 1000 | + const vals=pts.map(p=>p.followers); | |
| 1001 | + const min=Math.min(...vals),max=Math.max(...vals),span=Math.max(1,max-min); | |
| 1002 | + const x=i=>pad+i*(w-2*pad)/(pts.length-1); | |
| 1003 | + const y=v=>h-pad-((v-min)/span)*(h-2*pad); | |
| 1004 | + const line=pts.map((p,i)=>`${x(i).toFixed(1)},${y(p.followers).toFixed(1)}`).join(" "); | |
| 1005 | + const area=`${pad},${h-pad} ${line} ${(w-pad).toFixed(1)},${h-pad}`; | |
| 1006 | + return `<div class="spark-wrap"><svg class="spark-svg" viewBox="0 0 ${w} ${h}" preserveAspectRatio="none" aria-hidden="true"> | |
| 1007 | + <polygon class="spark-fill" points="${area}"/><polyline class="spark-line" points="${line}"/></svg> | |
| 1008 | + <div class="spark-meta"><span>${esc(pts[0].day)}</span><span>${fmt.format(min)} → ${fmt.format(max)}</span><span>${esc(pts[pts.length-1].day)}</span></div></div>`; | |
| 1009 | +} | |
| 1010 | + | |
| 936 | 1011 | async function renderCreator(id){ |
| 937 | 1012 | app.innerHTML=header()+`<div class="container"><div class="spin"></div></div>`+footer()+tabbar("");bindNav(); |
| 938 | 1013 | let c;try{c=await api("/api/creators/"+encodeURIComponent(id))} |
@@ -943,16 +1018,35 @@ async function renderCreator(id){ | ||
| 943 | 1018 | const niches=(c.niches||[]).map(n=>`<span class="mchip">${NICHE_LBL[n]||n}</span>`).join(""); |
| 944 | 1019 | const P=(c.platforms||[]).slice().sort((x,y)=>(y.followers||0)-(x.followers||0)); |
| 945 | 1020 | const g=computeAgg(c); |
| 1021 | + const ins=c.insights||{}; | |
| 1022 | + const banner=c.banner_url||P.map(a=>(a.metrics||{}).banner).find(Boolean); | |
| 946 | 1023 | // bandeau de métriques CALCULÉES (multi-plateforme) |
| 947 | 1024 | const kpi=(v,l)=>v==null?"":`<div class="hkpi"><b>${typeof v==="number"?fmt.format(v):esc(v)}</b><span>${l}</span></div>`; |
| 1025 | + const gkpi=(d,l)=>!d||!d.delta?"":`<div class="hkpi"><b class="${d.delta>=0?"growth-up":"growth-down"}">${d.delta>=0?"+":""}${fmt.format(d.delta)}</b><span>${l}</span></div>`; | |
| 948 | 1026 | const heroKpis=[ |
| 949 | 1027 | kpi(g.reach,"abonnés cumulés"), |
| 1028 | + gkpi(ins.growth_7d,"abonnés / 7 j"), | |
| 1029 | + gkpi(ins.growth_30d,"abonnés / 30 j"), | |
| 950 | 1030 | kpi(g.nPlat,"plateformes"), |
| 951 | − kpi(g.avgEng!=null?Math.round(g.avgEng*100)/100+" %":null,"engagement moyen"), | |
| 952 | − kpi(g.maxCad!=null?g.maxCad:null,"pubs / semaine"), | |
| 1031 | + kpi(ins.avg_engagement_pct!=null?ins.avg_engagement_pct+" %":(g.avgEng!=null?Math.round(g.avgEng*100)/100+" %":null),"engagement moyen"), | |
| 1032 | + kpi(ins.publications_per_week!=null?ins.publications_per_week:g.maxCad,"pubs / semaine"), | |
| 953 | 1033 | kpi(g.content||null,"contenus analysés"), |
| 954 | 1034 | kpi(g.main?(PLAT[g.main.platform]||PLAT.autre).label:null,"plateforme nº 1"), |
| 955 | 1035 | ].filter(Boolean).join(""); |
| 1036 | + // Ka Score /100 (insights serveur) + sparkline d'audience (snapshots) | |
| 1037 | + const KS_MAX={audience:40,engagement:25,presence:20,rythme:10,verification:5}; | |
| 1038 | + const KS_LBL={audience:"Audience",engagement:"Engagement",presence:"Multi-plateforme", | |
| 1039 | + rythme:"Rythme de pub.",verification:"Vérification"}; | |
| 1040 | + const ksParts=ins.ka_score_parts||{}; | |
| 1041 | + const ksHtml=ins.ka_score!=null?`<h3 class="cg-title">Ka Score</h3><div class="kascore"> | |
| 1042 | + <div class="kascore-num"><b>${ins.ka_score}</b><span>/ 100</span></div> | |
| 1043 | + <div class="kascore-parts">${Object.keys(KS_MAX).map(k=>`<div class="kspart"> | |
| 1044 | + <span class="kspart-lab">${KS_LBL[k]}</span> | |
| 1045 | + <span class="kspart-track"><span class="kspart-fill" style="width:${Math.min(100,Math.round((ksParts[k]||0)/KS_MAX[k]*100))}%"></span></span> | |
| 1046 | + <span class="kspart-val">${ksParts[k]!=null?ksParts[k]:0}/${KS_MAX[k]}</span></div>`).join("")}</div> | |
| 1047 | + </div>`:""; | |
| 1048 | + const hist=((ins.history||{}).total)||[]; | |
| 1049 | + const sparkHtml=hist.length>1?`<h3 class="cg-title">Évolution de l'audience</h3>${sparkSvg(hist)}`:""; | |
| 956 | 1050 | // barres comparatives de portée par plateforme (vue d'ensemble) |
| 957 | 1051 | const maxF=Math.max(1,...P.map(a=>a.followers||0)); |
| 958 | 1052 | const bars=P.map(a=>{const p=PLAT[a.platform]||PLAT.autre;const w=Math.round((a.followers||0)/maxF*100); |
@@ -967,8 +1061,10 @@ async function renderCreator(id){ | ||
| 967 | 1061 | <span class="ptab-ico" style="color:${p.color}">${icon(a.platform)}</span>${p.label} |
| 968 | 1062 | ${a.followers!=null?`<span class="ptab-n">${fmt.format(a.followers)}</span>`:""}</button>`;}).join(""); |
| 969 | 1063 | const ovPanel=`<section class="ppanel on" id="ov-panel" data-panel="ov"> |
| 1064 | + ${ksHtml} | |
| 970 | 1065 | <h3 class="cg-title">Portée par plateforme</h3> |
| 971 | 1066 | <div class="obars">${bars||"<p class=klabel>Aucune plateforme.</p>"}</div> |
| 1067 | + ${sparkHtml} | |
| 972 | 1068 | ${c.link_in_bio_url?`<a class="linkinbio" href="${esc(c.link_in_bio_url)}" target="_blank" rel="noopener">${icon("linktree")} Sa page de liens officielle ↗</a>`:""} |
| 973 | 1069 | </section>`; |
| 974 | 1070 | const panels=ovPanel+P.map((a,i)=>platformPanel(a,i,false)).join(""); |
@@ -976,6 +1072,7 @@ async function renderCreator(id){ | ||
| 976 | 1072 | app.innerHTML=header()+` |
| 977 | 1073 | <div class="container cpage"> |
| 978 | 1074 | <a class="backlink" href="/" data-nav>← Tous les créateurs</a> |
| 1075 | + ${banner?`<div class="cbanner"><img src="${esc(banner)}" alt="" referrerpolicy="no-referrer" onerror="this.parentElement.remove()"></div>`:""} | |
| 979 | 1076 | <div class="cpage-head"> |
| 980 | 1077 | <div class="cavatar">${esc(initials(c.display_name))}${photoImg(c)}</div> |
| 981 | 1078 | <div style="flex:1;min-width:200px"><h1>${esc(c.display_name)}${verified?checkSvg:""}</h1> |
| 982 | 1079 | |