Fiches légendaires : upgrade majeur des 13 acteurs Apify (bannières, covers TikTok, miniatures YouTube/Twitch, media X, previews Snapchat, /about YouTube, socialMedias Twitch GQL, liens sociaux Patreon) + pipeline étendu (banner_url, metric_maps, cross-links), snapshots quotidiens d'audience, insights/Ka Score /100 par créateur (croissance 7j/30j, engagement pondéré, rythme), fiche frontend enrichie (bannière héro, Ka Score, sparkline, bloc live) — acteurs versionnés dans actors/, 8 poussés et validés par runs de test
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
98 changed files +4,340 −13
added
actors/ka-discord/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-discord) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-discord/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-discord", | |
| 4 | + "title": "KA Discord — serveurs via invitation", | |
| 5 | + "description": "Serveurs Discord via l'API publique d'invitation : membres approximatifs, en ligne, boosts, nom/description. Code d'invit sensible à la casse.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-discord/.actor/input_schema.json
+38 −0
@@ -0,0 +1,38 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-discord input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles / codes", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles (patreon) ou codes d'invitation (discord).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [] | |
| 12 | + }, | |
| 13 | + "requestDelay": { | |
| 14 | + "title": "Délai (s)", | |
| 15 | + "type": "integer", | |
| 16 | + "default": 1, | |
| 17 | + "minimum": 0, | |
| 18 | + "description": "Pause entre requêtes." | |
| 19 | + }, | |
| 20 | + "concurrency": { | |
| 21 | + "title": "Concurrence", | |
| 22 | + "type": "integer", | |
| 23 | + "default": 5, | |
| 24 | + "minimum": 1, | |
| 25 | + "maximum": 10, | |
| 26 | + "description": "Requêtes simultanées." | |
| 27 | + }, | |
| 28 | + "proxyConfiguration": { | |
| 29 | + "title": "Proxy", | |
| 30 | + "type": "object", | |
| 31 | + "editor": "proxy", | |
| 32 | + "description": "Proxy Apify.", | |
| 33 | + "prefill": { | |
| 34 | + "useApifyProxy": true | |
| 35 | + } | |
| 36 | + } | |
| 37 | + } | |
| 38 | +} | |
| \ No newline at end of file | ||
added
actors/ka-discord/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-discord) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-discord/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-discord) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-discord/src/main.py
+87 −0
@@ -0,0 +1,87 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-discord) | |
| 4 | +# Desc: Serveurs Discord via l'API PUBLIQUE d'invitation | |
| 5 | +# /api/v9/invites/{code}?with_counts=true : membres approximatifs, | |
| 6 | +# présents en ligne, nom/description/boosts du serveur. Le CODE d'invit | |
| 7 | +# est SENSIBLE À LA CASSE → il vient de l'URL, pas du handle minusculé. | |
| 8 | +# ============================================================================== | |
| 9 | +from __future__ import annotations | |
| 10 | + | |
| 11 | +import asyncio | |
| 12 | +import json | |
| 13 | + | |
| 14 | +from apify import Actor | |
| 15 | + | |
| 16 | +from .net import Fetcher | |
| 17 | + | |
| 18 | +INVITE_API = ("https://discord.com/api/v9/invites/{code}" | |
| 19 | + "?with_counts=true&with_expiration=true") | |
| 20 | + | |
| 21 | + | |
| 22 | +async def main() -> None: | |
| 23 | + async with Actor: | |
| 24 | + inp = await Actor.get_input() or {} | |
| 25 | + # ici « usernames » = codes d'invitation (casse préservée par l'appelant) | |
| 26 | + codes = [c.strip() for c in (inp.get("usernames") or []) if c.strip()] | |
| 27 | + proxy = await Actor.create_proxy_configuration( | |
| 28 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 29 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 30 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 6)) | |
| 31 | + | |
| 32 | + async def one(code: str) -> None: | |
| 33 | + async with sem: | |
| 34 | + miss = {"kind": "profile", "platform": "discord", | |
| 35 | + "username": code, "found": False} | |
| 36 | + try: | |
| 37 | + resp = await fetcher.get( | |
| 38 | + INVITE_API.format(code=code), session_id=code[:12], | |
| 39 | + headers={"Accept": "application/json"}) | |
| 40 | + except Exception as exc: | |
| 41 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 42 | + return | |
| 43 | + if resp.status_code == 404: | |
| 44 | + await Actor.push_data({**miss, "error": "invite_invalide"}) | |
| 45 | + return | |
| 46 | + if resp.status_code != 200: | |
| 47 | + await Actor.push_data( | |
| 48 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 49 | + return | |
| 50 | + try: | |
| 51 | + d = json.loads(resp.text) | |
| 52 | + except Exception: | |
| 53 | + await Actor.push_data({**miss, "error": "bad_json"}) | |
| 54 | + return | |
| 55 | + guild = d.get("guild") or {} | |
| 56 | + icon = guild.get("icon") | |
| 57 | + gid = guild.get("id") | |
| 58 | + banner_hash = guild.get("banner") | |
| 59 | + splash_hash = guild.get("splash") | |
| 60 | + await Actor.push_data({ | |
| 61 | + "kind": "profile", | |
| 62 | + "platform": "discord", | |
| 63 | + "found": True, | |
| 64 | + "username": code, | |
| 65 | + "guild_id": gid, | |
| 66 | + "full_name": guild.get("name"), | |
| 67 | + "biography": guild.get("description"), | |
| 68 | + "followers": d.get("approximate_member_count"), # membres | |
| 69 | + "members": d.get("approximate_member_count"), | |
| 70 | + "online": d.get("approximate_presence_count"), | |
| 71 | + "boosts": guild.get("premium_subscription_count"), | |
| 72 | + "verified": ("VERIFIED" in (guild.get("features") or [])), | |
| 73 | + "partnered": ("PARTNERED" in (guild.get("features") or [])), | |
| 74 | + "channel": (d.get("channel") or {}).get("name"), | |
| 75 | + "avatar": (f"https://cdn.discordapp.com/icons/{gid}/{icon}.png" | |
| 76 | + if gid and icon else None), | |
| 77 | + "banner": (f"https://cdn.discordapp.com/banners/{gid}/" | |
| 78 | + f"{banner_hash}.png?size=1024" | |
| 79 | + if gid and banner_hash else None), | |
| 80 | + "splash": (f"https://cdn.discordapp.com/splashes/{gid}/" | |
| 81 | + f"{splash_hash}.jpg?size=1024" | |
| 82 | + if gid and splash_hash else None), | |
| 83 | + "vanity_url_code": guild.get("vanity_url_code"), | |
| 84 | + }) | |
| 85 | + | |
| 86 | + await asyncio.gather(*[one(c) for c in codes]) | |
| 87 | + Actor.log.info(f"terminé : {len(codes)} invitations") | |
added
actors/ka-discord/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-facebook/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-facebook) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-facebook/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-facebook", | |
| 4 | + "title": "KA Facebook — pages publiques", | |
| 5 | + "description": "Métadonnées de pages Facebook publiques (nom, abonnés/mentions J'aime, catégorie, avatar) extraites du HTML public via proxy résidentiel + empreinte Chrome. Expérimental : Facebook mure agressivement.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
added
actors/ka-facebook/.actor/input_schema.json
+43 −0
@@ -0,0 +1,43 @@ | ||
| 1 | +{ | |
| 2 | + "title": "KA Facebook input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Pages à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Slugs de pages Facebook (facebook.com/<slug>) ou URLs complètes.", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "facebook" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "requestDelay": { | |
| 16 | + "title": "Délai entre requêtes (s)", | |
| 17 | + "type": "integer", | |
| 18 | + "default": 2, | |
| 19 | + "minimum": 0, | |
| 20 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 21 | + }, | |
| 22 | + "concurrency": { | |
| 23 | + "title": "Requêtes simultanées", | |
| 24 | + "type": "integer", | |
| 25 | + "default": 2, | |
| 26 | + "minimum": 1, | |
| 27 | + "maximum": 6, | |
| 28 | + "description": "Nombre de requêtes menées en parallèle." | |
| 29 | + }, | |
| 30 | + "proxyConfiguration": { | |
| 31 | + "title": "Proxy", | |
| 32 | + "type": "object", | |
| 33 | + "editor": "proxy", | |
| 34 | + "description": "Résidentiel obligatoire (Facebook rejette les IP datacenter).", | |
| 35 | + "prefill": { | |
| 36 | + "useApifyProxy": true, | |
| 37 | + "apifyProxyGroups": [ | |
| 38 | + "RESIDENTIAL" | |
| 39 | + ] | |
| 40 | + } | |
| 41 | + } | |
| 42 | + } | |
| 43 | +} | |
| \ No newline at end of file | ||
added
actors/ka-facebook/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-facebook) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-facebook/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-facebook) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-facebook/src/main.py
+121 −0
@@ -0,0 +1,121 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-facebook) | |
| 4 | +# Desc: Pages Facebook publiques — extraction best-effort du HTML servi aux | |
| 5 | +# navigateurs non connectés : compteur d'abonnés (JSON embarqué | |
| 6 | +# follower_count / texte « N followers/abonnés »), nom (og:title), | |
| 7 | +# avatar (og:image), catégorie. EXPÉRIMENTAL : FB mure agressivement ; | |
| 8 | +# un résultat found=False n'est pas une erreur du pipeline. | |
| 9 | +# ============================================================================== | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import asyncio | |
| 13 | +import html as htmllib | |
| 14 | +import re | |
| 15 | + | |
| 16 | +from apify import Actor | |
| 17 | + | |
| 18 | +from .net import Fetcher | |
| 19 | + | |
| 20 | +PAGE_URL = "https://www.facebook.com/{u}" | |
| 21 | + | |
| 22 | +_OG_RE = { | |
| 23 | + "title": re.compile(r'<meta property="og:title" content="([^"]*)"'), | |
| 24 | + "image": re.compile(r'<meta property="og:image" content="([^"]*)"'), | |
| 25 | + "description": re.compile( | |
| 26 | + r'<meta (?:property="og:description"|name="description") ' | |
| 27 | + r'content="([^"]*)"'), | |
| 28 | +} | |
| 29 | +_FOLLOWER_RES = ( | |
| 30 | + re.compile(r'"follower_count"\s*:\s*(\d+)'), | |
| 31 | + re.compile(r'"global_likers_count"\s*:\s*(\d+)'), | |
| 32 | +) | |
| 33 | +# texte « 1,2 M followers » / « 12 k abonnés » / « 4 016 971 mentions J’aime » | |
| 34 | +_TEXT_COUNT_RE = re.compile( | |
| 35 | + r'([\d][\d\s .,]*)\s*([KkMm]?)\s*' | |
| 36 | + r'(?:followers|abonnés|mentions\s+J’aime|J’aime|likes)', | |
| 37 | + re.I) | |
| 38 | +_CATEGORY_RE = re.compile(r'"category_name"\s*:\s*"([^"]+)"') | |
| 39 | +_VERIFIED_RE = re.compile(r'"is_verified"\s*:\s*(true|false)') | |
| 40 | + | |
| 41 | + | |
| 42 | +def parse_text_count(text: str) -> int | None: | |
| 43 | + m = _TEXT_COUNT_RE.search(text or "") | |
| 44 | + if not m: | |
| 45 | + return None | |
| 46 | + num = m.group(1).replace(" ", "").replace(" ", "").replace(",", ".") | |
| 47 | + try: | |
| 48 | + val = float(num) | |
| 49 | + except ValueError: | |
| 50 | + return None | |
| 51 | + unit = (m.group(2) or "").lower() | |
| 52 | + if unit == "k": | |
| 53 | + val *= 1_000 | |
| 54 | + elif unit == "m": | |
| 55 | + val *= 1_000_000 | |
| 56 | + return int(val) | |
| 57 | + | |
| 58 | + | |
| 59 | +async def main() -> None: | |
| 60 | + async with Actor: | |
| 61 | + inp = await Actor.get_input() or {} | |
| 62 | + usernames = [] | |
| 63 | + for raw in (inp.get("usernames") or []): | |
| 64 | + raw = (raw or "").strip() | |
| 65 | + if not raw: | |
| 66 | + continue | |
| 67 | + raw = re.sub(r"^https?://(www\.|m\.)?facebook\.com/", "", raw) | |
| 68 | + usernames.append(raw.strip("/").split("?")[0]) | |
| 69 | + proxy = await Actor.create_proxy_configuration( | |
| 70 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 71 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 2)) | |
| 72 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 2)) | |
| 73 | + | |
| 74 | + async def one(u: str) -> None: | |
| 75 | + async with sem: | |
| 76 | + miss = {"kind": "profile", "platform": "facebook", | |
| 77 | + "username": u, "found": False} | |
| 78 | + try: | |
| 79 | + resp = await fetcher.get(PAGE_URL.format(u=u), | |
| 80 | + session_id=u) | |
| 81 | + except Exception as exc: | |
| 82 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 83 | + return | |
| 84 | + text = resp.text or "" | |
| 85 | + if resp.status_code != 200 or len(text) < 5000: | |
| 86 | + await Actor.push_data( | |
| 87 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 88 | + return | |
| 89 | + followers = None | |
| 90 | + for rx in _FOLLOWER_RES: | |
| 91 | + m = rx.search(text) | |
| 92 | + if m: | |
| 93 | + followers = int(m.group(1)) | |
| 94 | + break | |
| 95 | + og = {k: htmllib.unescape(m.group(1)) if (m := rx.search(text)) | |
| 96 | + else None for k, rx in _OG_RE.items()} | |
| 97 | + if followers is None: | |
| 98 | + followers = parse_text_count(og.get("description") or "") | |
| 99 | + if followers is None: # texte « N followers/abonnés » du corps | |
| 100 | + followers = parse_text_count(text) | |
| 101 | + if followers is None and not og.get("title"): | |
| 102 | + await Actor.push_data({**miss, "error": "login_wall"}) | |
| 103 | + return | |
| 104 | + cat = _CATEGORY_RE.search(text) | |
| 105 | + ver = _VERIFIED_RE.search(text) | |
| 106 | + await Actor.push_data({ | |
| 107 | + "kind": "profile", | |
| 108 | + "platform": "facebook", | |
| 109 | + "found": True, | |
| 110 | + "username": u, | |
| 111 | + "full_name": og.get("title"), | |
| 112 | + "biography": og.get("description"), | |
| 113 | + "followers": followers, | |
| 114 | + "category": cat.group(1) if cat else None, | |
| 115 | + "is_verified": (ver.group(1) == "true") if ver else None, | |
| 116 | + "avatar": og.get("image"), | |
| 117 | + "og_description": og.get("description"), | |
| 118 | + }) | |
| 119 | + | |
| 120 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 121 | + Actor.log.info(f"terminé : {len(usernames)} pages") | |
added
actors/ka-facebook/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-fansly/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-fansly) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-fansly/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-fansly", | |
| 4 | + "title": "KA Fansly — profils", | |
| 5 | + "description": "Profils Fansly via l'API publique (abonnés, bio, badge, compteurs de médias, avatar) — lots de 20 handles.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-fansly/.actor/input_schema.json
+38 −0
@@ -0,0 +1,38 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-fansly input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles/slugs à traiter (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [] | |
| 12 | + }, | |
| 13 | + "requestDelay": { | |
| 14 | + "title": "Délai entre requêtes (s)", | |
| 15 | + "type": "integer", | |
| 16 | + "default": 1, | |
| 17 | + "minimum": 0, | |
| 18 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 19 | + }, | |
| 20 | + "concurrency": { | |
| 21 | + "title": "Requêtes simultanées", | |
| 22 | + "type": "integer", | |
| 23 | + "default": 3, | |
| 24 | + "minimum": 1, | |
| 25 | + "maximum": 10, | |
| 26 | + "description": "Nombre de requêtes menées en parallèle." | |
| 27 | + }, | |
| 28 | + "proxyConfiguration": { | |
| 29 | + "title": "Proxy", | |
| 30 | + "type": "object", | |
| 31 | + "editor": "proxy", | |
| 32 | + "description": "Configuration du proxy Apify.", | |
| 33 | + "prefill": { | |
| 34 | + "useApifyProxy": true | |
| 35 | + } | |
| 36 | + } | |
| 37 | + } | |
| 38 | +} | |
| \ No newline at end of file | ||
added
actors/ka-fansly/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-fansly) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-fansly/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-fansly) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-fansly/src/main.py
+93 −0
@@ -0,0 +1,93 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-fansly) | |
| 4 | +# Desc: Profils Fansly via l'API publique apiv3.fansly.com/api/v1/account | |
| 5 | +# (?usernames=) : abonnés (followCount), bio, badge, compteurs de | |
| 6 | +# médias (images/vidéos), avatar/bannière. Lots de 20 handles/requête. | |
| 7 | +# ============================================================================== | |
| 8 | +from __future__ import annotations | |
| 9 | + | |
| 10 | +import asyncio | |
| 11 | +import json | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | + | |
| 15 | +from .net import Fetcher | |
| 16 | + | |
| 17 | +API_URL = ("https://apiv3.fansly.com/api/v1/account" | |
| 18 | + "?usernames={us}&ngsw-bypass=true") | |
| 19 | + | |
| 20 | + | |
| 21 | +def _loc(media: dict | None) -> str | None: | |
| 22 | + """Objet média Fansly → URL de la première variante disponible.""" | |
| 23 | + if not isinstance(media, dict): | |
| 24 | + return None | |
| 25 | + for variant in ([media] + (media.get("variants") or [])): | |
| 26 | + for loc in (variant.get("locations") or []): | |
| 27 | + if loc.get("location"): | |
| 28 | + return loc["location"] | |
| 29 | + return None | |
| 30 | + | |
| 31 | + | |
| 32 | +async def main() -> None: | |
| 33 | + async with Actor: | |
| 34 | + inp = await Actor.get_input() or {} | |
| 35 | + usernames = [u.strip().lstrip("@").lower() | |
| 36 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 37 | + proxy = await Actor.create_proxy_configuration( | |
| 38 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 39 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 40 | + | |
| 41 | + found: set[str] = set() | |
| 42 | + for i in range(0, len(usernames), 20): | |
| 43 | + batch = usernames[i:i + 20] | |
| 44 | + try: | |
| 45 | + resp = await fetcher.get(API_URL.format(us=",".join(batch)), | |
| 46 | + session_id=f"b{i}", | |
| 47 | + headers={"Accept": | |
| 48 | + "application/json"}) | |
| 49 | + data = (json.loads(resp.text) | |
| 50 | + if resp.status_code == 200 else {}) | |
| 51 | + except Exception as exc: | |
| 52 | + for u in batch: | |
| 53 | + await Actor.push_data( | |
| 54 | + {"kind": "profile", "platform": "fansly", | |
| 55 | + "username": u, "found": False, | |
| 56 | + "error": str(exc)[:200]}) | |
| 57 | + continue | |
| 58 | + for acc in (data.get("response") or []): | |
| 59 | + u = str(acc.get("username") or "").lower() | |
| 60 | + if not u: | |
| 61 | + continue | |
| 62 | + found.add(u) | |
| 63 | + stats = acc.get("timelineStats") or {} | |
| 64 | + await Actor.push_data({ | |
| 65 | + "kind": "profile", | |
| 66 | + "platform": "fansly", | |
| 67 | + "found": True, | |
| 68 | + "id": acc.get("id"), | |
| 69 | + "username": u, | |
| 70 | + "full_name": acc.get("displayName") or u, | |
| 71 | + "biography": (acc.get("about") or "")[:1000], | |
| 72 | + "followers": acc.get("followCount"), | |
| 73 | + "following": acc.get("followingCount"), | |
| 74 | + "is_verified": bool((acc.get("statusInfo") or {}) | |
| 75 | + .get("verified") | |
| 76 | + or acc.get("verified")), | |
| 77 | + "posts_count": acc.get("postCount") | |
| 78 | + or stats.get("postCount"), | |
| 79 | + "images_count": stats.get("imageCount"), | |
| 80 | + "videos_count": stats.get("videoCount"), | |
| 81 | + "likes": acc.get("accountMediaLikes"), | |
| 82 | + "avatar": _loc(acc.get("avatar")), | |
| 83 | + "banner": _loc(acc.get("banner")), | |
| 84 | + "location": acc.get("location"), | |
| 85 | + }) | |
| 86 | + for u in batch: | |
| 87 | + if u not in found: | |
| 88 | + await Actor.push_data( | |
| 89 | + {"kind": "profile", "platform": "fansly", | |
| 90 | + "username": u, "found": False, | |
| 91 | + "error": f"not_found_{resp.status_code}"}) | |
| 92 | + Actor.log.info(f"terminé : {len(usernames)} profils, " | |
| 93 | + f"{len(found)} trouvés") | |
added
actors/ka-fansly/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-instagram/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-instagram) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-instagram/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-instagram", | |
| 4 | + "title": "KA Instagram — profils détaillés + recherche", | |
| 5 | + "description": "Profils Instagram publics ultra détaillés (abonnés, bio, liens, 12 derniers posts avec likes/commentaires/vues, engagement, profils reliés) + recherche d'utilisateurs par mot-clé. Endpoints web publics via proxy résidentiel.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
added
actors/ka-instagram/.actor/input_schema.json
+49 −0
@@ -0,0 +1,49 @@ | ||
| 1 | +{ | |
| 2 | + "title": "KA Instagram input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles Instagram (sans @) dont on veut le profil détaillé.", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "instagram" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "queries": { | |
| 16 | + "title": "Recherches (découverte)", | |
| 17 | + "type": "array", | |
| 18 | + "description": "Mots-clés de recherche d'utilisateurs (topsearch).", | |
| 19 | + "editor": "stringList" | |
| 20 | + }, | |
| 21 | + "requestDelay": { | |
| 22 | + "title": "Délai entre requêtes (s)", | |
| 23 | + "type": "integer", | |
| 24 | + "default": 1, | |
| 25 | + "minimum": 0, | |
| 26 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 27 | + }, | |
| 28 | + "concurrency": { | |
| 29 | + "title": "Requêtes simultanées", | |
| 30 | + "type": "integer", | |
| 31 | + "default": 3, | |
| 32 | + "minimum": 1, | |
| 33 | + "maximum": 10, | |
| 34 | + "description": "Nombre de requêtes menées en parallèle." | |
| 35 | + }, | |
| 36 | + "proxyConfiguration": { | |
| 37 | + "title": "Proxy", | |
| 38 | + "type": "object", | |
| 39 | + "editor": "proxy", | |
| 40 | + "description": "Résidentiel recommandé (Instagram bloque les IP datacenter).", | |
| 41 | + "prefill": { | |
| 42 | + "useApifyProxy": true, | |
| 43 | + "apifyProxyGroups": [ | |
| 44 | + "RESIDENTIAL" | |
| 45 | + ] | |
| 46 | + } | |
| 47 | + } | |
| 48 | + } | |
| 49 | +} | |
| \ No newline at end of file | ||
added
actors/ka-instagram/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-instagram) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-instagram/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-instagram) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-instagram/src/main.py
+243 −0
@@ -0,0 +1,243 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-instagram) | |
| 4 | +# Desc: Profils Instagram publics ULTRA DÉTAILLÉS via l'endpoint web officiel | |
| 5 | +# web_profile_info (JSON du web public) : identité complète, compteurs, | |
| 6 | +# 12 derniers posts (likes/commentaires/vues/légende/musique/lieu/ | |
| 7 | +# co-auteurs/épinglés), agrégats d'engagement et de cadence, profils | |
| 8 | +# reliés — + recherche d'utilisateurs (topsearch) en découverte. | |
| 9 | +# ============================================================================== | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import asyncio | |
| 13 | +import json | |
| 14 | +import urllib.parse | |
| 15 | +from datetime import datetime, timezone | |
| 16 | + | |
| 17 | +from apify import Actor | |
| 18 | + | |
| 19 | +from .net import Fetcher | |
| 20 | + | |
| 21 | +PROFILE_API = ("https://i.instagram.com/api/v1/users/web_profile_info/" | |
| 22 | + "?username={u}") | |
| 23 | +SEARCH_API = ("https://www.instagram.com/web/search/topsearch/" | |
| 24 | + "?context=blended&query={q}&count=30") | |
| 25 | +SEARCH_API_ALT = ("https://i.instagram.com/api/v1/web/search/topsearch/" | |
| 26 | + "?context=blended&query={q}&count=30") | |
| 27 | +HDRS = {"x-ig-app-id": "936619743392459", "Accept": "application/json"} | |
| 28 | + | |
| 29 | + | |
| 30 | +def _iso(ts) -> str | None: | |
| 31 | + try: | |
| 32 | + return datetime.fromtimestamp(int(ts), tz=timezone.utc) \ | |
| 33 | + .strftime("%Y-%m-%dT%H:%M:%SZ") | |
| 34 | + except (TypeError, ValueError, OSError): | |
| 35 | + return None | |
| 36 | + | |
| 37 | + | |
| 38 | +def _avg(vals: list) -> int | None: | |
| 39 | + vals = [v for v in vals if isinstance(v, (int, float))] | |
| 40 | + return round(sum(vals) / len(vals)) if vals else None | |
| 41 | + | |
| 42 | + | |
| 43 | +def parse_post(node: dict) -> dict: | |
| 44 | + """Nœud média du fil → post détaillé (métriques, musique, lieu, épinglé).""" | |
| 45 | + cap = (node.get("edge_media_to_caption") or {}).get("edges") or [] | |
| 46 | + music = node.get("clips_music_attribution_info") or {} | |
| 47 | + dims = node.get("dimensions") or {} | |
| 48 | + return { | |
| 49 | + "id": node.get("id"), | |
| 50 | + "shortcode": node.get("shortcode"), | |
| 51 | + "url": f"https://www.instagram.com/p/{node.get('shortcode')}/", | |
| 52 | + "type": ("reel" if node.get("product_type") == "clips" | |
| 53 | + else "video" if node.get("is_video") else | |
| 54 | + "carousel" if node.get("edge_sidecar_to_children") | |
| 55 | + else "image"), | |
| 56 | + "caption": ((cap[0].get("node") or {}).get("text", "") | |
| 57 | + if cap else "")[:500], | |
| 58 | + "likes": (node.get("edge_liked_by") or {}).get("count"), | |
| 59 | + "comments": (node.get("edge_media_to_comment") or {}).get("count"), | |
| 60 | + "video_views": node.get("video_view_count"), | |
| 61 | + "video_duration_s": node.get("video_duration"), | |
| 62 | + "timestamp": node.get("taken_at_timestamp"), | |
| 63 | + "posted_at": _iso(node.get("taken_at_timestamp")), | |
| 64 | + "display_url": node.get("display_url"), | |
| 65 | + "thumbnail_url": node.get("thumbnail_src"), | |
| 66 | + "width": dims.get("width"), | |
| 67 | + "height": dims.get("height"), | |
| 68 | + "is_pinned": bool(node.get("pinned_for_users")), | |
| 69 | + "comments_disabled": node.get("comments_disabled"), | |
| 70 | + "location": (node.get("location") or {}).get("name"), | |
| 71 | + "coauthors": [c.get("username") | |
| 72 | + for c in (node.get("coauthor_producers") or []) | |
| 73 | + if c.get("username")], | |
| 74 | + "music": ({"artist": music.get("artist_name"), | |
| 75 | + "song": music.get("song_name"), | |
| 76 | + "uses_original_audio": music.get("uses_original_audio")} | |
| 77 | + if music.get("song_name") or music.get("artist_name") | |
| 78 | + else None), | |
| 79 | + "accessibility_caption": (node.get("accessibility_caption") | |
| 80 | + or "")[:200] or None, | |
| 81 | + "tagged_users": [((t.get("node") or {}).get("user") or {}) | |
| 82 | + .get("username") | |
| 83 | + for t in ((node.get("edge_media_to_tagged_user") | |
| 84 | + or {}).get("edges") or [])][:10], | |
| 85 | + } | |
| 86 | + | |
| 87 | + | |
| 88 | +def parse_user(user: dict) -> dict: | |
| 89 | + """Objet user web_profile_info → enregistrement riche standardisé.""" | |
| 90 | + posts = [parse_post((e.get("node") or {})) | |
| 91 | + for e in ((user.get("edge_owner_to_timeline_media") or {}) | |
| 92 | + .get("edges") or [])] | |
| 93 | + followers = (user.get("edge_followed_by") or {}).get("count") | |
| 94 | + avg_likes = _avg([p["likes"] for p in posts]) | |
| 95 | + avg_comments = _avg([p["comments"] for p in posts]) | |
| 96 | + videos = [p for p in posts if p["type"] in ("video", "reel")] | |
| 97 | + avg_video_views = _avg([p["video_views"] for p in videos]) | |
| 98 | + engagement = None | |
| 99 | + if followers and avg_likes is not None: | |
| 100 | + engagement = round( | |
| 101 | + (avg_likes + (avg_comments or 0)) / followers * 100, 3) | |
| 102 | + timestamps = sorted(p["timestamp"] for p in posts | |
| 103 | + if isinstance(p["timestamp"], int)) | |
| 104 | + posts_per_week = None | |
| 105 | + if len(timestamps) >= 3 and timestamps[-1] > timestamps[0]: | |
| 106 | + span_weeks = (timestamps[-1] - timestamps[0]) / 604_800 | |
| 107 | + posts_per_week = round((len(timestamps) - 1) / span_weeks, 2) | |
| 108 | + top_post = max((p for p in posts if isinstance(p["likes"], int)), | |
| 109 | + key=lambda p: p["likes"], default=None) | |
| 110 | + related = [] | |
| 111 | + for edge in ((user.get("edge_related_profiles") or {}).get("edges") or []): | |
| 112 | + n = edge.get("node") or {} | |
| 113 | + if n.get("username"): | |
| 114 | + related.append({"username": n["username"], | |
| 115 | + "full_name": n.get("full_name"), | |
| 116 | + "is_verified": n.get("is_verified"), | |
| 117 | + "is_private": n.get("is_private")}) | |
| 118 | + return { | |
| 119 | + "kind": "profile", | |
| 120 | + "platform": "instagram", | |
| 121 | + "found": True, | |
| 122 | + "id": user.get("id"), | |
| 123 | + "username": user.get("username"), | |
| 124 | + "full_name": user.get("full_name"), | |
| 125 | + "biography": user.get("biography"), | |
| 126 | + "pronouns": user.get("pronouns") or None, | |
| 127 | + "external_url": user.get("external_url"), | |
| 128 | + "bio_links": [b.get("url") for b in (user.get("bio_links") or []) | |
| 129 | + if b.get("url")], | |
| 130 | + "followers": followers, | |
| 131 | + "following": (user.get("edge_follow") or {}).get("count"), | |
| 132 | + "posts_count": (user.get("edge_owner_to_timeline_media") | |
| 133 | + or {}).get("count"), | |
| 134 | + "highlight_reels": user.get("highlight_reel_count"), | |
| 135 | + "is_verified": user.get("is_verified"), | |
| 136 | + "is_private": user.get("is_private"), | |
| 137 | + "is_business": user.get("is_business_account"), | |
| 138 | + "is_professional": user.get("is_professional_account"), | |
| 139 | + "category": user.get("category_name"), | |
| 140 | + "business_category": user.get("business_category_name"), | |
| 141 | + "business_address": user.get("business_address_json"), | |
| 142 | + "has_clips": user.get("has_clips"), | |
| 143 | + "has_channel": user.get("has_channel"), | |
| 144 | + "has_ar_effects": user.get("has_ar_effects"), | |
| 145 | + "recently_joined": user.get("is_joined_recently"), | |
| 146 | + "avatar": user.get("profile_pic_url_hd") | |
| 147 | + or user.get("profile_pic_url"), | |
| 148 | + # agrégats calculés sur les 12 derniers posts | |
| 149 | + "avg_likes": avg_likes, | |
| 150 | + "avg_comments": avg_comments, | |
| 151 | + "avg_video_views": avg_video_views, | |
| 152 | + "engagement_rate_pct": engagement, | |
| 153 | + "posts_per_week": posts_per_week, | |
| 154 | + "last_post_at": _iso(timestamps[-1]) if timestamps else None, | |
| 155 | + "video_share_pct": (round(len(videos) / len(posts) * 100) | |
| 156 | + if posts else None), | |
| 157 | + "top_post": ({"url": top_post["url"], "likes": top_post["likes"], | |
| 158 | + "comments": top_post["comments"], | |
| 159 | + "type": top_post["type"]} if top_post else None), | |
| 160 | + "recent_posts": posts, | |
| 161 | + "related_profiles": related, | |
| 162 | + } | |
| 163 | + | |
| 164 | + | |
| 165 | +async def main() -> None: | |
| 166 | + async with Actor: | |
| 167 | + inp = await Actor.get_input() or {} | |
| 168 | + usernames = [u.strip().lstrip("@").lower() | |
| 169 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 170 | + queries = [q.strip() for q in (inp.get("queries") or []) | |
| 171 | + if q and q.strip()] | |
| 172 | + proxy = await Actor.create_proxy_configuration( | |
| 173 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 174 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 175 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 3)) | |
| 176 | + | |
| 177 | + async def one_profile(u: str) -> None: | |
| 178 | + async with sem: | |
| 179 | + miss = {"kind": "profile", "platform": "instagram", | |
| 180 | + "username": u, "found": False} | |
| 181 | + try: | |
| 182 | + resp = await fetcher.get(PROFILE_API.format(u=u), | |
| 183 | + headers=HDRS, session_id=u) | |
| 184 | + except Exception as exc: | |
| 185 | + await Actor.push_data({**miss, "error": str(exc)[:200]}) | |
| 186 | + return | |
| 187 | + if resp.status_code == 404: | |
| 188 | + await Actor.push_data({**miss, "error": "http_404"}) | |
| 189 | + return | |
| 190 | + user = None | |
| 191 | + if resp.status_code == 200: | |
| 192 | + try: | |
| 193 | + user = ((json.loads(resp.text).get("data") or {}) | |
| 194 | + .get("user")) | |
| 195 | + except Exception: | |
| 196 | + user = None | |
| 197 | + if not user: | |
| 198 | + await Actor.push_data( | |
| 199 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 200 | + return | |
| 201 | + await Actor.push_data(parse_user(user)) | |
| 202 | + | |
| 203 | + async def one_search(q: str) -> None: | |
| 204 | + async with sem: | |
| 205 | + data = {} | |
| 206 | + for endpoint in (SEARCH_API, SEARCH_API_ALT): | |
| 207 | + u2 = endpoint.format(q=urllib.parse.quote(q)) | |
| 208 | + try: | |
| 209 | + resp = await fetcher.get(u2, headers=HDRS, | |
| 210 | + session_id=f"srch{q[:4]}") | |
| 211 | + except Exception as exc: | |
| 212 | + Actor.log.warning(f"recherche '{q}' : {exc}") | |
| 213 | + continue | |
| 214 | + if resp.status_code == 200: | |
| 215 | + try: | |
| 216 | + data = json.loads(resp.text) | |
| 217 | + except Exception: | |
| 218 | + data = {} | |
| 219 | + if data.get("users"): | |
| 220 | + break | |
| 221 | + if not data.get("users"): | |
| 222 | + Actor.log.warning( | |
| 223 | + f"recherche '{q}' : 0 user " | |
| 224 | + f"(http {resp.status_code}) {resp.text[:200]!r}") | |
| 225 | + for item in data.get("users") or []: | |
| 226 | + u = item.get("user") or {} | |
| 227 | + if not u.get("username") or u.get("is_private"): | |
| 228 | + continue | |
| 229 | + await Actor.push_data({ | |
| 230 | + "kind": "search_user", | |
| 231 | + "platform": "instagram", | |
| 232 | + "query": q, | |
| 233 | + "username": u.get("username"), | |
| 234 | + "full_name": u.get("full_name"), | |
| 235 | + "is_verified": u.get("is_verified"), | |
| 236 | + "followers": u.get("follower_count"), | |
| 237 | + "avatar": u.get("profile_pic_url"), | |
| 238 | + }) | |
| 239 | + | |
| 240 | + await asyncio.gather(*[one_profile(u) for u in usernames], | |
| 241 | + *[one_search(q) for q in queries]) | |
| 242 | + Actor.log.info(f"terminé : {len(usernames)} profils, " | |
| 243 | + f"{len(queries)} recherches") | |
added
actors/ka-instagram/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-kick/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-kick) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-kick/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-kick", | |
| 4 | + "title": "KA Kick — chaînes détaillées", | |
| 5 | + "description": "Chaînes Kick via l'API v2 publique (abonnés, live, bio + liens sociaux auto-déclarés) — proxy résidentiel pour Cloudflare.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-kick/.actor/input_schema.json
+43 −0
@@ -0,0 +1,43 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-kick input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles/slugs à traiter (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [ | |
| 12 | + "xqc" | |
| 13 | + ] | |
| 14 | + }, | |
| 15 | + "requestDelay": { | |
| 16 | + "title": "Délai entre requêtes (s)", | |
| 17 | + "type": "integer", | |
| 18 | + "default": 1, | |
| 19 | + "minimum": 0, | |
| 20 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 21 | + }, | |
| 22 | + "concurrency": { | |
| 23 | + "title": "Requêtes simultanées", | |
| 24 | + "type": "integer", | |
| 25 | + "default": 3, | |
| 26 | + "minimum": 1, | |
| 27 | + "maximum": 10, | |
| 28 | + "description": "Nombre de requêtes menées en parallèle." | |
| 29 | + }, | |
| 30 | + "proxyConfiguration": { | |
| 31 | + "title": "Proxy", | |
| 32 | + "type": "object", | |
| 33 | + "editor": "proxy", | |
| 34 | + "description": "Configuration du proxy Apify.", | |
| 35 | + "prefill": { | |
| 36 | + "useApifyProxy": true, | |
| 37 | + "apifyProxyGroups": [ | |
| 38 | + "RESIDENTIAL" | |
| 39 | + ] | |
| 40 | + } | |
| 41 | + } | |
| 42 | + } | |
| 43 | +} | |
| \ No newline at end of file | ||
added
actors/ka-kick/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-kick) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-kick/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-kick) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-kick/src/main.py
+103 −0
@@ -0,0 +1,103 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-kick) | |
| 4 | +# Desc: Chaînes Kick détaillées via l'API publique /api/v2/channels/{slug} | |
| 5 | +# (Cloudflare : proxy résidentiel + empreinte Chrome) : abonnés, badge, | |
| 6 | +# live en cours, bio + LIENS SOCIAUX AUTO-DÉCLARÉS (instagram/twitter/ | |
| 7 | +# youtube/tiktok/facebook/discord → cross_link 0.90 côté crea-ka). | |
| 8 | +# ============================================================================== | |
| 9 | +from __future__ import annotations | |
| 10 | + | |
| 11 | +import asyncio | |
| 12 | +import json | |
| 13 | + | |
| 14 | +from apify import Actor | |
| 15 | + | |
| 16 | +from .net import Fetcher | |
| 17 | + | |
| 18 | +API_URL = "https://kick.com/api/v2/channels/{u}" | |
| 19 | +SOCIAL_KEYS = ("instagram", "twitter", "youtube", "tiktok", "facebook", | |
| 20 | + "discord") | |
| 21 | + | |
| 22 | + | |
| 23 | +async def main() -> None: | |
| 24 | + async with Actor: | |
| 25 | + inp = await Actor.get_input() or {} | |
| 26 | + usernames = [u.strip().lstrip("@").lower() | |
| 27 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 28 | + proxy = await Actor.create_proxy_configuration( | |
| 29 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 30 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) | |
| 31 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 3)) | |
| 32 | + | |
| 33 | + async def one(u: str) -> None: | |
| 34 | + async with sem: | |
| 35 | + miss = {"kind": "profile", "platform": "kick", | |
| 36 | + "username": u, "found": False} | |
| 37 | + resp = None | |
| 38 | + for attempt in range(3): # mur Cloudflare → nouvelle IP | |
| 39 | + try: | |
| 40 | + resp = await fetcher.get( | |
| 41 | + API_URL.format(u=u), session_id=f"{u}k{attempt}", | |
| 42 | + headers={"Accept": "application/json"}) | |
| 43 | + except Exception as exc: | |
| 44 | + await Actor.push_data({**miss, | |
| 45 | + "error": str(exc)[:200]}) | |
| 46 | + return | |
| 47 | + if resp.status_code == 404: | |
| 48 | + await Actor.push_data({**miss, "error": "http_404"}) | |
| 49 | + return | |
| 50 | + if resp.status_code == 200 and \ | |
| 51 | + (resp.text or "").startswith("{"): | |
| 52 | + break | |
| 53 | + if resp.status_code != 200 or \ | |
| 54 | + not (resp.text or "").startswith("{"): | |
| 55 | + await Actor.push_data( | |
| 56 | + {**miss, "error": f"http_{resp.status_code}"}) | |
| 57 | + return | |
| 58 | + try: | |
| 59 | + ch = json.loads(resp.text) | |
| 60 | + except Exception: | |
| 61 | + await Actor.push_data({**miss, "error": "bad_json"}) | |
| 62 | + return | |
| 63 | + user = ch.get("user") or {} | |
| 64 | + live = ch.get("livestream") or {} | |
| 65 | + cats = [((c.get("category") or {}).get("name")) | |
| 66 | + for c in (ch.get("recent_categories") or []) | |
| 67 | + if isinstance(c, dict)] | |
| 68 | + socials = {k: (user.get(k) or "").strip() | |
| 69 | + for k in SOCIAL_KEYS if (user.get(k) or "").strip()} | |
| 70 | + await Actor.push_data({ | |
| 71 | + "kind": "profile", | |
| 72 | + "platform": "kick", | |
| 73 | + "found": True, | |
| 74 | + "id": ch.get("id"), | |
| 75 | + "username": ch.get("slug"), | |
| 76 | + "full_name": user.get("username"), | |
| 77 | + "biography": user.get("bio"), | |
| 78 | + "followers": ch.get("followers_count") | |
| 79 | + or ch.get("followersCount"), | |
| 80 | + "is_verified": bool(ch.get("verified")), | |
| 81 | + "is_banned": bool(ch.get("is_banned")), | |
| 82 | + "subscription_enabled": ch.get("subscription_enabled"), | |
| 83 | + "is_live_now": bool(live), | |
| 84 | + "live_viewers": live.get("viewer_count"), | |
| 85 | + "live_title": (live.get("session_title") or "")[:200] | |
| 86 | + or None, | |
| 87 | + "live_thumbnail": ((live.get("thumbnail") or {}).get("url") | |
| 88 | + if isinstance(live.get("thumbnail"), | |
| 89 | + dict) else None), | |
| 90 | + "live_category": next( | |
| 91 | + (c.get("name") for c in (live.get("categories") or []) | |
| 92 | + if isinstance(c, dict) and c.get("name")), None), | |
| 93 | + "vod_enabled": ch.get("vod_enabled"), | |
| 94 | + "avatar": user.get("profile_pic"), | |
| 95 | + "banner": ((ch.get("banner_image") or {}).get("url") | |
| 96 | + if isinstance(ch.get("banner_image"), dict) | |
| 97 | + else None), | |
| 98 | + "recent_categories": [c for c in cats if c][:5], | |
| 99 | + "social_links": socials, | |
| 100 | + }) | |
| 101 | + | |
| 102 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 103 | + Actor.log.info(f"terminé : {len(usernames)} chaînes") | |
added
actors/ka-kick/src/net.py
+69 −0
@@ -0,0 +1,69 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/net.py | |
| 4 | +# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + | |
| 5 | +# proxy Apify (résidentiel par défaut), throttling poli + retries | |
| 6 | +# ============================================================================== | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import random | |
| 11 | +import re | |
| 12 | + | |
| 13 | +from apify import Actor | |
| 14 | +from curl_cffi import requests as cffi | |
| 15 | + | |
| 16 | +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " | |
| 17 | + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") | |
| 18 | + | |
| 19 | + | |
| 20 | +class Fetcher: | |
| 21 | + """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" | |
| 22 | + | |
| 23 | + def __init__(self, proxy_configuration, delay: float = 1.0) -> None: | |
| 24 | + self.proxy_configuration = proxy_configuration | |
| 25 | + self.delay = delay | |
| 26 | + self._lock = asyncio.Lock() | |
| 27 | + self._last = 0.0 | |
| 28 | + | |
| 29 | + async def _throttle(self) -> None: | |
| 30 | + async with self._lock: | |
| 31 | + now = asyncio.get_event_loop().time() | |
| 32 | + wait = self.delay - (now - self._last) | |
| 33 | + if wait > 0: | |
| 34 | + await asyncio.sleep(wait) | |
| 35 | + self._last = asyncio.get_event_loop().time() | |
| 36 | + | |
| 37 | + async def get(self, url: str, headers: dict | None = None, | |
| 38 | + retries: int = 3, session_id: str | None = None, | |
| 39 | + impersonate: str | None = "chrome"): | |
| 40 | + hdrs = {"User-Agent": UA, | |
| 41 | + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} | |
| 42 | + if headers: | |
| 43 | + hdrs.update(headers) | |
| 44 | + last_exc: Exception | None = None | |
| 45 | + for attempt in range(retries): | |
| 46 | + await self._throttle() | |
| 47 | + proxy = None | |
| 48 | + if self.proxy_configuration: | |
| 49 | + sid = session_id or f"s{random.randint(1, 999_999)}" | |
| 50 | + sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" | |
| 51 | + proxy = await self.proxy_configuration.new_url( | |
| 52 | + session_id=f"{sid}r{attempt}") | |
| 53 | + try: | |
| 54 | + resp = await asyncio.to_thread( | |
| 55 | + cffi.get, url, headers=hdrs, impersonate=impersonate, | |
| 56 | + timeout=60, allow_redirects=True, | |
| 57 | + proxies={"http": proxy, "https": proxy} if proxy else None) | |
| 58 | + if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: | |
| 59 | + Actor.log.warning( | |
| 60 | + f"HTTP {resp.status_code} {url} — retry {attempt + 1}") | |
| 61 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 62 | + continue | |
| 63 | + return resp | |
| 64 | + except Exception as exc: # réseau/proxy : on retente | |
| 65 | + last_exc = exc | |
| 66 | + await asyncio.sleep(1.5 * (attempt + 1)) | |
| 67 | + if last_exc: | |
| 68 | + raise last_exc | |
| 69 | + raise RuntimeError(f"échec après {retries} tentatives : {url}") | |
added
actors/ka-onlyfans/.actor/Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: Dockerfile (ka-onlyfans) | |
| 4 | +# Desc: Acteur Apify KA — Dockerfile | |
| 5 | +# ============================================================================== | |
| 6 | +FROM apify/actor-python:3.12 | |
| 7 | +COPY requirements.txt ./ | |
| 8 | +RUN pip install --no-cache-dir -r requirements.txt | |
| 9 | +COPY . ./ | |
| 10 | +CMD ["python3", "-m", "src"] | |
added
actors/ka-onlyfans/.actor/actor.json
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +{ | |
| 2 | + "actorSpecification": 1, | |
| 3 | + "name": "ka-onlyfans", | |
| 4 | + "title": "KA OnlyFans — profils publics", | |
| 5 | + "description": "Profils OnlyFans publics best-effort (posts, photos, vidéos, J'aime, prix d'abonnement, bio, avatar) — EXPÉRIMENTAL.", | |
| 6 | + "version": "0.1", | |
| 7 | + "buildTag": "latest", | |
| 8 | + "input": "./input_schema.json", | |
| 9 | + "dockerfile": "./Dockerfile" | |
| 10 | +} | |
| \ No newline at end of file | ||
added
actors/ka-onlyfans/.actor/input_schema.json
+41 −0
@@ -0,0 +1,41 @@ | ||
| 1 | +{ | |
| 2 | + "title": "ka-onlyfans input", | |
| 3 | + "type": "object", | |
| 4 | + "schemaVersion": 1, | |
| 5 | + "properties": { | |
| 6 | + "usernames": { | |
| 7 | + "title": "Handles à enrichir", | |
| 8 | + "type": "array", | |
| 9 | + "description": "Handles/slugs à traiter (sans @).", | |
| 10 | + "editor": "stringList", | |
| 11 | + "prefill": [] | |
| 12 | + }, | |
| 13 | + "requestDelay": { | |
| 14 | + "title": "Délai entre requêtes (s)", | |
| 15 | + "type": "integer", | |
| 16 | + "default": 1, | |
| 17 | + "minimum": 0, | |
| 18 | + "description": "Pause minimale entre deux requêtes sortantes, en secondes." | |
| 19 | + }, | |
| 20 | + "concurrency": { | |
| 21 | + "title": "Requêtes simultanées", | |
| 22 | + "type": "integer", | |
| 23 | + "default": 3, | |
| 24 | + "minimum": 1, | |
| 25 | + "maximum": 10, | |
| 26 | + "description": "Nombre de requêtes menées en parallèle." | |
| 27 | + }, | |
| 28 | + "proxyConfiguration": { | |
| 29 | + "title": "Proxy", | |
| 30 | + "type": "object", | |
| 31 | + "editor": "proxy", | |
| 32 | + "description": "Configuration du proxy Apify.", | |
| 33 | + "prefill": { | |
| 34 | + "useApifyProxy": true, | |
| 35 | + "apifyProxyGroups": [ | |
| 36 | + "RESIDENTIAL" | |
| 37 | + ] | |
| 38 | + } | |
| 39 | + } | |
| 40 | + } | |
| 41 | +} | |
| \ No newline at end of file | ||
added
actors/ka-onlyfans/requirements.txt
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: requirements.txt (ka-onlyfans) | |
| 4 | +# Desc: Acteur Apify KA — requirements.txt | |
| 5 | +# ============================================================================== | |
| 6 | +apify | |
| 7 | +curl_cffi>=0.9 | |
added
actors/ka-onlyfans/src/__main__.py
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/__main__.py (ka-onlyfans) | |
| 4 | +# Desc: Point d'entrée de l'acteur (asyncio.run). | |
| 5 | +# ============================================================================== | |
| 6 | +import asyncio | |
| 7 | + | |
| 8 | +from .main import main | |
| 9 | + | |
| 10 | +asyncio.run(main()) | |
added
actors/ka-onlyfans/src/main.py
+112 −0
@@ -0,0 +1,112 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: src/main.py (ka-onlyfans) | |
| 4 | +# Desc: Profils OnlyFans publics (page de profil hors connexion) — extraction | |
| 5 | +# best-effort des compteurs embarqués : posts, photos, vidéos, J'aime | |
| 6 | +# (favoritedCount), prix d'abonnement, badge, bio, avatar/bannière. | |
| 7 | +# EXPÉRIMENTAL : OnlyFans change souvent son rendu ; found=False n'est | |
| 8 | +# pas une erreur du pipeline. | |
| 9 | +# ============================================================================== | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import asyncio | |
| 13 | +import html as htmllib | |
| 14 | +import json | |
| 15 | +import re | |
| 16 | + | |
| 17 | +from apify import Actor | |
| 18 | + | |
| 19 | +from .net import Fetcher | |
| 20 | + | |
| 21 | +PROFILE_URL = "https://onlyfans.com/{u}" | |
| 22 | + | |
| 23 | +_NUM_RES = { | |
| 24 | + "posts_count": re.compile(r'"postsCount"\s*:\s*(\d+)'), | |
| 25 | + "photos_count": re.compile(r'"photosCount"\s*:\s*(\d+)'), | |
| 26 | + "videos_count": re.compile(r'"videosCount"\s*:\s*(\d+)'), | |
| 27 | + "likes": re.compile(r'"favoritedCount"\s*:\s*(\d+)'), | |
| 28 | + "streams_count": re.compile(r'"finishedStreamsCount"\s*:\s*(\d+)'), | |
| 29 | +} | |
| 30 | +_PRICE_RE = re.compile(r'"subscribePrice"\s*:\s*([\d.]+)') | |
| 31 | +_STR_RES = { | |
| 32 | + "full_name": re.compile(r'"name"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 33 | + "biography": re.compile(r'"rawAbout"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 34 | + "location": re.compile(r'"location"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 35 | + "website": re.compile(r'"website"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 36 | + "avatar": re.compile(r'"avatar"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 37 | + "banner": re.compile(r'"header"\s*:\s*"((?:[^"\\]|\\.)*)"'), | |
| 38 | +} | |
| 39 | +_VERIFIED_RE = re.compile(r'"isVerified"\s*:\s*(true|false)') | |
| 40 | +_OG_TITLE_RE = re.compile(r'<meta property="og:title" content="([^"]*)"') | |
| 41 | + | |
| 42 | + | |
| 43 | +def _dec(raw: str) -> str: | |
| 44 | + try: | |
| 45 | + return htmllib.unescape(json.loads(f'"{raw}"')) | |
| 46 | + except Exception: | |
| 47 | + return raw | |
| 48 | + | |
| 49 | + | |
| 50 | +async def main() -> None: | |
| 51 | + async with Actor: | |
| 52 | + inp = await Actor.get_input() or {} | |
| 53 | + usernames = [u.strip().lstrip("@").lower() | |
| 54 | + for u in (inp.get("usernames") or []) if u and u.strip()] | |
| 55 | + proxy = await Actor.create_proxy_configuration( | |
| 56 | + actor_proxy_input=inp.get("proxyConfiguration")) | |
| 57 | + fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 2)) | |
| 58 | + sem = asyncio.Semaphore(int(inp.get("concurrency") or 2)) | |
| 59 | + | |
| 60 | + async def one(u: str) -> None: | |
| 61 | + async with sem: | |
| 62 | + miss = {"kind": "profile", "platform": "onlyfans", | |
| 63 | + "username": u, "found": False} | |
| 64 | + resp = None | |
| 65 | + for attempt in range(3): # mur Cloudflare → nouvelle IP | |
| 66 | + try: | |
| 67 | + resp = await fetcher.get(PROFILE_URL.format(u=u), | |
| 68 | + session_id=f"{u}o{attempt}") | |
| 69 | + except Exception as exc: | |
| 70 | + await Actor.push_data({**miss, | |
| 71 | + "error": str(exc)[:200]}) | |
| 72 | + return | |
| 73 | + if resp.status_code == 404: | |
| 74 | + await Actor.push_data({**miss, "error": "http_404"}) | |
| 75 | + return | |
| 76 | + if resp.status_code == 200 and \ | |
| 77 | + "postsCount" in (resp.text or ""): | |
| 78 | + break | |
| 79 | + text = resp.text or "" | |
| 80 | + if resp.status_code != 200 or "postsCount" not in text: | |
| 81 | + await Actor.push_data( | |
| 82 | + {**miss, "error": f"wall_{resp.status_code}"}) | |
| 83 | + return | |
| 84 | + rec: dict = {"kind": "profile", "platform": "onlyfans", | |
| 85 | + "found": True, "username": u} | |
| 86 | + for key, rx in _NUM_RES.items(): | |
| 87 | + m = rx.search(text) | |
| 88 | + rec[key] = int(m.group(1)) if m else None | |
| 89 | + for key, rx in _STR_RES.items(): | |
| 90 | + m = rx.search(text) | |
| 91 | + rec[key] = _dec(m.group(1)) if m else None | |
| 92 | + if rec.get("biography"): | |
| 93 | + rec["biography"] = re.sub(r"<[^>]+>", " ", | |
| 94 | + rec["biography"])[:1000].strip() | |
| 95 | + m = _PRICE_RE.search(text) | |
| 96 | + rec["subscribe_price_usd"] = float(m.group(1)) if m else None | |
| 97 | + rec["is_free"] = (rec["subscribe_price_usd"] == 0.0 | |
| 98 | + if rec["subscribe_price_usd"] is not None | |
| 99 | + else None) | |
| 100 | + m = _VERIFIED_RE.search(text) | |
| 101 | + rec["is_verified"] = (m.group(1) == "true") if m else None | |
| 102 | + if not rec.get("full_name"): | |
| 103 | + m = _OG_TITLE_RE.search(text) | |
| 104 | + if m: | |
| 105 | + rec["full_name"] = htmllib.unescape( | |
| 106 | + m.group(1)).split(" OnlyFans")[0].strip() | |
| 107 | + # « followers » au sens crea-ka : le compteur public = J'aime | |
| 108 | + rec["followers"] = None | |
| 109 | + await Actor.push_data(rec) | |
| 110 | + | |
| 111 | + await asyncio.gather(*[one(u) for u in usernames]) | |
| 112 | + Actor.log.info(f"terminé : {len(usernames)} profils") | |
added
actors/ka-onlyfans/src/net.py
+69 −0
Large diff (69 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-patreon/.actor/Dockerfile
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-patreon/.actor/actor.json
+10 −0
Large diff (11 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-patreon/.actor/input_schema.json
+41 −0
Large diff (42 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-patreon/requirements.txt
+7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-patreon/src/__main__.py
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-patreon/src/main.py
+114 −0
Large diff (114 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-patreon/src/net.py
+69 −0
Large diff (69 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-snapchat/.actor/Dockerfile
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-snapchat/.actor/actor.json
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-snapchat/.actor/input_schema.json
+43 −0
Large diff (44 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-snapchat/requirements.txt
+7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-snapchat/src/__main__.py
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-snapchat/src/main.py
+121 −0
Large diff (121 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-snapchat/src/net.py
+69 −0
Large diff (69 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-threads/.actor/Dockerfile
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-threads/.actor/actor.json
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-threads/.actor/input_schema.json
+43 −0
Large diff (44 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-threads/requirements.txt
+7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-threads/src/__main__.py
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-threads/src/main.py
+99 −0
Large diff (99 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-threads/src/net.py
+69 −0
Large diff (69 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-tiktok/.actor/Dockerfile
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-tiktok/.actor/actor.json
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-tiktok/.actor/input_schema.json
+43 −0
Large diff (44 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-tiktok/requirements.txt
+7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-tiktok/src/__main__.py
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-tiktok/src/main.py
+228 −0
Large diff (228 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-tiktok/src/net.py
+69 −0
Large diff (69 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-twitch/.actor/Dockerfile
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-twitch/.actor/actor.json
+10 −0
Large diff (11 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-twitch/.actor/input_schema.json
+40 −0
Large diff (41 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-twitch/requirements.txt
+7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-twitch/src/__main__.py
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-twitch/src/main.py
+158 −0
Large diff (158 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-twitch/src/net.py
+69 −0
Large diff (69 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-x/.actor/Dockerfile
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-x/.actor/actor.json
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-x/.actor/input_schema.json
+48 −0
Large diff (49 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-x/requirements.txt
+7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-x/src/__main__.py
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-x/src/main.py
+217 −0
Large diff (217 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-x/src/net.py
+69 −0
Large diff (69 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-youtube/.actor/Dockerfile
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-youtube/.actor/actor.json
+10 −0
Large diff (11 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-youtube/.actor/input_schema.json
+46 −0
Large diff (47 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-youtube/requirements.txt
+7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-youtube/src/__main__.py
+10 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-youtube/src/main.py
+303 −0
Large diff (303 lines) is not loaded by default to keep the page fast.Load diff
added
actors/ka-youtube/src/net.py
+69 −0
Large diff (69 lines) is not loaded by default to keep the page fast.Load diff
modified
creaka/connectors/apify_social.py
+53 −3
Large diff (124 lines) is not loaded by default to keep the page fast.Load diff
modified
creaka/db.py
+56 −0
Large diff (74 lines) is not loaded by default to keep the page fast.Load diff
modified
creaka/schema.py
+1 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
modified
creaka/stats.py
+90 −0
Large diff (93 lines) is not loaded by default to keep the page fast.Load diff
modified
creaka/web.py
+3 −0
Large diff (9 lines) is not loaded by default to keep the page fast.Load diff
modified
frontend/dist/index.html
+102 −5
Large diff (177 lines) is not loaded by default to keep the page fast.Load diff
modified
frontend/src/index.template.html
+102 −5
Large diff (178 lines) is not loaded by default to keep the page fast.Load diff