SPB Git
spboucher.ai

spb/crea-ka

Public

Créa·Ka — annuaire public cross-plateforme des créateurs de contenu québécois (crea-ka.com)

quebeccreateursinfluenceursannuaireka

Upgrade majeur round 2 des 13 acteurs Apify : net.py commun durci (retries 403/407/408/425/429/500/502/503/504, backoff exponentiel + jitter, rotation d'IP par tentative, POST avec retries — Twitch GQL passe par ce moteur) + nouveaux champs par plateforme : IG (bio mentions/hashtags, flag Threads relié → cross-link, business_email public, IGTV), TikTok (likes_given, avg_saves, épinglés, flag mineur isUnderAge18 → is_minor §15), YouTube (videos_per_month, has_shorts, live), X (top_mentions §12.1, avg_quotes, parts liens/RT, verified_type), FB (page_id, website → cross-link, rating), Threads (bio_links → cross-links forts, pk), Twitch (totalCount vidéos, type/début de live), Kick (début/langue/maturité du live), Patreon (+ socials IG/TikTok/Twitch/Discord, published_at), Fansly (tiers/prix, createdAt), Discord (features, premium_tier, expiration), OnlyFans (joinDate, audios), Snapchat (bannière héro, snapcode) — 13 poussés, 12 validés par runs de test ; connecteur : metric_maps étendus, cross-links IG→Threads et bio_links Threads, 51 tests verts

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Aug 21, 2026) parent 0cad882 Browse files

27 changed files +732 −157

27 changed files
Load all 27 files Unified Split
modified actors/ka-discord/src/main.py +6 −0
@@ -81,6 +81,12 @@ async def main() -> None:
81 f"{splash_hash}.jpg?size=1024" 81 f"{splash_hash}.jpg?size=1024"
82 if gid and splash_hash else None), 82 if gid and splash_hash else None),
83 "vanity_url_code": guild.get("vanity_url_code"), 83 "vanity_url_code": guild.get("vanity_url_code"),
84+ "features": sorted(guild.get("features") or [])[:12],
85+ "nsfw_level": guild.get("nsfw_level"),
86+ "premium_tier": guild.get("premium_tier"),
87+ "invite_expires_at": d.get("expires_at"),
88+ "inviter": ((d.get("inviter") or {}).get("username")
89+ or None),
84 }) 90 })
85 91
86 await asyncio.gather(*[one(c) for c in codes]) 92 await asyncio.gather(*[one(c) for c in codes])
modified actors/ka-discord/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-facebook/src/main.py +32 −0
@@ -11,6 +11,7 @@ from __future__ import annotations
11 11
12 import asyncio 12 import asyncio
13 import html as htmllib 13 import html as htmllib
14+import json
14 import re 15 import re
15 16
16 from apify import Actor 17 from apify import Actor
@@ -37,6 +38,17 @@ _TEXT_COUNT_RE = re.compile(
37 re.I) 38 re.I)
38 _CATEGORY_RE = re.compile(r'"category_name"\s*:\s*"([^"]+)"') 39 _CATEGORY_RE = re.compile(r'"category_name"\s*:\s*"([^"]+)"')
39 _VERIFIED_RE = re.compile(r'"is_verified"\s*:\s*(true|false)') 40 _VERIFIED_RE = re.compile(r'"is_verified"\s*:\s*(true|false)')
41+_PAGE_ID_RES = (
42+ re.compile(r'"page_id"\s*:\s*"?(\d{6,})"?'),
43+ re.compile(r'"pageID"\s*:\s*"(\d{6,})"'),
44+ re.compile(r'"delegate_page_id"\s*:\s*"(\d{6,})"'),
45+)
46+# site web auto-déclaré de la page → cross-link côté crea-ka
47+_WEBSITE_RE = re.compile(
48+ r'"website(?:s)?"\s*:\s*\[?\s*"((?:https?:)?(?:[^"\\]|\\.)+)"')
49+_RATING_RE = re.compile(r'"overall_star_rating"\s*:\s*\{[^{}]*?'
50+ r'"value"\s*:\s*([\d.]+)')
51+_OG_URL_RE = re.compile(r'<meta property="og:url" content="([^"]*)"')
40 52
41 53
42 def parse_text_count(text: str) -> int | None: 54 def parse_text_count(text: str) -> int | None:
@@ -103,16 +115,36 @@ async def main() -> None:
103 return 115 return
104 cat = _CATEGORY_RE.search(text) 116 cat = _CATEGORY_RE.search(text)
105 ver = _VERIFIED_RE.search(text) 117 ver = _VERIFIED_RE.search(text)
118+ page_id = next((m.group(1) for rx in _PAGE_ID_RES
119+ if (m := rx.search(text))), None)
120+ site = _WEBSITE_RE.search(text)
121+ website = None
122+ if site:
123+ try:
124+ website = json.loads(f'"{site.group(1)}"').strip()
125+ except Exception:
126+ website = site.group(1)
127+ if website and not website.startswith("http"):
128+ website = f"https://{website}"
129+ if website and "facebook.com" in website:
130+ website = None # auto-référence sans valeur
131+ rating = _RATING_RE.search(text)
132+ canon = _OG_URL_RE.search(text)
106 await Actor.push_data({ 133 await Actor.push_data({
107 "kind": "profile", 134 "kind": "profile",
108 "platform": "facebook", 135 "platform": "facebook",
109 "found": True, 136 "found": True,
110 "username": u, 137 "username": u,
138+ "page_id": page_id,
111 "full_name": og.get("title"), 139 "full_name": og.get("title"),
112 "biography": og.get("description"), 140 "biography": og.get("description"),
113 "followers": followers, 141 "followers": followers,
114 "category": cat.group(1) if cat else None, 142 "category": cat.group(1) if cat else None,
115 "is_verified": (ver.group(1) == "true") if ver else None, 143 "is_verified": (ver.group(1) == "true") if ver else None,
144+ "website": website,
145+ "rating": float(rating.group(1)) if rating else None,
146+ "canonical_url": (htmllib.unescape(canon.group(1))
147+ if canon else None),
116 "avatar": og.get("image"), 148 "avatar": og.get("image"),
117 "og_description": og.get("description"), 149 "og_description": og.get("description"),
118 }) 150 })
modified actors/ka-facebook/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-fansly/src/main.py +7 −0
@@ -61,6 +61,9 @@ async def main() -> None:
61 continue 61 continue
62 found.add(u) 62 found.add(u)
63 stats = acc.get("timelineStats") or {} 63 stats = acc.get("timelineStats") or {}
64+ tiers = [{"name": t.get("name"), "price": t.get("price")}
65+ for t in (acc.get("subscriptionTiers") or [])
66+ if isinstance(t, dict) and t.get("name")]
64 await Actor.push_data({ 67 await Actor.push_data({
65 "kind": "profile", 68 "kind": "profile",
66 "platform": "fansly", 69 "platform": "fansly",
@@ -82,6 +85,10 @@ async def main() -> None:
82 "avatar": _loc(acc.get("avatar")), 85 "avatar": _loc(acc.get("avatar")),
83 "banner": _loc(acc.get("banner")), 86 "banner": _loc(acc.get("banner")),
84 "location": acc.get("location"), 87 "location": acc.get("location"),
88+ "account_created_at": acc.get("createdAt"),
89+ "subscription_tiers": tiers[:5] or None,
90+ "subscription_price": (tiers[0].get("price")
91+ if tiers else None),
85 }) 92 })
86 for u in batch: 93 for u in batch:
87 if u not in found: 94 if u not in found:
modified actors/ka-fansly/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-instagram/src/main.py +21 −0
@@ -90,6 +90,15 @@ def parse_user(user: dict) -> dict:
90 posts = [parse_post((e.get("node") or {})) 90 posts = [parse_post((e.get("node") or {}))
91 for e in ((user.get("edge_owner_to_timeline_media") or {}) 91 for e in ((user.get("edge_owner_to_timeline_media") or {})
92 .get("edges") or [])] 92 .get("edges") or [])]
93+ bio_ent = user.get("biography_with_entities") or {}
94+ bio_mentions, bio_hashtags = [], []
95+ for ent in (bio_ent.get("entities") or []):
96+ eu = ((ent or {}).get("user") or {}).get("username")
97+ eh = ((ent or {}).get("hashtag") or {}).get("name")
98+ if eu:
99+ bio_mentions.append(eu)
100+ if eh:
101+ bio_hashtags.append(eh)
93 followers = (user.get("edge_followed_by") or {}).get("count") 102 followers = (user.get("edge_followed_by") or {}).get("count")
94 avg_likes = _avg([p["likes"] for p in posts]) 103 avg_likes = _avg([p["likes"] for p in posts])
95 avg_comments = _avg([p["comments"] for p in posts]) 104 avg_comments = _avg([p["comments"] for p in posts])
@@ -123,6 +132,8 @@ def parse_user(user: dict) -> dict:
123 "username": user.get("username"), 132 "username": user.get("username"),
124 "full_name": user.get("full_name"), 133 "full_name": user.get("full_name"),
125 "biography": user.get("biography"), 134 "biography": user.get("biography"),
135+ "bio_mentions": bio_mentions[:10], # @comptes cités dans la bio
136+ "bio_hashtags": bio_hashtags[:10],
126 "pronouns": user.get("pronouns") or None, 137 "pronouns": user.get("pronouns") or None,
127 "external_url": user.get("external_url"), 138 "external_url": user.get("external_url"),
128 "bio_links": [b.get("url") for b in (user.get("bio_links") or []) 139 "bio_links": [b.get("url") for b in (user.get("bio_links") or [])
@@ -131,6 +142,10 @@ def parse_user(user: dict) -> dict:
131 "following": (user.get("edge_follow") or {}).get("count"), 142 "following": (user.get("edge_follow") or {}).get("count"),
132 "posts_count": (user.get("edge_owner_to_timeline_media") 143 "posts_count": (user.get("edge_owner_to_timeline_media")
133 or {}).get("count"), 144 or {}).get("count"),
145+ "igtv_videos_count": (user.get("edge_felix_video_timeline")
146+ or {}).get("count"),
147+ "mutual_followers": (user.get("edge_mutual_followed_by")
148+ or {}).get("count"),
134 "highlight_reels": user.get("highlight_reel_count"), 149 "highlight_reels": user.get("highlight_reel_count"),
135 "is_verified": user.get("is_verified"), 150 "is_verified": user.get("is_verified"),
136 "is_private": user.get("is_private"), 151 "is_private": user.get("is_private"),
@@ -139,6 +154,12 @@ def parse_user(user: dict) -> dict:
139 "category": user.get("category_name"), 154 "category": user.get("category_name"),
140 "business_category": user.get("business_category_name"), 155 "business_category": user.get("business_category_name"),
141 "business_address": user.get("business_address_json"), 156 "business_address": user.get("business_address_json"),
157+ # courriel PRO affiché publiquement par le créateur (§15 : public only)
158+ "business_email": user.get("business_email") or None,
159+ # présent = compte Threads relié (cross-link fort, même handle)
160+ "has_threads": (bool(user.get("has_onboarded_to_text_post_app"))
161+ if "has_onboarded_to_text_post_app" in user
162+ else None),
142 "has_clips": user.get("has_clips"), 163 "has_clips": user.get("has_clips"),
143 "has_channel": user.get("has_channel"), 164 "has_channel": user.get("has_channel"),
144 "has_ar_effects": user.get("has_ar_effects"), 165 "has_ar_effects": user.get("has_ar_effects"),
modified actors/ka-instagram/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-kick/src/main.py +5 −0
@@ -82,6 +82,11 @@ async def main() -> None:
82 "subscription_enabled": ch.get("subscription_enabled"), 82 "subscription_enabled": ch.get("subscription_enabled"),
83 "is_live_now": bool(live), 83 "is_live_now": bool(live),
84 "live_viewers": live.get("viewer_count"), 84 "live_viewers": live.get("viewer_count"),
85+ "live_started_at": (live.get("start_time")
86+ or live.get("created_at")),
87+ "live_is_mature": (bool(live.get("is_mature"))
88+ if live else None),
89+ "live_language": live.get("language"),
85 "live_title": (live.get("session_title") or "")[:200] 90 "live_title": (live.get("session_title") or "")[:200]
86 or None, 91 or None,
87 "live_thumbnail": ((live.get("thumbnail") or {}).get("url") 92 "live_thumbnail": ((live.get("thumbnail") or {}).get("url")
modified actors/ka-kick/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-onlyfans/src/main.py +8 −0
@@ -26,7 +26,11 @@ _NUM_RES = {
26 "videos_count": re.compile(r'"videosCount"\s*:\s*(\d+)'), 26 "videos_count": re.compile(r'"videosCount"\s*:\s*(\d+)'),
27 "likes": re.compile(r'"favoritedCount"\s*:\s*(\d+)'), 27 "likes": re.compile(r'"favoritedCount"\s*:\s*(\d+)'),
28 "streams_count": re.compile(r'"finishedStreamsCount"\s*:\s*(\d+)'), 28 "streams_count": re.compile(r'"finishedStreamsCount"\s*:\s*(\d+)'),
29+ "audios_count": re.compile(r'"audiosCount"\s*:\s*(\d+)'),
30+ "medias_count": re.compile(r'"mediasCount"\s*:\s*(\d+)'),
29 } 31 }
32+_JOIN_RE = re.compile(r'"joinDate"\s*:\s*"((?:[^"\\]|\\.)*)"')
33+_PERFORMER_RE = re.compile(r'"isPerformer"\s*:\s*(true|false)')
30 _PRICE_RE = re.compile(r'"subscribePrice"\s*:\s*([\d.]+)') 34 _PRICE_RE = re.compile(r'"subscribePrice"\s*:\s*([\d.]+)')
31 _STR_RES = { 35 _STR_RES = {
32 "full_name": re.compile(r'"name"\s*:\s*"((?:[^"\\]|\\.)*)"'), 36 "full_name": re.compile(r'"name"\s*:\s*"((?:[^"\\]|\\.)*)"'),
@@ -99,6 +103,10 @@ async def main() -> None:
99 else None) 103 else None)
100 m = _VERIFIED_RE.search(text) 104 m = _VERIFIED_RE.search(text)
101 rec["is_verified"] = (m.group(1) == "true") if m else None 105 rec["is_verified"] = (m.group(1) == "true") if m else None
106+ m = _JOIN_RE.search(text)
107+ rec["join_date"] = _dec(m.group(1)) if m else None
108+ m = _PERFORMER_RE.search(text)
109+ rec["is_performer"] = (m.group(1) == "true") if m else None
102 if not rec.get("full_name"): 110 if not rec.get("full_name"):
103 m = _OG_TITLE_RE.search(text) 111 m = _OG_TITLE_RE.search(text)
104 if m: 112 if m:
modified actors/ka-onlyfans/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-patreon/src/main.py +12 −0
@@ -34,7 +34,13 @@ _SOCIALS = {
34 "facebook": re.compile(r'"facebook"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'), 34 "facebook": re.compile(r'"facebook"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
35 "twitter": re.compile(r'"twitter"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'), 35 "twitter": re.compile(r'"twitter"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
36 "youtube": re.compile(r'"youtube"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'), 36 "youtube": re.compile(r'"youtube"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
37+ "instagram": re.compile(r'"instagram"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
38+ "tiktok": re.compile(r'"tiktok"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
39+ "twitch": re.compile(r'"twitch"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
40+ "discord": re.compile(r'"discord"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
37 } 41 }
42+_PUBLISHED_RE = re.compile(r'"published_at"\s*:\s*"([^"]+)"')
43+_PPN_RE = re.compile(r'"pay_per_name"\s*:\s*"([^"]+)"')
38 44
39 45
40 def _dec(raw: str) -> str: 46 def _dec(raw: str) -> str:
@@ -103,6 +109,12 @@ async def main() -> None:
103 "is_monthly": (monthly.group(1) == "true" 109 "is_monthly": (monthly.group(1) == "true"
104 if monthly else None), 110 if monthly else None),
105 "is_nsfw": nsfw.group(1) == "true" if nsfw else None, 111 "is_nsfw": nsfw.group(1) == "true" if nsfw else None,
112+ "published_at": (pub.group(1) if
113+ (pub := _PUBLISHED_RE.search(text))
114+ else None),
115+ "pay_per_name": (ppn.group(1)[:40] if
116+ (ppn := _PPN_RE.search(text))
117+ else None),
106 "creation_name": (_dec(creation.group(1))[:200] 118 "creation_name": (_dec(creation.group(1))[:200]
107 if creation else None), 119 if creation else None),
108 "avatar": htmllib.unescape(img.group(1)) if img else None, 120 "avatar": htmllib.unescape(img.group(1)) if img else None,
modified actors/ka-patreon/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-snapchat/src/main.py +3 −0
@@ -103,6 +103,9 @@ async def main() -> None:
103 "website": info.get("websiteUrl"), 103 "website": info.get("websiteUrl"),
104 "address": info.get("address"), 104 "address": info.get("address"),
105 "avatar": info.get("profilePictureUrl"), 105 "avatar": info.get("profilePictureUrl"),
106+ "banner": (info.get("squareHeroImageUrl")
107+ or info.get("heroImageUrl") or None),
108+ "snapcode": info.get("snapcodeImageUrl") or None,
106 "has_story": bool(info.get("hasStory")), 109 "has_story": bool(info.get("hasStory")),
107 "has_spotlight": bool(info.get("hasSpotlightHighlights")), 110 "has_spotlight": bool(info.get("hasSpotlightHighlights")),
108 "has_curated_highlights": bool( 111 "has_curated_highlights": bool(
modified actors/ka-snapchat/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-threads/src/main.py +14 −0
@@ -24,6 +24,10 @@ _BIO_RE = re.compile(r'"biography"\s*:\s*"((?:[^"\\]|\\.)*)"')
24 _NAME_RE = re.compile(r'"full_name"\s*:\s*"((?:[^"\\]|\\.)*)"') 24 _NAME_RE = re.compile(r'"full_name"\s*:\s*"((?:[^"\\]|\\.)*)"')
25 _VERIFIED_RE = re.compile(r'"is_verified"\s*:\s*(true|false)') 25 _VERIFIED_RE = re.compile(r'"is_verified"\s*:\s*(true|false)')
26 _PIC_RE = re.compile(r'"profile_pic_url"\s*:\s*"((?:[^"\\]|\\.)*)"') 26 _PIC_RE = re.compile(r'"profile_pic_url"\s*:\s*"((?:[^"\\]|\\.)*)"')
27+# liens auto-déclarés de la bio → cross-links forts côté crea-ka (§12.1)
28+_BIO_LINKS_RE = re.compile(r'"bio_links"\s*:\s*\[(.{0,2000}?)\]', re.S)
29+_LINK_URL_RE = re.compile(r'"url"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"')
30+_PK_RE = re.compile(r'"pk"\s*:\s*"?(\d{4,})"?')
27 # posts : paires texte + like_count dans les payloads thread_items 31 # posts : paires texte + like_count dans les payloads thread_items
28 _POST_RE = re.compile( 32 _POST_RE = re.compile(
29 r'"caption"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"[^{}]*?\}' 33 r'"caption"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"[^{}]*?\}'
@@ -69,6 +73,14 @@ async def main() -> None:
69 name = _NAME_RE.search(text) 73 name = _NAME_RE.search(text)
70 ver = _VERIFIED_RE.search(text) 74 ver = _VERIFIED_RE.search(text)
71 pic = _PIC_RE.search(text) 75 pic = _PIC_RE.search(text)
76+ pk = _PK_RE.search(text)
77+ bio_links: list[str] = []
78+ bl = _BIO_LINKS_RE.search(text)
79+ if bl:
80+ for m2 in _LINK_URL_RE.finditer(bl.group(1)):
81+ url2 = _dec(m2.group(1))
82+ if url2 not in bio_links:
83+ bio_links.append(url2)
72 likes = [p["likes"] for p in posts] 84 likes = [p["likes"] for p in posts]
73 top = max(posts, key=lambda p: p["likes"], default=None) 85 top = max(posts, key=lambda p: p["likes"], default=None)
74 await Actor.push_data({ 86 await Actor.push_data({
@@ -76,8 +88,10 @@ async def main() -> None:
76 "platform": "threads", 88 "platform": "threads",
77 "found": True, 89 "found": True,
78 "username": u, 90 "username": u,
91+ "id": pk.group(1) if pk else None,
79 "full_name": _dec(name.group(1)) if name else None, 92 "full_name": _dec(name.group(1)) if name else None,
80 "biography": _dec(bio.group(1)) if bio else None, 93 "biography": _dec(bio.group(1)) if bio else None,
94+ "bio_links": bio_links[:5] or None,
81 "followers": int(fol.group(1)), 95 "followers": int(fol.group(1)),
82 "is_verified": (ver.group(1) == "true") if ver else None, 96 "is_verified": (ver.group(1) == "true") if ver else None,
83 "avatar": _dec(pic.group(1)) if pic else None, 97 "avatar": _dec(pic.group(1)) if pic else None,
modified actors/ka-threads/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-tiktok/src/main.py +12 −0
@@ -138,6 +138,8 @@ async def main() -> None:
138 if isinstance(v["comments"], int)] 138 if isinstance(v["comments"], int)]
139 shares = [v["shares"] for v in videos 139 shares = [v["shares"] for v in videos
140 if isinstance(v["shares"], int)] 140 if isinstance(v["shares"], int)]
141+ saves = [v["saves"] for v in videos
142+ if isinstance(v["saves"], int)]
141 avg_views = round(sum(views) / len(views)) if views else None 143 avg_views = round(sum(views) / len(views)) if views else None
142 avg_likes = round(sum(likes) / len(likes)) if likes else None 144 avg_likes = round(sum(likes) / len(likes)) if likes else None
143 engagement = None 145 engagement = None
@@ -187,6 +189,7 @@ async def main() -> None:
187 "followers": followers, 189 "followers": followers,
188 "following": _num(stats.get("followingCount")), 190 "following": _num(stats.get("followingCount")),
189 "friends": _num(stats.get("friendCount")), 191 "friends": _num(stats.get("friendCount")),
192+ "likes_given": _num(stats.get("diggCount")),
190 "total_likes": total_likes, 193 "total_likes": total_likes,
191 "videos_count": videos_count, 194 "videos_count": videos_count,
192 "avg_likes_per_video_lifetime": ( 195 "avg_likes_per_video_lifetime": (
@@ -196,6 +199,11 @@ async def main() -> None:
196 "is_organization": bool(user.get("isOrganization")), 199 "is_organization": bool(user.get("isOrganization")),
197 "is_seller": bool(user.get("ttSeller")), 200 "is_seller": bool(user.get("ttSeller")),
198 "is_live_now": bool(user.get("roomId")), 201 "is_live_now": bool(user.get("roomId")),
202+ # drapeau mineur déclaré par TikTok → régime restreint §15
203+ "is_under_18": (bool(user.get("isUnderAge18"))
204+ if "isUnderAge18" in user else None),
205+ "is_embed_banned": (bool(user.get("isEmbedBanned"))
206+ if "isEmbedBanned" in user else None),
199 "commerce_category": (commerce.get("category") or None 207 "commerce_category": (commerce.get("category") or None
200 if commerce.get("commerceUser") 208 if commerce.get("commerceUser")
201 else None), 209 else None),
@@ -211,6 +219,10 @@ async def main() -> None:
211 if comments else None), 219 if comments else None),
212 "avg_shares": (round(sum(shares) / len(shares)) 220 "avg_shares": (round(sum(shares) / len(shares))
213 if shares else None), 221 if shares else None),
222+ "avg_saves": (round(sum(saves) / len(saves))
223+ if saves else None),
224+ "pinned_videos_count": (sum(1 for v in videos
225+ if v["is_pinned"]) or None),
214 "engagement_rate_pct": engagement, 226 "engagement_rate_pct": engagement,
215 "videos_per_week": videos_per_week, 227 "videos_per_week": videos_per_week,
216 "last_video_at": last_video_at, 228 "last_video_at": last_video_at,
modified actors/ka-tiktok/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-twitch/src/main.py +10 −14
@@ -35,10 +35,11 @@ query($login: String!) {
35 primaryTeam { displayName } 35 primaryTeam { displayName }
36 channel { socialMedias { name title url } } 36 channel { socialMedias { name title url } }
37 lastBroadcast { startedAt title game { displayName } } 37 lastBroadcast { startedAt title game { displayName } }
38− stream { viewersCount createdAt title 38+ stream { viewersCount createdAt title type
39 previewImageURL(width: 640, height: 360) 39 previewImageURL(width: 640, height: 360)
40 game { displayName } } 40 game { displayName } }
41 videos(first: 10, sort: TIME) { 41 videos(first: 10, sort: TIME) {
42+ totalCount
42 edges { node { 43 edges { node {
43 id title viewCount lengthSeconds publishedAt 44 id title viewCount lengthSeconds publishedAt
44 previewThumbnailURL(width: 320, height: 180) 45 previewThumbnailURL(width: 320, height: 180)
@@ -60,23 +61,14 @@ async def main() -> None:
60 fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1)) 61 fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1))
61 sem = asyncio.Semaphore(int(inp.get("concurrency") or 4)) 62 sem = asyncio.Semaphore(int(inp.get("concurrency") or 4))
62 63
63− import random
64− from curl_cffi import requests as cffi
65−
66 async def gql(login: str) -> dict | None: 64 async def gql(login: str) -> dict | None:
67 body = json.dumps({"query": QUERY, 65 body = json.dumps({"query": QUERY,
68 "variables": {"login": login}}) 66 "variables": {"login": login}})
69− p = None 67+ # fetcher.post = retries/backoff/rotation d'IP du net.py commun
70− if fetcher.proxy_configuration: 68+ resp = await fetcher.post(
71− p = await fetcher.proxy_configuration.new_url( 69+ GQL_URL, data=body, session_id=f"tw{login}",
72− session_id=f"tw{random.randint(1, 999_999)}")
73− await fetcher._throttle()
74− resp = await asyncio.to_thread(
75− cffi.post, GQL_URL, data=body,
76 headers={"Client-ID": CLIENT_ID, "User-Agent": UA, 70 headers={"Client-ID": CLIENT_ID, "User-Agent": UA,
77− "Content-Type": "application/json"}, 71+ "Content-Type": "application/json"})
78− impersonate="chrome", timeout=45,
79− proxies={"http": p, "https": p} if p else None)
80 if resp.status_code != 200: 72 if resp.status_code != 200:
81 raise RuntimeError(f"gql http_{resp.status_code}") 73 raise RuntimeError(f"gql http_{resp.status_code}")
82 return (resp.json().get("data") or {}).get("user") 74 return (resp.json().get("data") or {}).get("user")
@@ -131,6 +123,8 @@ async def main() -> None:
131 "avatar": user.get("profileImageURL"), 123 "avatar": user.get("profileImageURL"),
132 "banner": user.get("bannerImageURL"), 124 "banner": user.get("bannerImageURL"),
133 "is_live_now": bool(stream), 125 "is_live_now": bool(stream),
126+ "live_type": stream.get("type"),
127+ "live_started_at": stream.get("createdAt"),
134 "live_viewers": stream.get("viewersCount"), 128 "live_viewers": stream.get("viewersCount"),
135 "live_title": (stream.get("title") or "")[:200] or None, 129 "live_title": (stream.get("title") or "")[:200] or None,
136 "live_thumbnail": stream.get("previewImageURL"), 130 "live_thumbnail": stream.get("previewImageURL"),
@@ -140,6 +134,8 @@ async def main() -> None:
140 or None, 134 or None,
141 "last_broadcast_game": ((last.get("game") or {}) 135 "last_broadcast_game": ((last.get("game") or {})
142 .get("displayName")), 136 .get("displayName")),
137+ "videos_count": ((user.get("videos") or {})
138+ .get("totalCount")),
143 "avg_video_views": (round(sum(views) / len(views)) 139 "avg_video_views": (round(sum(views) / len(views))
144 if views else None), 140 if views else None),
145 "recent_games": list(dict.fromkeys(games))[:5], 141 "recent_games": list(dict.fromkeys(games))[:5],
modified actors/ka-twitch/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-x/src/main.py +19 −0
@@ -163,9 +163,14 @@ async def main() -> None:
163 top = max((t for t in own if isinstance(t["likes"], int)), 163 top = max((t for t in own if isinstance(t["likes"], int)),
164 key=lambda t: t["likes"], default=None) 164 key=lambda t: t["likes"], default=None)
165 hashtags: dict[str, int] = {} 165 hashtags: dict[str, int] = {}
166+ mentions: dict[str, int] = {}
166 for t in own: 167 for t in own:
167 for h in t["hashtags"]: 168 for h in t["hashtags"]:
168 hashtags[h.lower()] = hashtags.get(h.lower(), 0) + 1 169 hashtags[h.lower()] = hashtags.get(h.lower(), 0) + 1
170+ for mn in t["mentions"]:
171+ mentions[mn.lower()] = mentions.get(mn.lower(), 0) + 1
172+ quotes = [t["quotes"] for t in own
173+ if isinstance(t["quotes"], int)]
169 created_ts = _ts(user.get("created_at")) 174 created_ts = _ts(user.get("created_at"))
170 await Actor.push_data({ 175 await Actor.push_data({
171 "kind": "profile", 176 "kind": "profile",
@@ -187,6 +192,7 @@ async def main() -> None:
187 "is_verified": bool(user.get("verified") 192 "is_verified": bool(user.get("verified")
188 or user.get("is_blue_verified")), 193 or user.get("is_blue_verified")),
189 "is_blue_verified": user.get("is_blue_verified"), 194 "is_blue_verified": user.get("is_blue_verified"),
195+ "verified_type": user.get("verified_type"),
190 "is_protected": user.get("protected"), 196 "is_protected": user.get("protected"),
191 "created_at": user.get("created_at"), 197 "created_at": user.get("created_at"),
192 "account_age_days": (round((time.time() - created_ts) 198 "account_age_days": (round((time.time() - created_ts)
@@ -200,6 +206,8 @@ async def main() -> None:
200 "avg_retweets": avg_rts, 206 "avg_retweets": avg_rts,
201 "avg_replies": (round(sum(reps) / len(reps)) 207 "avg_replies": (round(sum(reps) / len(reps))
202 if reps else None), 208 if reps else None),
209+ "avg_quotes": (round(sum(quotes) / len(quotes))
210+ if quotes else None),
203 "engagement_rate_pct": engagement, 211 "engagement_rate_pct": engagement,
204 "tweets_per_week": tweets_per_week, 212 "tweets_per_week": tweets_per_week,
205 "last_tweet_at": own[0]["created_at"] if own else None, 213 "last_tweet_at": own[0]["created_at"] if own else None,
@@ -211,8 +219,19 @@ async def main() -> None:
211 for t in own) 219 for t in own)
212 / len(own) * 100) 220 / len(own) * 100)
213 if own else None), 221 if own else None),
222+ "link_share_pct": (round(sum(bool(t["links"])
223+ for t in own)
224+ / len(own) * 100)
225+ if own else None),
226+ "retweet_share_pct": (round(sum(t["is_retweet"]
227+ for t in tweets)
228+ / len(tweets) * 100)
229+ if tweets else None),
214 "top_hashtags": sorted(hashtags, key=hashtags.get, 230 "top_hashtags": sorted(hashtags, key=hashtags.get,
215 reverse=True)[:8], 231 reverse=True)[:8],
232+ # comptes les plus mentionnés → signal d'identité §12.1
233+ "top_mentions": sorted(mentions, key=mentions.get,
234+ reverse=True)[:8],
216 "top_tweet": ({"url": top["url"], "likes": top["likes"], 235 "top_tweet": ({"url": top["url"], "likes": top["likes"],
217 "retweets": top["retweets"]} 236 "retweets": top["retweets"]}
218 if top else None), 237 if top else None),
modified actors/ka-x/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified actors/ka-youtube/src/main.py +30 −0
@@ -164,6 +164,32 @@ def _s(v):
164 return v.get("content") if isinstance(v, dict) else v 164 return v.get("content") if isinstance(v, dict) else v
165 165
166 166
167+_REL_RE = re.compile(
168+ r"(\d+)\s*(hour|day|week|month|year|heure|jour|semaine|mois|an)", re.I)
169+_REL_DAYS = {"hour": 1 / 24, "heure": 1 / 24, "day": 1, "jour": 1,
170+ "week": 7, "semaine": 7, "month": 30.44, "mois": 30.44,
171+ "year": 365.25, "an": 365.25}
172+
173+
174+def _rel_days(published: str | None) -> float | None:
175+ """« 3 weeks ago » / « il y a 2 mois » → ancienneté approx. en jours."""
176+ m = _REL_RE.search(published or "")
177+ if not m:
178+ return None
179+ unit = m.group(2).lower()
180+ unit = next((k for k in _REL_DAYS if unit.startswith(k)), None)
181+ return float(m.group(1)) * _REL_DAYS[unit] if unit else None
182+
183+
184+def _cadence(vids: list[dict]) -> float | None:
185+ """Vidéos/mois estimées d'après les dates relatives (approximatif)."""
186+ ages = sorted(a for a in (_rel_days(v.get("published")) for v in vids)
187+ if a is not None)
188+ if len(ages) < 3 or ages[-1] <= ages[0]:
189+ return None
190+ return round((len(ages) - 1) / ((ages[-1] - ages[0]) / 30.44), 2)
191+
192+
167 def _banner_of(data: dict) -> str | None: 193 def _banner_of(data: dict) -> str | None:
168 """Bannière de chaîne : nouvel en-tête (imageBannerViewModel) ou ancien.""" 194 """Bannière de chaîne : nouvel en-tête (imageBannerViewModel) ou ancien."""
169 b = _walk(data, "imageBannerViewModel") 195 b = _walk(data, "imageBannerViewModel")
@@ -291,6 +317,10 @@ async def main() -> None:
291 "external_links": about.get("external_links") or [], 317 "external_links": about.get("external_links") or [],
292 "avg_views": (round(sum(views) / len(views)) 318 "avg_views": (round(sum(views) / len(views))
293 if views else None), 319 if views else None),
320+ "videos_per_month": _cadence(vids),
321+ "is_live_now": ('"style":"LIVE"' in text
322+ or '"label":"LIVE"' in text) or None,
323+ "has_shorts": ('"title":"Shorts"' in text) or None,
294 "top_video": (max( 324 "top_video": (max(
295 (v for v in vids if isinstance(v["views"], int)), 325 (v for v in vids if isinstance(v["views"], int)),
296 key=lambda v: v["views"], default=None)), 326 key=lambda v: v["views"], default=None)),
modified actors/ka-youtube/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 # Author: Simon-Pierre Boucher <contact@spboucher.ai> 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 # File: src/net.py 3 # File: src/net.py
4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5−# proxy Apify (résidentiel par défaut), throttling poli + retries 5+# proxy Apify (résidentiel par défaut), throttling poli, retries à
6+# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7+# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8+# GET et POST partagent le même moteur de retries.
6 # ============================================================================== 9 # ==============================================================================
7 from __future__ import annotations 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22+# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23+RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24+
25+
26+def _backoff(attempt: int) -> float:
27+ """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28+ return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29+
19 30
20 class Fetcher: 31 class Fetcher:
21− """GET poli avec proxy Apify + impersonation Chrome + retries 403/429.""" 32+ """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 def __init__(self, proxy_configuration, delay: float = 1.0) -> None: 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 self.proxy_configuration = proxy_configuration 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 await asyncio.sleep(wait) 45 await asyncio.sleep(wait)
35 self._last = asyncio.get_event_loop().time() 46 self._last = asyncio.get_event_loop().time()
36 47
37− async def get(self, url: str, headers: dict | None = None, 48+ async def request(self, method: str, url: str,
38− retries: int = 3, session_id: str | None = None, 49+ headers: dict | None = None, data=None,
39− impersonate: str | None = "chrome"): 50+ retries: int = 4, session_id: str | None = None,
51+ impersonate: str | None = "chrome"):
40 hdrs = {"User-Agent": UA, 52 hdrs = {"User-Agent": UA,
41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 if headers: 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 if self.proxy_configuration: 60 if self.proxy_configuration:
49 sid = session_id or f"s{random.randint(1, 999_999)}" 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63+ # nouvelle session proxy (nouvelle IP) à chaque tentative
51 proxy = await self.proxy_configuration.new_url( 64 proxy = await self.proxy_configuration.new_url(
52 session_id=f"{sid}r{attempt}") 65 session_id=f"{sid}r{attempt}")
53 try: 66 try:
54 resp = await asyncio.to_thread( 67 resp = await asyncio.to_thread(
55− cffi.get, url, headers=hdrs, impersonate=impersonate, 68+ cffi.request, method, url, headers=hdrs, data=data,
56− timeout=60, allow_redirects=True, 69+ impersonate=impersonate, timeout=60, allow_redirects=True,
57 proxies={"http": proxy, "https": proxy} if proxy else None) 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58− if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1: 71+ if resp.status_code in RETRY_STATUSES \
72+ and attempt < retries - 1:
59 Actor.log.warning( 73 Actor.log.warning(
60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}") 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61− await asyncio.sleep(1.5 * (attempt + 1)) 75+ await asyncio.sleep(_backoff(attempt))
62 continue 76 continue
63 return resp 77 return resp
64 except Exception as exc: # réseau/proxy : on retente 78 except Exception as exc: # réseau/proxy : on retente
65 last_exc = exc 79 last_exc = exc
66− await asyncio.sleep(1.5 * (attempt + 1)) 80+ await asyncio.sleep(_backoff(attempt))
67 if last_exc: 81 if last_exc:
68 raise last_exc 82 raise last_exc
69 raise RuntimeError(f"échec après {retries} tentatives : {url}") 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84+
85+ async def get(self, url: str, headers: dict | None = None,
86+ retries: int = 4, session_id: str | None = None,
87+ impersonate: str | None = "chrome"):
88+ return await self.request("GET", url, headers=headers,
89+ retries=retries, session_id=session_id,
90+ impersonate=impersonate)
91+
92+ async def post(self, url: str, data=None, headers: dict | None = None,
93+ retries: int = 4, session_id: str | None = None,
94+ impersonate: str | None = "chrome"):
95+ return await self.request("POST", url, data=data, headers=headers,
96+ retries=retries, session_id=session_id,
97+ impersonate=impersonate)
modified creaka/connectors/apify_social.py +59 −13
Large diff (161 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere