SPB Git forge
spboucher.ai ↗

spb/crea-ka

Public

Créa·Ka — annuaire public cross-plateforme des créateurs de contenu québécois (crea-ka.com)

quebeccreateursinfluenceursannuaireka
52commits 1branches 0releases
11.3 MBsize
maindefault branch
19 days agolast push
Python 73.6% HTML 13.2% TypeScript 6% JavaScript 4.5% CSS 1.7% Dockerfile 0.6%

Upgrade majeur round 2 des 13 acteurs Apify : net.py commun durci (retries 403/407/408/425/429/500/502/503/504, backoff exponentiel + jitter, rotation d'IP par tentative, POST avec retries — Twitch GQL passe par ce moteur) + nouveaux champs par plateforme : IG (bio mentions/hashtags, flag Threads relié → cross-link, business_email public, IGTV), TikTok (likes_given, avg_saves, épinglés, flag mineur isUnderAge18 → is_minor §15), YouTube (videos_per_month, has_shorts, live), X (top_mentions §12.1, avg_quotes, parts liens/RT, verified_type), FB (page_id, website → cross-link, rating), Threads (bio_links → cross-links forts, pk), Twitch (totalCount vidéos, type/début de live), Kick (début/langue/maturité du live), Patreon (+ socials IG/TikTok/Twitch/Discord, published_at), Fansly (tiers/prix, createdAt), Discord (features, premium_tier, expiration), OnlyFans (joinDate, audios), Snapchat (bannière héro, snapcode) — 13 poussés, 12 validés par runs de test ; connecteur : metric_maps étendus, cross-links IG→Threads et bio_links Threads, 51 tests verts

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Simon-Pierre Boucher committed 1 mo ago (Aug 21, 2026) parent 0cad882

27 changed files +732 −157

modified actors/ka-discord/src/main.py +6 −0
@@ -81,6 +81,12 @@ async def main() -> None:
81 81 f"{splash_hash}.jpg?size=1024"
82 82 if gid and splash_hash else None),
83 83 "vanity_url_code": guild.get("vanity_url_code"),
84 + "features": sorted(guild.get("features") or [])[:12],
85 + "nsfw_level": guild.get("nsfw_level"),
86 + "premium_tier": guild.get("premium_tier"),
87 + "invite_expires_at": d.get("expires_at"),
88 + "inviter": ((d.get("inviter") or {}).get("username")
89 + or None),
84 90 })
85 91
86 92 await asyncio.gather(*[one(c) for c in codes])
modified actors/ka-discord/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-facebook/src/main.py +32 −0
@@ -11,6 +11,7 @@ from __future__ import annotations
11 11
12 12 import asyncio
13 13 import html as htmllib
14 +import json
14 15 import re
15 16
16 17 from apify import Actor
@@ -37,6 +38,17 @@ _TEXT_COUNT_RE = re.compile(
37 38 re.I)
38 39 _CATEGORY_RE = re.compile(r'"category_name"\s*:\s*"([^"]+)"')
39 40 _VERIFIED_RE = re.compile(r'"is_verified"\s*:\s*(true|false)')
41 +_PAGE_ID_RES = (
42 + re.compile(r'"page_id"\s*:\s*"?(\d{6,})"?'),
43 + re.compile(r'"pageID"\s*:\s*"(\d{6,})"'),
44 + re.compile(r'"delegate_page_id"\s*:\s*"(\d{6,})"'),
45 +)
46 +# site web auto-déclaré de la page → cross-link côté crea-ka
47 +_WEBSITE_RE = re.compile(
48 + r'"website(?:s)?"\s*:\s*\[?\s*"((?:https?:)?(?:[^"\\]|\\.)+)"')
49 +_RATING_RE = re.compile(r'"overall_star_rating"\s*:\s*\{[^{}]*?'
50 + r'"value"\s*:\s*([\d.]+)')
51 +_OG_URL_RE = re.compile(r'<meta property="og:url" content="([^"]*)"')
40 52
41 53
42 54 def parse_text_count(text: str) -> int | None:
@@ -103,16 +115,36 @@ async def main() -> None:
103 115 return
104 116 cat = _CATEGORY_RE.search(text)
105 117 ver = _VERIFIED_RE.search(text)
118 + page_id = next((m.group(1) for rx in _PAGE_ID_RES
119 + if (m := rx.search(text))), None)
120 + site = _WEBSITE_RE.search(text)
121 + website = None
122 + if site:
123 + try:
124 + website = json.loads(f'"{site.group(1)}"').strip()
125 + except Exception:
126 + website = site.group(1)
127 + if website and not website.startswith("http"):
128 + website = f"https://{website}"
129 + if website and "facebook.com" in website:
130 + website = None # auto-référence sans valeur
131 + rating = _RATING_RE.search(text)
132 + canon = _OG_URL_RE.search(text)
106 133 await Actor.push_data({
107 134 "kind": "profile",
108 135 "platform": "facebook",
109 136 "found": True,
110 137 "username": u,
138 + "page_id": page_id,
111 139 "full_name": og.get("title"),
112 140 "biography": og.get("description"),
113 141 "followers": followers,
114 142 "category": cat.group(1) if cat else None,
115 143 "is_verified": (ver.group(1) == "true") if ver else None,
144 + "website": website,
145 + "rating": float(rating.group(1)) if rating else None,
146 + "canonical_url": (htmllib.unescape(canon.group(1))
147 + if canon else None),
116 148 "avatar": og.get("image"),
117 149 "og_description": og.get("description"),
118 150 })
modified actors/ka-facebook/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-fansly/src/main.py +7 −0
@@ -61,6 +61,9 @@ async def main() -> None:
61 61 continue
62 62 found.add(u)
63 63 stats = acc.get("timelineStats") or {}
64 + tiers = [{"name": t.get("name"), "price": t.get("price")}
65 + for t in (acc.get("subscriptionTiers") or [])
66 + if isinstance(t, dict) and t.get("name")]
64 67 await Actor.push_data({
65 68 "kind": "profile",
66 69 "platform": "fansly",
@@ -82,6 +85,10 @@ async def main() -> None:
82 85 "avatar": _loc(acc.get("avatar")),
83 86 "banner": _loc(acc.get("banner")),
84 87 "location": acc.get("location"),
88 + "account_created_at": acc.get("createdAt"),
89 + "subscription_tiers": tiers[:5] or None,
90 + "subscription_price": (tiers[0].get("price")
91 + if tiers else None),
85 92 })
86 93 for u in batch:
87 94 if u not in found:
modified actors/ka-fansly/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-instagram/src/main.py +21 −0
@@ -90,6 +90,15 @@ def parse_user(user: dict) -> dict:
90 90 posts = [parse_post((e.get("node") or {}))
91 91 for e in ((user.get("edge_owner_to_timeline_media") or {})
92 92 .get("edges") or [])]
93 + bio_ent = user.get("biography_with_entities") or {}
94 + bio_mentions, bio_hashtags = [], []
95 + for ent in (bio_ent.get("entities") or []):
96 + eu = ((ent or {}).get("user") or {}).get("username")
97 + eh = ((ent or {}).get("hashtag") or {}).get("name")
98 + if eu:
99 + bio_mentions.append(eu)
100 + if eh:
101 + bio_hashtags.append(eh)
93 102 followers = (user.get("edge_followed_by") or {}).get("count")
94 103 avg_likes = _avg([p["likes"] for p in posts])
95 104 avg_comments = _avg([p["comments"] for p in posts])
@@ -123,6 +132,8 @@ def parse_user(user: dict) -> dict:
123 132 "username": user.get("username"),
124 133 "full_name": user.get("full_name"),
125 134 "biography": user.get("biography"),
135 + "bio_mentions": bio_mentions[:10], # @comptes cités dans la bio
136 + "bio_hashtags": bio_hashtags[:10],
126 137 "pronouns": user.get("pronouns") or None,
127 138 "external_url": user.get("external_url"),
128 139 "bio_links": [b.get("url") for b in (user.get("bio_links") or [])
@@ -131,6 +142,10 @@ def parse_user(user: dict) -> dict:
131 142 "following": (user.get("edge_follow") or {}).get("count"),
132 143 "posts_count": (user.get("edge_owner_to_timeline_media")
133 144 or {}).get("count"),
145 + "igtv_videos_count": (user.get("edge_felix_video_timeline")
146 + or {}).get("count"),
147 + "mutual_followers": (user.get("edge_mutual_followed_by")
148 + or {}).get("count"),
134 149 "highlight_reels": user.get("highlight_reel_count"),
135 150 "is_verified": user.get("is_verified"),
136 151 "is_private": user.get("is_private"),
@@ -139,6 +154,12 @@ def parse_user(user: dict) -> dict:
139 154 "category": user.get("category_name"),
140 155 "business_category": user.get("business_category_name"),
141 156 "business_address": user.get("business_address_json"),
157 + # courriel PRO affiché publiquement par le créateur (§15 : public only)
158 + "business_email": user.get("business_email") or None,
159 + # présent = compte Threads relié (cross-link fort, même handle)
160 + "has_threads": (bool(user.get("has_onboarded_to_text_post_app"))
161 + if "has_onboarded_to_text_post_app" in user
162 + else None),
142 163 "has_clips": user.get("has_clips"),
143 164 "has_channel": user.get("has_channel"),
144 165 "has_ar_effects": user.get("has_ar_effects"),
modified actors/ka-instagram/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-kick/src/main.py +5 −0
@@ -82,6 +82,11 @@ async def main() -> None:
82 82 "subscription_enabled": ch.get("subscription_enabled"),
83 83 "is_live_now": bool(live),
84 84 "live_viewers": live.get("viewer_count"),
85 + "live_started_at": (live.get("start_time")
86 + or live.get("created_at")),
87 + "live_is_mature": (bool(live.get("is_mature"))
88 + if live else None),
89 + "live_language": live.get("language"),
85 90 "live_title": (live.get("session_title") or "")[:200]
86 91 or None,
87 92 "live_thumbnail": ((live.get("thumbnail") or {}).get("url")
modified actors/ka-kick/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-onlyfans/src/main.py +8 −0
@@ -26,7 +26,11 @@ _NUM_RES = {
26 26 "videos_count": re.compile(r'"videosCount"\s*:\s*(\d+)'),
27 27 "likes": re.compile(r'"favoritedCount"\s*:\s*(\d+)'),
28 28 "streams_count": re.compile(r'"finishedStreamsCount"\s*:\s*(\d+)'),
29 + "audios_count": re.compile(r'"audiosCount"\s*:\s*(\d+)'),
30 + "medias_count": re.compile(r'"mediasCount"\s*:\s*(\d+)'),
29 31 }
32 +_JOIN_RE = re.compile(r'"joinDate"\s*:\s*"((?:[^"\\]|\\.)*)"')
33 +_PERFORMER_RE = re.compile(r'"isPerformer"\s*:\s*(true|false)')
30 34 _PRICE_RE = re.compile(r'"subscribePrice"\s*:\s*([\d.]+)')
31 35 _STR_RES = {
32 36 "full_name": re.compile(r'"name"\s*:\s*"((?:[^"\\]|\\.)*)"'),
@@ -99,6 +103,10 @@ async def main() -> None:
99 103 else None)
100 104 m = _VERIFIED_RE.search(text)
101 105 rec["is_verified"] = (m.group(1) == "true") if m else None
106 + m = _JOIN_RE.search(text)
107 + rec["join_date"] = _dec(m.group(1)) if m else None
108 + m = _PERFORMER_RE.search(text)
109 + rec["is_performer"] = (m.group(1) == "true") if m else None
102 110 if not rec.get("full_name"):
103 111 m = _OG_TITLE_RE.search(text)
104 112 if m:
modified actors/ka-onlyfans/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-patreon/src/main.py +12 −0
@@ -34,7 +34,13 @@ _SOCIALS = {
34 34 "facebook": re.compile(r'"facebook"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
35 35 "twitter": re.compile(r'"twitter"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
36 36 "youtube": re.compile(r'"youtube"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
37 + "instagram": re.compile(r'"instagram"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
38 + "tiktok": re.compile(r'"tiktok"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
39 + "twitch": re.compile(r'"twitch"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
40 + "discord": re.compile(r'"discord"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"'),
37 41 }
42 +_PUBLISHED_RE = re.compile(r'"published_at"\s*:\s*"([^"]+)"')
43 +_PPN_RE = re.compile(r'"pay_per_name"\s*:\s*"([^"]+)"')
38 44
39 45
40 46 def _dec(raw: str) -> str:
@@ -103,6 +109,12 @@ async def main() -> None:
103 109 "is_monthly": (monthly.group(1) == "true"
104 110 if monthly else None),
105 111 "is_nsfw": nsfw.group(1) == "true" if nsfw else None,
112 + "published_at": (pub.group(1) if
113 + (pub := _PUBLISHED_RE.search(text))
114 + else None),
115 + "pay_per_name": (ppn.group(1)[:40] if
116 + (ppn := _PPN_RE.search(text))
117 + else None),
106 118 "creation_name": (_dec(creation.group(1))[:200]
107 119 if creation else None),
108 120 "avatar": htmllib.unescape(img.group(1)) if img else None,
modified actors/ka-patreon/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-snapchat/src/main.py +3 −0
@@ -103,6 +103,9 @@ async def main() -> None:
103 103 "website": info.get("websiteUrl"),
104 104 "address": info.get("address"),
105 105 "avatar": info.get("profilePictureUrl"),
106 + "banner": (info.get("squareHeroImageUrl")
107 + or info.get("heroImageUrl") or None),
108 + "snapcode": info.get("snapcodeImageUrl") or None,
106 109 "has_story": bool(info.get("hasStory")),
107 110 "has_spotlight": bool(info.get("hasSpotlightHighlights")),
108 111 "has_curated_highlights": bool(
modified actors/ka-snapchat/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-threads/src/main.py +14 −0
@@ -24,6 +24,10 @@ _BIO_RE = re.compile(r'"biography"\s*:\s*"((?:[^"\\]|\\.)*)"')
24 24 _NAME_RE = re.compile(r'"full_name"\s*:\s*"((?:[^"\\]|\\.)*)"')
25 25 _VERIFIED_RE = re.compile(r'"is_verified"\s*:\s*(true|false)')
26 26 _PIC_RE = re.compile(r'"profile_pic_url"\s*:\s*"((?:[^"\\]|\\.)*)"')
27 +# liens auto-déclarés de la bio → cross-links forts côté crea-ka (§12.1)
28 +_BIO_LINKS_RE = re.compile(r'"bio_links"\s*:\s*\[(.{0,2000}?)\]', re.S)
29 +_LINK_URL_RE = re.compile(r'"url"\s*:\s*"(https?:(?:[^"\\]|\\.)+)"')
30 +_PK_RE = re.compile(r'"pk"\s*:\s*"?(\d{4,})"?')
27 31 # posts : paires texte + like_count dans les payloads thread_items
28 32 _POST_RE = re.compile(
29 33 r'"caption"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"[^{}]*?\}'
@@ -69,6 +73,14 @@ async def main() -> None:
69 73 name = _NAME_RE.search(text)
70 74 ver = _VERIFIED_RE.search(text)
71 75 pic = _PIC_RE.search(text)
76 + pk = _PK_RE.search(text)
77 + bio_links: list[str] = []
78 + bl = _BIO_LINKS_RE.search(text)
79 + if bl:
80 + for m2 in _LINK_URL_RE.finditer(bl.group(1)):
81 + url2 = _dec(m2.group(1))
82 + if url2 not in bio_links:
83 + bio_links.append(url2)
72 84 likes = [p["likes"] for p in posts]
73 85 top = max(posts, key=lambda p: p["likes"], default=None)
74 86 await Actor.push_data({
@@ -76,8 +88,10 @@ async def main() -> None:
76 88 "platform": "threads",
77 89 "found": True,
78 90 "username": u,
91 + "id": pk.group(1) if pk else None,
79 92 "full_name": _dec(name.group(1)) if name else None,
80 93 "biography": _dec(bio.group(1)) if bio else None,
94 + "bio_links": bio_links[:5] or None,
81 95 "followers": int(fol.group(1)),
82 96 "is_verified": (ver.group(1) == "true") if ver else None,
83 97 "avatar": _dec(pic.group(1)) if pic else None,
modified actors/ka-threads/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-tiktok/src/main.py +12 −0
@@ -138,6 +138,8 @@ async def main() -> None:
138 138 if isinstance(v["comments"], int)]
139 139 shares = [v["shares"] for v in videos
140 140 if isinstance(v["shares"], int)]
141 + saves = [v["saves"] for v in videos
142 + if isinstance(v["saves"], int)]
141 143 avg_views = round(sum(views) / len(views)) if views else None
142 144 avg_likes = round(sum(likes) / len(likes)) if likes else None
143 145 engagement = None
@@ -187,6 +189,7 @@ async def main() -> None:
187 189 "followers": followers,
188 190 "following": _num(stats.get("followingCount")),
189 191 "friends": _num(stats.get("friendCount")),
192 + "likes_given": _num(stats.get("diggCount")),
190 193 "total_likes": total_likes,
191 194 "videos_count": videos_count,
192 195 "avg_likes_per_video_lifetime": (
@@ -196,6 +199,11 @@ async def main() -> None:
196 199 "is_organization": bool(user.get("isOrganization")),
197 200 "is_seller": bool(user.get("ttSeller")),
198 201 "is_live_now": bool(user.get("roomId")),
202 + # drapeau mineur déclaré par TikTok → régime restreint §15
203 + "is_under_18": (bool(user.get("isUnderAge18"))
204 + if "isUnderAge18" in user else None),
205 + "is_embed_banned": (bool(user.get("isEmbedBanned"))
206 + if "isEmbedBanned" in user else None),
199 207 "commerce_category": (commerce.get("category") or None
200 208 if commerce.get("commerceUser")
201 209 else None),
@@ -211,6 +219,10 @@ async def main() -> None:
211 219 if comments else None),
212 220 "avg_shares": (round(sum(shares) / len(shares))
213 221 if shares else None),
222 + "avg_saves": (round(sum(saves) / len(saves))
223 + if saves else None),
224 + "pinned_videos_count": (sum(1 for v in videos
225 + if v["is_pinned"]) or None),
214 226 "engagement_rate_pct": engagement,
215 227 "videos_per_week": videos_per_week,
216 228 "last_video_at": last_video_at,
modified actors/ka-tiktok/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-twitch/src/main.py +10 −14
@@ -35,10 +35,11 @@ query($login: String!) {
35 35 primaryTeam { displayName }
36 36 channel { socialMedias { name title url } }
37 37 lastBroadcast { startedAt title game { displayName } }
38 − stream { viewersCount createdAt title
38 + stream { viewersCount createdAt title type
39 39 previewImageURL(width: 640, height: 360)
40 40 game { displayName } }
41 41 videos(first: 10, sort: TIME) {
42 + totalCount
42 43 edges { node {
43 44 id title viewCount lengthSeconds publishedAt
44 45 previewThumbnailURL(width: 320, height: 180)
@@ -60,23 +61,14 @@ async def main() -> None:
60 61 fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1))
61 62 sem = asyncio.Semaphore(int(inp.get("concurrency") or 4))
62 63
63 − import random
64 − from curl_cffi import requests as cffi
65 −
66 64 async def gql(login: str) -> dict | None:
67 65 body = json.dumps({"query": QUERY,
68 66 "variables": {"login": login}})
69 − p = None
70 − if fetcher.proxy_configuration:
71 − p = await fetcher.proxy_configuration.new_url(
72 − session_id=f"tw{random.randint(1, 999_999)}")
73 − await fetcher._throttle()
74 − resp = await asyncio.to_thread(
75 − cffi.post, GQL_URL, data=body,
67 + # fetcher.post = retries/backoff/rotation d'IP du net.py commun
68 + resp = await fetcher.post(
69 + GQL_URL, data=body, session_id=f"tw{login}",
76 70 headers={"Client-ID": CLIENT_ID, "User-Agent": UA,
77 − "Content-Type": "application/json"},
78 − impersonate="chrome", timeout=45,
79 − proxies={"http": p, "https": p} if p else None)
71 + "Content-Type": "application/json"})
80 72 if resp.status_code != 200:
81 73 raise RuntimeError(f"gql http_{resp.status_code}")
82 74 return (resp.json().get("data") or {}).get("user")
@@ -131,6 +123,8 @@ async def main() -> None:
131 123 "avatar": user.get("profileImageURL"),
132 124 "banner": user.get("bannerImageURL"),
133 125 "is_live_now": bool(stream),
126 + "live_type": stream.get("type"),
127 + "live_started_at": stream.get("createdAt"),
134 128 "live_viewers": stream.get("viewersCount"),
135 129 "live_title": (stream.get("title") or "")[:200] or None,
136 130 "live_thumbnail": stream.get("previewImageURL"),
@@ -140,6 +134,8 @@ async def main() -> None:
140 134 or None,
141 135 "last_broadcast_game": ((last.get("game") or {})
142 136 .get("displayName")),
137 + "videos_count": ((user.get("videos") or {})
138 + .get("totalCount")),
143 139 "avg_video_views": (round(sum(views) / len(views))
144 140 if views else None),
145 141 "recent_games": list(dict.fromkeys(games))[:5],
modified actors/ka-twitch/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-x/src/main.py +19 −0
@@ -163,9 +163,14 @@ async def main() -> None:
163 163 top = max((t for t in own if isinstance(t["likes"], int)),
164 164 key=lambda t: t["likes"], default=None)
165 165 hashtags: dict[str, int] = {}
166 + mentions: dict[str, int] = {}
166 167 for t in own:
167 168 for h in t["hashtags"]:
168 169 hashtags[h.lower()] = hashtags.get(h.lower(), 0) + 1
170 + for mn in t["mentions"]:
171 + mentions[mn.lower()] = mentions.get(mn.lower(), 0) + 1
172 + quotes = [t["quotes"] for t in own
173 + if isinstance(t["quotes"], int)]
169 174 created_ts = _ts(user.get("created_at"))
170 175 await Actor.push_data({
171 176 "kind": "profile",
@@ -187,6 +192,7 @@ async def main() -> None:
187 192 "is_verified": bool(user.get("verified")
188 193 or user.get("is_blue_verified")),
189 194 "is_blue_verified": user.get("is_blue_verified"),
195 + "verified_type": user.get("verified_type"),
190 196 "is_protected": user.get("protected"),
191 197 "created_at": user.get("created_at"),
192 198 "account_age_days": (round((time.time() - created_ts)
@@ -200,6 +206,8 @@ async def main() -> None:
200 206 "avg_retweets": avg_rts,
201 207 "avg_replies": (round(sum(reps) / len(reps))
202 208 if reps else None),
209 + "avg_quotes": (round(sum(quotes) / len(quotes))
210 + if quotes else None),
203 211 "engagement_rate_pct": engagement,
204 212 "tweets_per_week": tweets_per_week,
205 213 "last_tweet_at": own[0]["created_at"] if own else None,
@@ -211,8 +219,19 @@ async def main() -> None:
211 219 for t in own)
212 220 / len(own) * 100)
213 221 if own else None),
222 + "link_share_pct": (round(sum(bool(t["links"])
223 + for t in own)
224 + / len(own) * 100)
225 + if own else None),
226 + "retweet_share_pct": (round(sum(t["is_retweet"]
227 + for t in tweets)
228 + / len(tweets) * 100)
229 + if tweets else None),
214 230 "top_hashtags": sorted(hashtags, key=hashtags.get,
215 231 reverse=True)[:8],
232 + # comptes les plus mentionnés → signal d'identité §12.1
233 + "top_mentions": sorted(mentions, key=mentions.get,
234 + reverse=True)[:8],
216 235 "top_tweet": ({"url": top["url"], "likes": top["likes"],
217 236 "retweets": top["retweets"]}
218 237 if top else None),
modified actors/ka-x/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified actors/ka-youtube/src/main.py +30 −0
@@ -164,6 +164,32 @@ def _s(v):
164 164 return v.get("content") if isinstance(v, dict) else v
165 165
166 166
167 +_REL_RE = re.compile(
168 + r"(\d+)\s*(hour|day|week|month|year|heure|jour|semaine|mois|an)", re.I)
169 +_REL_DAYS = {"hour": 1 / 24, "heure": 1 / 24, "day": 1, "jour": 1,
170 + "week": 7, "semaine": 7, "month": 30.44, "mois": 30.44,
171 + "year": 365.25, "an": 365.25}
172 +
173 +
174 +def _rel_days(published: str | None) -> float | None:
175 + """« 3 weeks ago » / « il y a 2 mois » → ancienneté approx. en jours."""
176 + m = _REL_RE.search(published or "")
177 + if not m:
178 + return None
179 + unit = m.group(2).lower()
180 + unit = next((k for k in _REL_DAYS if unit.startswith(k)), None)
181 + return float(m.group(1)) * _REL_DAYS[unit] if unit else None
182 +
183 +
184 +def _cadence(vids: list[dict]) -> float | None:
185 + """Vidéos/mois estimées d'après les dates relatives (approximatif)."""
186 + ages = sorted(a for a in (_rel_days(v.get("published")) for v in vids)
187 + if a is not None)
188 + if len(ages) < 3 or ages[-1] <= ages[0]:
189 + return None
190 + return round((len(ages) - 1) / ((ages[-1] - ages[0]) / 30.44), 2)
191 +
192 +
167 193 def _banner_of(data: dict) -> str | None:
168 194 """Bannière de chaîne : nouvel en-tête (imageBannerViewModel) ou ancien."""
169 195 b = _walk(data, "imageBannerViewModel")
@@ -291,6 +317,10 @@ async def main() -> None:
291 317 "external_links": about.get("external_links") or [],
292 318 "avg_views": (round(sum(views) / len(views))
293 319 if views else None),
320 + "videos_per_month": _cadence(vids),
321 + "is_live_now": ('"style":"LIVE"' in text
322 + or '"label":"LIVE"' in text) or None,
323 + "has_shorts": ('"title":"Shorts"' in text) or None,
294 324 "top_video": (max(
295 325 (v for v in vids if isinstance(v["views"], int)),
296 326 key=lambda v: v["views"], default=None)),
modified actors/ka-youtube/src/net.py +38 −10
@@ -2,7 +2,10 @@
2 2 # Author: Simon-Pierre Boucher <contact@spboucher.ai>
3 3 # File: src/net.py
4 4 # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +
5 −# proxy Apify (résidentiel par défaut), throttling poli + retries
5 +# proxy Apify (résidentiel par défaut), throttling poli, retries à
6 +# backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504
7 +# avec rotation de session proxy (nouvelle IP) à chaque tentative.
8 +# GET et POST partagent le même moteur de retries.
6 9 # ==============================================================================
7 10 from __future__ import annotations
8 11
@@ -16,9 +19,17 @@ from curl_cffi import requests as cffi
16 19 UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
17 20 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
18 21
22 +# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur
23 +RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})
24 +
25 +
26 +def _backoff(attempt: int) -> float:
27 + """Backoff exponentiel plafonné + jitter (anti-troupeau)."""
28 + return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)
29 +
19 30
20 31 class Fetcher:
21 − """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""
32 + """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""
22 33
23 34 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:
24 35 self.proxy_configuration = proxy_configuration
@@ -34,9 +45,10 @@ class Fetcher:
34 45 await asyncio.sleep(wait)
35 46 self._last = asyncio.get_event_loop().time()
36 47
37 − async def get(self, url: str, headers: dict | None = None,
38 − retries: int = 3, session_id: str | None = None,
39 − impersonate: str | None = "chrome"):
48 + async def request(self, method: str, url: str,
49 + headers: dict | None = None, data=None,
50 + retries: int = 4, session_id: str | None = None,
51 + impersonate: str | None = "chrome"):
40 52 hdrs = {"User-Agent": UA,
41 53 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
42 54 if headers:
@@ -48,22 +60,38 @@ class Fetcher:
48 60 if self.proxy_configuration:
49 61 sid = session_id or f"s{random.randint(1, 999_999)}"
50 62 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"
63 + # nouvelle session proxy (nouvelle IP) à chaque tentative
51 64 proxy = await self.proxy_configuration.new_url(
52 65 session_id=f"{sid}r{attempt}")
53 66 try:
54 67 resp = await asyncio.to_thread(
55 − cffi.get, url, headers=hdrs, impersonate=impersonate,
56 − timeout=60, allow_redirects=True,
68 + cffi.request, method, url, headers=hdrs, data=data,
69 + impersonate=impersonate, timeout=60, allow_redirects=True,
57 70 proxies={"http": proxy, "https": proxy} if proxy else None)
58 − if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:
71 + if resp.status_code in RETRY_STATUSES \
72 + and attempt < retries - 1:
59 73 Actor.log.warning(
60 74 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")
61 − await asyncio.sleep(1.5 * (attempt + 1))
75 + await asyncio.sleep(_backoff(attempt))
62 76 continue
63 77 return resp
64 78 except Exception as exc: # réseau/proxy : on retente
65 79 last_exc = exc
66 − await asyncio.sleep(1.5 * (attempt + 1))
80 + await asyncio.sleep(_backoff(attempt))
67 81 if last_exc:
68 82 raise last_exc
69 83 raise RuntimeError(f"échec après {retries} tentatives : {url}")
84 +
85 + async def get(self, url: str, headers: dict | None = None,
86 + retries: int = 4, session_id: str | None = None,
87 + impersonate: str | None = "chrome"):
88 + return await self.request("GET", url, headers=headers,
89 + retries=retries, session_id=session_id,
90 + impersonate=impersonate)
91 +
92 + async def post(self, url: str, data=None, headers: dict | None = None,
93 + retries: int = 4, session_id: str | None = None,
94 + impersonate: str | None = "chrome"):
95 + return await self.request("POST", url, data=data, headers=headers,
96 + retries=retries, session_id=session_id,
97 + impersonate=impersonate)
modified creaka/connectors/apify_social.py +59 −13
@@ -331,7 +331,11 @@ class ApifyInstagram(_ApifySocialEnrich):
331 331 "posts_per_week": "posts_per_week",
332 332 "last_post_at": "last_post_at",
333 333 "video_share_pct": "video_share_pct",
334 − "top_post": "top_post"}
334 + "top_post": "top_post",
335 + "bio_mentions": "bio_mentions",
336 + "bio_hashtags": "bio_hashtags",
337 + "igtv_videos_count": "igtv_videos",
338 + "business_email": "business_email"}
335 339
336 340 def bio_urls(self, it: dict) -> list[str]:
337 341 return [it.get("external_url") or ""] + (it.get("bio_links") or [])
@@ -342,6 +346,12 @@ class ApifyInstagram(_ApifySocialEnrich):
342 346 or []) if r.get("username")]
343 347 if related: # matière première de découvertes futures (§12.1 mention)
344 348 acc.metrics["related_profiles"] = related[:10]
349 + # flag Meta « compte Threads relié » : même handle → cross_link fort
350 + if it.get("has_threads") and it.get("username"):
351 + u = it["username"]
352 + cr.platforms = merge_accounts(cr.platforms, [
353 + account("threads", u, "cross_link",
354 + url=f"https://www.threads.com/@{u}").finalize()])
345 355
346 356
347 357 class ApifyTikTok(_ApifySocialEnrich):
@@ -362,11 +372,19 @@ class ApifyTikTok(_ApifySocialEnrich):
362 372 "videos_per_week": "videos_per_week",
363 373 "last_video_at": "last_video_at",
364 374 "top_hashtags": "top_hashtags", "language": "language",
365 − "top_video": "top_video"}
375 + "top_video": "top_video",
376 + "likes_given": "likes_given", "avg_saves": "avg_saves",
377 + "pinned_videos_count": "pinned_videos"}
366 378
367 379 def bio_urls(self, it: dict) -> list[str]:
368 380 return [it.get("bio_link") or ""]
369 381
382 + def apply(self, cr, acc, it):
383 + super().apply(cr, acc, it)
384 + # drapeau mineur DÉCLARÉ par TikTok → régime restreint §15
385 + if it.get("is_under_18") is True:
386 + cr.is_minor = True
387 +
370 388
371 389 class ApifyX(_ApifySocialEnrich):
372 390 source_id = "x-apify"
@@ -389,7 +407,11 @@ class ApifyX(_ApifySocialEnrich):
389 407 "reply_share_pct": "reply_share_pct",
390 408 "is_blue_verified": "is_blue_verified",
391 409 "likes_given": "likes_given", "website": "website",
392 − "top_hashtags": "top_hashtags", "top_tweet": "top_tweet"}
410 + "top_hashtags": "top_hashtags", "top_tweet": "top_tweet",
411 + "avg_quotes": "avg_quotes", "verified_type": "verified_type",
412 + "link_share_pct": "link_share_pct",
413 + "retweet_share_pct": "retweet_share_pct",
414 + "top_mentions": "top_mentions"}
393 415
394 416 def extra_input(self) -> dict:
395 417 return {"maxTweets": 20}
@@ -405,7 +427,12 @@ class ApifyFacebook(_ApifySocialEnrich):
405 427 run_concurrency = 6 # maximum du schéma d'input de ka-facebook
406 428 cap = 400
407 429 revisit_days = 7
408 − metric_map = {"category": "category"}
430 + metric_map = {"category": "category", "page_id": "page_id",
431 + "website": "website", "rating": "rating"}
432 +
433 + def bio_urls(self, it: dict) -> list[str]:
434 + # site web auto-déclaré de la page → cross_link (§12.1)
435 + return [it.get("website") or ""]
409 436
410 437
411 438 class ApifyThreads(_ApifySocialEnrich):
@@ -416,7 +443,12 @@ class ApifyThreads(_ApifySocialEnrich):
416 443 cap = 60
417 444 revisit_days = 7
418 445 content_key = "recent_posts"
419 − metric_map = {"avg_likes": "avg_likes", "top_post": "top_post"}
446 + metric_map = {"avg_likes": "avg_likes", "top_post": "top_post",
447 + "bio_links": "bio_links"}
448 +
449 + def bio_urls(self, it: dict) -> list[str]:
450 + # liens de bio AUTO-DÉCLARÉS Threads → cross_link fort (§12.1)
451 + return list(it.get("bio_links") or [])
420 452
421 453
422 454 class ApifySnapchat(_ApifySocialEnrich):
@@ -433,7 +465,7 @@ class ApifySnapchat(_ApifySocialEnrich):
433 465 "lenses_count": "lenses",
434 466 "story_previews": "story_previews",
435 467 "spotlight_previews": "spotlight_previews",
436 − "website": "website",
468 + "website": "website", "snapcode": "snapcode",
437 469 "spotlight_highlights_count": "spotlight_highlights"}
438 470
439 471 def bio_urls(self, it: dict) -> list[str]:
@@ -451,7 +483,9 @@ class ApifyYouTube(_ApifySocialEnrich):
451 483 "avg_views": "avg_views", "top_video": "top_video",
452 484 "total_views": "total_views", "joined_date": "joined_date",
453 485 "external_links": "external_links",
454 − "last_video_published": "last_video_published"}
486 + "last_video_published": "last_video_published",
487 + "videos_per_month": "videos_per_month",
488 + "has_shorts": "has_shorts", "is_live_now": "is_live_now"}
455 489
456 490 def bio_urls(self, it: dict) -> list[str]:
457 491 # liens externes AUTO-DÉCLARÉS de la page À propos → cross_link (§12.1)
@@ -476,7 +510,10 @@ class ApifyTwitch(_ApifySocialEnrich):
476 510 "live_thumbnail": "live_thumbnail",
477 511 "avg_video_views": "avg_video_views",
478 512 "social_links": "social_links",
479 − "recent_games": "recent_games"}
513 + "recent_games": "recent_games",
514 + "videos_count": "videos",
515 + "live_type": "live_type",
516 + "live_started_at": "live_started_at"}
480 517
481 518 def bio_urls(self, it: dict) -> list[str]:
482 519 # panneau « À propos » Twitch : liens sociaux AUTO-DÉCLARÉS (§12.1)
@@ -495,7 +532,9 @@ class ApifyKick(_ApifySocialEnrich):
495 532 "live_thumbnail": "live_thumbnail",
496 533 "live_category": "live_category",
497 534 "vod_enabled": "vod_enabled",
498 − "recent_categories": "recent_categories"}
535 + "recent_categories": "recent_categories",
536 + "live_started_at": "live_started_at",
537 + "live_language": "live_language"}
499 538
500 539 def cross_links(self, cr: Creator, it: dict) -> None:
501 540 """Liens sociaux AUTO-DÉCLARÉS de la fiche Kick → cross_link (§12.1)."""
@@ -529,7 +568,8 @@ class ApifyOnlyFans(_ApifySocialEnrich):
529 568 "videos_count": "videos", "likes": "likes",
530 569 "streams_count": "streams",
531 570 "subscribe_price_usd": "subscribe_price_usd",
532 − "is_free": "is_free", "location": "location"}
571 + "is_free": "is_free", "location": "location",
572 + "join_date": "join_date", "audios_count": "audios"}
533 573
534 574 def bio_urls(self, it: dict) -> list[str]:
535 575 return [it.get("website") or ""]
@@ -543,7 +583,9 @@ class ApifyFansly(_ApifySocialEnrich):
543 583 revisit_days = 7
544 584 metric_map = {"posts_count": "posts", "images_count": "images",
545 585 "videos_count": "videos", "likes": "likes",
546 − "following": "following", "location": "location"}
586 + "following": "following", "location": "location",
587 + "subscription_price": "subscription_price",
588 + "account_created_at": "account_created_at"}
547 589
548 590
549 591 class ApifyPatreon(_ApifySocialEnrich):
@@ -555,7 +597,9 @@ class ApifyPatreon(_ApifySocialEnrich):
555 597 metric_map = {"patrons": "patrons", "posts_count": "posts",
556 598 "is_monthly": "is_monthly", "is_nsfw": "is_nsfw",
557 599 "creation_name": "creation_name",
558 − "social_links": "social_links"}
600 + "social_links": "social_links",
601 + "published_at": "published_at",
602 + "pay_per_name": "pay_per_name"}
559 603
560 604 def bio_urls(self, it: dict) -> list[str]:
561 605 # liens sociaux AUTO-DÉCLARÉS de la campagne Patreon → cross_link
@@ -571,7 +615,9 @@ class ApifyDiscord(_ApifySocialEnrich):
571 615 metric_map = {"members": "members", "online": "online", "boosts": "boosts",
572 616 "partnered": "partnered", "guild_id": "guild_id",
573 617 "channel": "channel", "splash": "splash",
574 − "vanity_url_code": "vanity_url_code"}
618 + "vanity_url_code": "vanity_url_code",
619 + "premium_tier": "premium_tier",
620 + "nsfw_level": "nsfw_level"}
575 621
576 622 def target_of(self, acc) -> str:
577 623 """Code d'invitation SENSIBLE À LA CASSE, repris de l'URL (pas du
578 624