Repository Public
1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File: src/main.py (ka-x)4# Desc: Profils X (Twitter) SANS clé via le service OFFICIEL de syndication5# des widgets (syndication.twitter.com/srv/timeline-profile) : objet6# user complet + timeline récente avec métriques par tweet. Conçu pour7# être appelé sans authentification ; l'empreinte TLS Chrome de8# curl_cffi évite les 429 réservés aux clients python-requests.9# ==============================================================================10from __future__ import annotations1112import asyncio13import json14import re15import time1617from apify import Actor1819from .net import Fetcher2021SYNDICATION_URL = ("https://syndication.twitter.com/srv/timeline-profile/"22 "screen-name/{u}")23_NEXT_DATA_RE = re.compile(24 r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', re.S)252627def _find_user(node, handle: str):28 """Parcourt le JSON Next.js : l'objet user dont screen_name == handle."""29 if isinstance(node, dict):30 if str(node.get("screen_name", "")).lower() == handle and \31 "followers_count" in node:32 return node33 for v in node.values():34 hit = _find_user(v, handle)35 if hit is not None:36 return hit37 elif isinstance(node, list):38 for v in node:39 hit = _find_user(v, handle)40 if hit is not None:41 return hit42 return None434445def parse_tweets(data: dict, handle: str, cap: int) -> list[dict]:46 entries = (((data.get("props") or {}).get("pageProps") or {})47 .get("timeline") or {}).get("entries") or []48 tweets: list[dict] = []49 for entry in entries:50 t = (entry.get("content") or {}).get("tweet") or {}51 if not t.get("id_str"):52 continue53 u = (t.get("user") or {})54 ent = t.get("entities") or {}55 media = (t.get("extended_entities") or ent).get("media") or []56 tweets.append({57 "id": t.get("id_str"),58 "url": f"https://x.com/{u.get('screen_name', handle)}"59 f"/status/{t.get('id_str')}",60 "text": (t.get("full_text") or t.get("text") or "")[:500],61 "likes": t.get("favorite_count"),62 "retweets": t.get("retweet_count"),63 "replies": t.get("reply_count"),64 "quotes": t.get("quote_count"),65 "created_at": t.get("created_at"),66 "is_retweet": bool(t.get("retweeted_status")),67 "is_reply": bool(t.get("in_reply_to_status_id_str")),68 "is_quote": bool(t.get("is_quote_status")),69 "lang": t.get("lang"),70 "hashtags": [h.get("text") for h in (ent.get("hashtags") or [])71 if h.get("text")][:8],72 "mentions": [m.get("screen_name")73 for m in (ent.get("user_mentions") or [])74 if m.get("screen_name")][:8],75 "media_type": (media[0].get("type") if media else None),76 "media_count": len(media) or None,77 "links": [x.get("expanded_url") for x in (ent.get("urls") or [])78 if x.get("expanded_url")][:4],79 })80 if len(tweets) >= cap:81 break82 return tweets838485def _ts(created_at: str) -> float | None:86 """« Tue Sep 12 23:56:24 +0000 2023 » → epoch (None si illisible)."""87 try:88 from datetime import datetime89 return datetime.strptime(created_at,90 "%a %b %d %H:%M:%S %z %Y").timestamp()91 except (TypeError, ValueError):92 return None939495async def main() -> None:96 async with Actor:97 inp = await Actor.get_input() or {}98 usernames = [u.strip().lstrip("@").lower()99 for u in (inp.get("usernames") or []) if u and u.strip()]100 max_tweets = int(inp.get("maxTweets", 20))101 proxy = await Actor.create_proxy_configuration(102 actor_proxy_input=inp.get("proxyConfiguration"))103 fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1))104 sem = asyncio.Semaphore(int(inp.get("concurrency") or 4))105106 async def one(u: str) -> None:107 async with sem:108 miss = {"kind": "profile", "platform": "x",109 "username": u, "found": False}110 try:111 resp = await fetcher.get(SYNDICATION_URL.format(u=u),112 session_id=u, impersonate=None)113 if resp.status_code == 429:114 resp = await fetcher.get(SYNDICATION_URL.format(u=u),115 session_id=f"{u}b",116 impersonate="chrome")117 except Exception as exc:118 await Actor.push_data({**miss, "error": str(exc)[:200]})119 return120 m = _NEXT_DATA_RE.search(resp.text or "")121 if resp.status_code != 200 or not m:122 await Actor.push_data(123 {**miss, "error": f"http_{resp.status_code}"})124 return125 try:126 data = json.loads(m.group(1))127 except Exception:128 await Actor.push_data({**miss, "error": "bad_json"})129 return130 user = _find_user(data, u)131 if not user:132 await Actor.push_data({**miss, "error": "user_not_found"})133 return134 tweets = parse_tweets(data, u, max_tweets)135 own = [t for t in tweets if not t["is_retweet"]]136 likes = [t["likes"] for t in own if isinstance(t["likes"], int)]137 rts = [t["retweets"] for t in own138 if isinstance(t["retweets"], int)]139 reps = [t["replies"] for t in own140 if isinstance(t["replies"], int)]141 followers = user.get("followers_count")142 avg_likes = round(sum(likes) / len(likes)) if likes else None143 avg_rts = round(sum(rts) / len(rts)) if rts else None144 engagement = None145 if followers and avg_likes is not None:146 engagement = round((avg_likes + (avg_rts or 0))147 / followers * 100, 3)148 stamps = sorted(s for s in (_ts(t["created_at"])149 for t in own) if s)150 tweets_per_week = None151 if len(stamps) >= 3 and stamps[-1] > stamps[0]:152 tweets_per_week = round(153 (len(stamps) - 1) / ((stamps[-1] - stamps[0])154 / 604_800), 2)155 top = max((t for t in own if isinstance(t["likes"], int)),156 key=lambda t: t["likes"], default=None)157 hashtags: dict[str, int] = {}158 for t in own:159 for h in t["hashtags"]:160 hashtags[h.lower()] = hashtags.get(h.lower(), 0) + 1161 created_ts = _ts(user.get("created_at"))162 await Actor.push_data({163 "kind": "profile",164 "platform": "x",165 "found": True,166 "id": user.get("id_str"),167 "username": user.get("screen_name"),168 "full_name": user.get("name"),169 "biography": user.get("description"),170 "location": user.get("location"),171 "website": ((user.get("entities") or {}).get("url") or {})172 .get("urls", [{}])[0].get("expanded_url")173 if (user.get("entities") or {}).get("url") else None,174 "followers": followers,175 "following": user.get("friends_count"),176 "tweets_count": user.get("statuses_count"),177 "listed_count": user.get("listed_count"),178 "likes_given": user.get("favourites_count"),179 "is_verified": bool(user.get("verified")180 or user.get("is_blue_verified")),181 "is_blue_verified": user.get("is_blue_verified"),182 "is_protected": user.get("protected"),183 "created_at": user.get("created_at"),184 "account_age_days": (round((time.time() - created_ts)185 / 86_400)186 if created_ts else None),187 "avatar": (user.get("profile_image_url_https") or "")188 .replace("_normal.", "_400x400."),189 "banner": user.get("profile_banner_url"),190 # agrégats sur les tweets récents (hors retweets)191 "avg_likes": avg_likes,192 "avg_retweets": avg_rts,193 "avg_replies": (round(sum(reps) / len(reps))194 if reps else None),195 "engagement_rate_pct": engagement,196 "tweets_per_week": tweets_per_week,197 "last_tweet_at": own[0]["created_at"] if own else None,198 "reply_share_pct": (round(sum(t["is_reply"]199 for t in own)200 / len(own) * 100)201 if own else None),202 "media_share_pct": (round(sum(bool(t["media_count"])203 for t in own)204 / len(own) * 100)205 if own else None),206 "top_hashtags": sorted(hashtags, key=hashtags.get,207 reverse=True)[:8],208 "top_tweet": ({"url": top["url"], "likes": top["likes"],209 "retweets": top["retweets"]}210 if top else None),211 "recent_tweets": tweets,212 })213214 await asyncio.gather(*[one(u) for u in usernames])215 Actor.log.info(f"terminé : {len(usernames)} profils")216