spb/crea-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   src/main.py (ka-x)4# Desc:   Profils X (Twitter) SANS clé via le service OFFICIEL de syndication5#         des widgets (syndication.twitter.com/srv/timeline-profile) : objet6#         user complet + timeline récente avec métriques par tweet. Conçu pour7#         être appelé sans authentification ; l'empreinte TLS Chrome de8#         curl_cffi évite les 429 réservés aux clients python-requests.9# ==============================================================================10from __future__ import annotations1112import asyncio13import json14import re15import time1617from apify import Actor1819from .net import Fetcher2021SYNDICATION_URL = ("https://syndication.twitter.com/srv/timeline-profile/"22                   "screen-name/{u}")23_NEXT_DATA_RE = re.compile(24    r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', re.S)252627def _find_user(node, handle: str):28    """Parcourt le JSON Next.js : l'objet user dont screen_name == handle."""29    if isinstance(node, dict):30        if str(node.get("screen_name", "")).lower() == handle and \31                "followers_count" in node:32            return node33        for v in node.values():34            hit = _find_user(v, handle)35            if hit is not None:36                return hit37    elif isinstance(node, list):38        for v in node:39            hit = _find_user(v, handle)40            if hit is not None:41                return hit42    return None434445def parse_tweets(data: dict, handle: str, cap: int) -> list[dict]:46    entries = (((data.get("props") or {}).get("pageProps") or {})47               .get("timeline") or {}).get("entries") or []48    tweets: list[dict] = []49    for entry in entries:50        t = (entry.get("content") or {}).get("tweet") or {}51        if not t.get("id_str"):52            continue53        u = (t.get("user") or {})54        ent = t.get("entities") or {}55        media = (t.get("extended_entities") or ent).get("media") or []56        tweets.append({57            "id": t.get("id_str"),58            "url": f"https://x.com/{u.get('screen_name', handle)}"59                   f"/status/{t.get('id_str')}",60            "text": (t.get("full_text") or t.get("text") or "")[:500],61            "likes": t.get("favorite_count"),62            "retweets": t.get("retweet_count"),63            "replies": t.get("reply_count"),64            "quotes": t.get("quote_count"),65            "created_at": t.get("created_at"),66            "is_retweet": bool(t.get("retweeted_status")),67            "is_reply": bool(t.get("in_reply_to_status_id_str")),68            "is_quote": bool(t.get("is_quote_status")),69            "lang": t.get("lang"),70            "hashtags": [h.get("text") for h in (ent.get("hashtags") or [])71                         if h.get("text")][:8],72            "mentions": [m.get("screen_name")73                         for m in (ent.get("user_mentions") or [])74                         if m.get("screen_name")][:8],75            "media_type": (media[0].get("type") if media else None),76            "media_count": len(media) or None,77            "media_urls": [x.get("media_url_https") for x in media78                           if x.get("media_url_https")][:4],79            "links": [x.get("expanded_url") for x in (ent.get("urls") or [])80                      if x.get("expanded_url")][:4],81        })82        if len(tweets) >= cap:83            break84    return tweets858687def _ts(created_at: str) -> float | None:88    """« Tue Sep 12 23:56:24 +0000 2023 » → epoch (None si illisible)."""89    try:90        from datetime import datetime91        return datetime.strptime(created_at,92                                 "%a %b %d %H:%M:%S %z %Y").timestamp()93    except (TypeError, ValueError):94        return None959697async def main() -> None:98    async with Actor:99        inp = await Actor.get_input() or {}100        usernames = [u.strip().lstrip("@").lower()101                     for u in (inp.get("usernames") or []) if u and u.strip()]102        max_tweets = int(inp.get("maxTweets", 20))103        proxy = await Actor.create_proxy_configuration(104            actor_proxy_input=inp.get("proxyConfiguration"))105        fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1))106        sem = asyncio.Semaphore(int(inp.get("concurrency") or 4))107108        async def one(u: str) -> None:109            async with sem:110                miss = {"kind": "profile", "platform": "x",111                        "username": u, "found": False}112                try:113                    resp = await fetcher.get(SYNDICATION_URL.format(u=u),114                                             session_id=u, impersonate=None)115                    # 429 : retries avec backoff + NOUVELLE session proxy116                    # (nouvelle IP résidentielle) — une seule relance ne117                    # suffisait pas (13/40 handles en 429, constat 2026-08-21)118                    for i, imp in enumerate(("chrome", None, "chrome")):119                        if resp.status_code != 429:120                            break121                        await asyncio.sleep(5 * (i + 1))122                        resp = await fetcher.get(SYNDICATION_URL.format(u=u),123                                                 session_id=f"{u}r{i}",124                                                 impersonate=imp)125                except Exception as exc:126                    await Actor.push_data({**miss, "error": str(exc)[:200]})127                    return128                m = _NEXT_DATA_RE.search(resp.text or "")129                if resp.status_code != 200 or not m:130                    await Actor.push_data(131                        {**miss, "error": f"http_{resp.status_code}"})132                    return133                try:134                    data = json.loads(m.group(1))135                except Exception:136                    await Actor.push_data({**miss, "error": "bad_json"})137                    return138                user = _find_user(data, u)139                if not user:140                    await Actor.push_data({**miss, "error": "user_not_found"})141                    return142                tweets = parse_tweets(data, u, max_tweets)143                own = [t for t in tweets if not t["is_retweet"]]144                likes = [t["likes"] for t in own if isinstance(t["likes"], int)]145                rts = [t["retweets"] for t in own146                       if isinstance(t["retweets"], int)]147                reps = [t["replies"] for t in own148                        if isinstance(t["replies"], int)]149                followers = user.get("followers_count")150                avg_likes = round(sum(likes) / len(likes)) if likes else None151                avg_rts = round(sum(rts) / len(rts)) if rts else None152                engagement = None153                if followers and avg_likes is not None:154                    engagement = round((avg_likes + (avg_rts or 0))155                                       / followers * 100, 3)156                stamps = sorted(s for s in (_ts(t["created_at"])157                                            for t in own) if s)158                tweets_per_week = None159                if len(stamps) >= 3 and stamps[-1] > stamps[0]:160                    tweets_per_week = round(161                        (len(stamps) - 1) / ((stamps[-1] - stamps[0])162                                             / 604_800), 2)163                top = max((t for t in own if isinstance(t["likes"], int)),164                          key=lambda t: t["likes"], default=None)165                hashtags: dict[str, int] = {}166                mentions: dict[str, int] = {}167                for t in own:168                    for h in t["hashtags"]:169                        hashtags[h.lower()] = hashtags.get(h.lower(), 0) + 1170                    for mn in t["mentions"]:171                        mentions[mn.lower()] = mentions.get(mn.lower(), 0) + 1172                quotes = [t["quotes"] for t in own173                          if isinstance(t["quotes"], int)]174                created_ts = _ts(user.get("created_at"))175                await Actor.push_data({176                    "kind": "profile",177                    "platform": "x",178                    "found": True,179                    "id": user.get("id_str"),180                    "username": user.get("screen_name"),181                    "full_name": user.get("name"),182                    "biography": user.get("description"),183                    "location": user.get("location"),184                    "website": ((user.get("entities") or {}).get("url") or {})185                        .get("urls", [{}])[0].get("expanded_url")186                        if (user.get("entities") or {}).get("url") else None,187                    "followers": followers,188                    "following": user.get("friends_count"),189                    "tweets_count": user.get("statuses_count"),190                    "listed_count": user.get("listed_count"),191                    "likes_given": user.get("favourites_count"),192                    "is_verified": bool(user.get("verified")193                                        or user.get("is_blue_verified")),194                    "is_blue_verified": user.get("is_blue_verified"),195                    "verified_type": user.get("verified_type"),196                    "is_protected": user.get("protected"),197                    "created_at": user.get("created_at"),198                    "account_age_days": (round((time.time() - created_ts)199                                               / 86_400)200                                         if created_ts else None),201                    "avatar": (user.get("profile_image_url_https") or "")202                        .replace("_normal.", "_400x400."),203                    "banner": user.get("profile_banner_url"),204                    # agrégats sur les tweets récents (hors retweets)205                    "avg_likes": avg_likes,206                    "avg_retweets": avg_rts,207                    "avg_replies": (round(sum(reps) / len(reps))208                                    if reps else None),209                    "avg_quotes": (round(sum(quotes) / len(quotes))210                                   if quotes else None),211                    "engagement_rate_pct": engagement,212                    "tweets_per_week": tweets_per_week,213                    "last_tweet_at": own[0]["created_at"] if own else None,214                    "reply_share_pct": (round(sum(t["is_reply"]215                                                  for t in own)216                                              / len(own) * 100)217                                        if own else None),218                    "media_share_pct": (round(sum(bool(t["media_count"])219                                                  for t in own)220                                              / len(own) * 100)221                                        if own else None),222                    "link_share_pct": (round(sum(bool(t["links"])223                                                 for t in own)224                                             / len(own) * 100)225                                       if own else None),226                    "retweet_share_pct": (round(sum(t["is_retweet"]227                                                    for t in tweets)228                                                / len(tweets) * 100)229                                          if tweets else None),230                    "top_hashtags": sorted(hashtags, key=hashtags.get,231                                           reverse=True)[:8],232                    # comptes les plus mentionnés → signal d'identité §12.1233                    "top_mentions": sorted(mentions, key=mentions.get,234                                           reverse=True)[:8],235                    "top_tweet": ({"url": top["url"], "likes": top["likes"],236                                   "retweets": top["retweets"]}237                                  if top else None),238                    "recent_tweets": tweets,239                })240241        await asyncio.gather(*[one(u) for u in usernames])242        Actor.log.info(f"terminé : {len(usernames)} profils")243