SPB Git

Repository Public

main.py 10.2 KB 216 lines python

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   src/main.py (ka-x)4# Desc:   Profils X (Twitter) SANS clé via le service OFFICIEL de syndication5#         des widgets (syndication.twitter.com/srv/timeline-profile) : objet6#         user complet + timeline récente avec métriques par tweet. Conçu pour7#         être appelé sans authentification ; l'empreinte TLS Chrome de8#         curl_cffi évite les 429 réservés aux clients python-requests.9# ==============================================================================10from __future__ import annotations1112import asyncio13import json14import re15import time1617from apify import Actor1819from .net import Fetcher2021SYNDICATION_URL = ("https://syndication.twitter.com/srv/timeline-profile/"22                   "screen-name/{u}")23_NEXT_DATA_RE = re.compile(24    r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', re.S)252627def _find_user(node, handle: str):28    """Parcourt le JSON Next.js : l'objet user dont screen_name == handle."""29    if isinstance(node, dict):30        if str(node.get("screen_name", "")).lower() == handle and \31                "followers_count" in node:32            return node33        for v in node.values():34            hit = _find_user(v, handle)35            if hit is not None:36                return hit37    elif isinstance(node, list):38        for v in node:39            hit = _find_user(v, handle)40            if hit is not None:41                return hit42    return None434445def parse_tweets(data: dict, handle: str, cap: int) -> list[dict]:46    entries = (((data.get("props") or {}).get("pageProps") or {})47               .get("timeline") or {}).get("entries") or []48    tweets: list[dict] = []49    for entry in entries:50        t = (entry.get("content") or {}).get("tweet") or {}51        if not t.get("id_str"):52            continue53        u = (t.get("user") or {})54        ent = t.get("entities") or {}55        media = (t.get("extended_entities") or ent).get("media") or []56        tweets.append({57            "id": t.get("id_str"),58            "url": f"https://x.com/{u.get('screen_name', handle)}"59                   f"/status/{t.get('id_str')}",60            "text": (t.get("full_text") or t.get("text") or "")[:500],61            "likes": t.get("favorite_count"),62            "retweets": t.get("retweet_count"),63            "replies": t.get("reply_count"),64            "quotes": t.get("quote_count"),65            "created_at": t.get("created_at"),66            "is_retweet": bool(t.get("retweeted_status")),67            "is_reply": bool(t.get("in_reply_to_status_id_str")),68            "is_quote": bool(t.get("is_quote_status")),69            "lang": t.get("lang"),70            "hashtags": [h.get("text") for h in (ent.get("hashtags") or [])71                         if h.get("text")][:8],72            "mentions": [m.get("screen_name")73                         for m in (ent.get("user_mentions") or [])74                         if m.get("screen_name")][:8],75            "media_type": (media[0].get("type") if media else None),76            "media_count": len(media) or None,77            "links": [x.get("expanded_url") for x in (ent.get("urls") or [])78                      if x.get("expanded_url")][:4],79        })80        if len(tweets) >= cap:81            break82    return tweets838485def _ts(created_at: str) -> float | None:86    """« Tue Sep 12 23:56:24 +0000 2023 » → epoch (None si illisible)."""87    try:88        from datetime import datetime89        return datetime.strptime(created_at,90                                 "%a %b %d %H:%M:%S %z %Y").timestamp()91    except (TypeError, ValueError):92        return None939495async def main() -> None:96    async with Actor:97        inp = await Actor.get_input() or {}98        usernames = [u.strip().lstrip("@").lower()99                     for u in (inp.get("usernames") or []) if u and u.strip()]100        max_tweets = int(inp.get("maxTweets", 20))101        proxy = await Actor.create_proxy_configuration(102            actor_proxy_input=inp.get("proxyConfiguration"))103        fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1))104        sem = asyncio.Semaphore(int(inp.get("concurrency") or 4))105106        async def one(u: str) -> None:107            async with sem:108                miss = {"kind": "profile", "platform": "x",109                        "username": u, "found": False}110                try:111                    resp = await fetcher.get(SYNDICATION_URL.format(u=u),112                                             session_id=u, impersonate=None)113                    if resp.status_code == 429:114                        resp = await fetcher.get(SYNDICATION_URL.format(u=u),115                                                 session_id=f"{u}b",116                                                 impersonate="chrome")117                except Exception as exc:118                    await Actor.push_data({**miss, "error": str(exc)[:200]})119                    return120                m = _NEXT_DATA_RE.search(resp.text or "")121                if resp.status_code != 200 or not m:122                    await Actor.push_data(123                        {**miss, "error": f"http_{resp.status_code}"})124                    return125                try:126                    data = json.loads(m.group(1))127                except Exception:128                    await Actor.push_data({**miss, "error": "bad_json"})129                    return130                user = _find_user(data, u)131                if not user:132                    await Actor.push_data({**miss, "error": "user_not_found"})133                    return134                tweets = parse_tweets(data, u, max_tweets)135                own = [t for t in tweets if not t["is_retweet"]]136                likes = [t["likes"] for t in own if isinstance(t["likes"], int)]137                rts = [t["retweets"] for t in own138                       if isinstance(t["retweets"], int)]139                reps = [t["replies"] for t in own140                        if isinstance(t["replies"], int)]141                followers = user.get("followers_count")142                avg_likes = round(sum(likes) / len(likes)) if likes else None143                avg_rts = round(sum(rts) / len(rts)) if rts else None144                engagement = None145                if followers and avg_likes is not None:146                    engagement = round((avg_likes + (avg_rts or 0))147                                       / followers * 100, 3)148                stamps = sorted(s for s in (_ts(t["created_at"])149                                            for t in own) if s)150                tweets_per_week = None151                if len(stamps) >= 3 and stamps[-1] > stamps[0]:152                    tweets_per_week = round(153                        (len(stamps) - 1) / ((stamps[-1] - stamps[0])154                                             / 604_800), 2)155                top = max((t for t in own if isinstance(t["likes"], int)),156                          key=lambda t: t["likes"], default=None)157                hashtags: dict[str, int] = {}158                for t in own:159                    for h in t["hashtags"]:160                        hashtags[h.lower()] = hashtags.get(h.lower(), 0) + 1161                created_ts = _ts(user.get("created_at"))162                await Actor.push_data({163                    "kind": "profile",164                    "platform": "x",165                    "found": True,166                    "id": user.get("id_str"),167                    "username": user.get("screen_name"),168                    "full_name": user.get("name"),169                    "biography": user.get("description"),170                    "location": user.get("location"),171                    "website": ((user.get("entities") or {}).get("url") or {})172                        .get("urls", [{}])[0].get("expanded_url")173                        if (user.get("entities") or {}).get("url") else None,174                    "followers": followers,175                    "following": user.get("friends_count"),176                    "tweets_count": user.get("statuses_count"),177                    "listed_count": user.get("listed_count"),178                    "likes_given": user.get("favourites_count"),179                    "is_verified": bool(user.get("verified")180                                        or user.get("is_blue_verified")),181                    "is_blue_verified": user.get("is_blue_verified"),182                    "is_protected": user.get("protected"),183                    "created_at": user.get("created_at"),184                    "account_age_days": (round((time.time() - created_ts)185                                               / 86_400)186                                         if created_ts else None),187                    "avatar": (user.get("profile_image_url_https") or "")188                        .replace("_normal.", "_400x400."),189                    "banner": user.get("profile_banner_url"),190                    # agrégats sur les tweets récents (hors retweets)191                    "avg_likes": avg_likes,192                    "avg_retweets": avg_rts,193                    "avg_replies": (round(sum(reps) / len(reps))194                                    if reps else None),195                    "engagement_rate_pct": engagement,196                    "tweets_per_week": tweets_per_week,197                    "last_tweet_at": own[0]["created_at"] if own else None,198                    "reply_share_pct": (round(sum(t["is_reply"]199                                                  for t in own)200                                              / len(own) * 100)201                                        if own else None),202                    "media_share_pct": (round(sum(bool(t["media_count"])203                                                  for t in own)204                                              / len(own) * 100)205                                        if own else None),206                    "top_hashtags": sorted(hashtags, key=hashtags.get,207                                           reverse=True)[:8],208                    "top_tweet": ({"url": top["url"], "likes": top["likes"],209                                   "retweets": top["retweets"]}210                                  if top else None),211                    "recent_tweets": tweets,212                })213214        await asyncio.gather(*[one(u) for u in usernames])215        Actor.log.info(f"terminé : {len(usernames)} profils")216
Files main
Menu

Theme