spb/crea-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   src/main.py (ka-tiktok)4# Desc:   Profils TikTok publics ULTRA DÉTAILLÉS via la page @handle : le JSON5#         SSR __UNIVERSAL_DATA_FOR_REHYDRATION__ contient user + stats complètes6#         (abonnés, cœurs, vidéos) ; vidéos récentes avec COVERS (miniatures),7#         durée, hashtags, musique, épinglés + cadence (videos_per_week) et8#         top hashtags quand TikTok les inclut dans le SSR (ItemModule/itemList).9# ==============================================================================10from __future__ import annotations1112import asyncio13import json14import re1516from apify import Actor1718from .net import Fetcher1920PROFILE_URL = "https://www.tiktok.com/@{u}"21_STATE_RE = re.compile(22    r'<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)</script>',23    re.S)24_ITEM_RE = re.compile(r'"ItemModule"\s*:\s*(\{.*?\})\s*,\s*"[A-Z]', re.S)252627def parse_videos(html: str, data: dict) -> list[dict]:28    """Vidéos récentes si le SSR les inclut (souvent absent — best effort)."""29    items: list[dict] = []30    raw: dict = {}31    scope = data.get("__DEFAULT_SCOPE__") or {}32    detail = scope.get("webapp.user-detail") or {}33    for it in (detail.get("itemList") or []):34        raw[it.get("id", str(len(raw)))] = it35    if not raw:36        m = _ITEM_RE.search(html)37        if m:38            try:39                raw = json.loads(m.group(1))40            except Exception:41                raw = {}42    for vid in list(raw.values())[:20]:43        stats = vid.get("stats") or vid.get("statsV2") or {}44        author = vid.get("author")45        handle = author if isinstance(author, str) else \46            (author or {}).get("uniqueId", "")47        video = vid.get("video") or {}48        music = vid.get("music") or {}49        items.append({50            "id": vid.get("id"),51            "url": f"https://www.tiktok.com/@{handle}/video/{vid.get('id')}",52            "caption": (vid.get("desc") or "")[:500],53            "views": _num(stats.get("playCount")),54            "likes": _num(stats.get("diggCount")),55            "comments": _num(stats.get("commentCount")),56            "shares": _num(stats.get("shareCount")),57            "saves": _num(stats.get("collectCount")),58            "timestamp": vid.get("createTime"),59            "cover": (video.get("cover") or video.get("dynamicCover")60                      or video.get("originCover")),61            "duration_s": _num(video.get("duration")),62            "width": _num(video.get("width")),63            "height": _num(video.get("height")),64            "is_pinned": bool(vid.get("isPinnedItem")),65            "hashtags": [c.get("hashtagName")66                         for c in (vid.get("textExtra") or [])67                         if c.get("hashtagName")][:8],68            "music": ({"title": music.get("title"),69                       "author": music.get("authorName"),70                       "original": music.get("original")}71                      if music.get("title") else None),72        })73    return items747576def _num(v):77    try:78        return int(v)79    except (TypeError, ValueError):80        return None818283async def main() -> None:84    async with Actor:85        inp = await Actor.get_input() or {}86        usernames = [u.strip().lstrip("@").lower()87                     for u in (inp.get("usernames") or []) if u and u.strip()]88        proxy = await Actor.create_proxy_configuration(89            actor_proxy_input=inp.get("proxyConfiguration"))90        fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1))91        sem = asyncio.Semaphore(int(inp.get("concurrency") or 3))9293        async def one(u: str) -> None:94            async with sem:95                miss = {"kind": "profile", "platform": "tiktok",96                        "username": u, "found": False}97                resp = None98                m = None99                for attempt in range(3):  # coquille sans SSR → nouvelle IP100                    try:101                        resp = await fetcher.get(PROFILE_URL.format(u=u),102                                                 session_id=f"{u}t{attempt}")103                    except Exception as exc:104                        await Actor.push_data({**miss,105                                               "error": str(exc)[:200]})106                        return107                    if resp.status_code == 404:108                        await Actor.push_data({**miss, "error": "http_404"})109                        return110                    m = _STATE_RE.search(resp.text or "")111                    if m and "webapp.user-detail" in (resp.text or ""):112                        break113                info = None114                data: dict = {}115                if m:116                    try:117                        data = json.loads(m.group(1))118                        info = ((data.get("__DEFAULT_SCOPE__") or {})119                                .get("webapp.user-detail") or {}).get("userInfo")120                    except Exception:121                        info = None122                if not info or not (info.get("user") or {}).get("uniqueId"):123                    await Actor.push_data(124                        {**miss, "error": f"shell_page_{resp.status_code}"})125                    return126                user = info.get("user") or {}127                stats = info.get("statsV2") or info.get("stats") or {}128                if user.get("privateAccount"):129                    await Actor.push_data({**miss, "error": "private"})130                    return131                followers = _num(stats.get("followerCount"))132                videos = parse_videos(resp.text, data)133                views = [v["views"] for v in videos134                         if isinstance(v["views"], int)]135                likes = [v["likes"] for v in videos136                         if isinstance(v["likes"], int)]137                comments = [v["comments"] for v in videos138                            if isinstance(v["comments"], int)]139                shares = [v["shares"] for v in videos140                          if isinstance(v["shares"], int)]141                saves = [v["saves"] for v in videos142                         if isinstance(v["saves"], int)]143                avg_views = round(sum(views) / len(views)) if views else None144                avg_likes = round(sum(likes) / len(likes)) if likes else None145                engagement = None146                if followers and avg_likes is not None:147                    engagement = round(148                        (avg_likes149                         + (round(sum(comments) / len(comments))150                            if comments else 0)151                         + (round(sum(shares) / len(shares))152                            if shares else 0)) / followers * 100, 3)153                top_video = max(154                    (v for v in videos if isinstance(v["views"], int)),155                    key=lambda v: v["views"], default=None)156                stamps = sorted(_num(v["timestamp"]) for v in videos157                                if _num(v["timestamp"]))158                videos_per_week = None159                if len(stamps) >= 3 and stamps[-1] > stamps[0]:160                    videos_per_week = round(161                        (len(stamps) - 1)162                        / ((stamps[-1] - stamps[0]) / 604_800), 2)163                last_video_at = None164                if stamps:165                    from datetime import datetime, timezone166                    last_video_at = datetime.fromtimestamp(167                        stamps[-1], tz=timezone.utc) \168                        .strftime("%Y-%m-%dT%H:%M:%SZ")169                tag_counts: dict[str, int] = {}170                for v in videos:171                    for h in v.get("hashtags") or []:172                        tag_counts[h.lower()] = tag_counts.get(h.lower(), 0) + 1173                commerce = user.get("commerceUserInfo") or {}174                bio_link = ((user.get("bioLink") or {}).get("link")175                            or "").strip()176                total_likes = _num(stats.get("heartCount")177                                   or stats.get("heart"))178                videos_count = _num(stats.get("videoCount"))179                await Actor.push_data({180                    "kind": "profile",181                    "platform": "tiktok",182                    "found": True,183                    "id": user.get("id"),184                    "sec_uid": user.get("secUid"),185                    "username": user.get("uniqueId"),186                    "full_name": user.get("nickname"),187                    "biography": user.get("signature"),188                    "bio_link": bio_link,189                    "followers": followers,190                    "following": _num(stats.get("followingCount")),191                    "friends": _num(stats.get("friendCount")),192                    "likes_given": _num(stats.get("diggCount")),193                    "total_likes": total_likes,194                    "videos_count": videos_count,195                    "avg_likes_per_video_lifetime": (196                        round(total_likes / videos_count)197                        if total_likes and videos_count else None),198                    "is_verified": bool(user.get("verified")),199                    "is_organization": bool(user.get("isOrganization")),200                    "is_seller": bool(user.get("ttSeller")),201                    "is_live_now": bool(user.get("roomId")),202                    # drapeau mineur déclaré par TikTok → régime restreint §15203                    "is_under_18": (bool(user.get("isUnderAge18"))204                                    if "isUnderAge18" in user else None),205                    "is_embed_banned": (bool(user.get("isEmbedBanned"))206                                        if "isEmbedBanned" in user else None),207                    "commerce_category": (commerce.get("category") or None208                                          if commerce.get("commerceUser")209                                          else None),210                    "account_created_at": user.get("createTime"),211                    "region": user.get("region"),212                    "language": user.get("language"),213                    "avatar": user.get("avatarLarger")214                              or user.get("avatarMedium"),215                    # agrégats sur les vidéos présentes dans le SSR216                    "avg_views": avg_views,217                    "avg_likes": avg_likes,218                    "avg_comments": (round(sum(comments) / len(comments))219                                     if comments else None),220                    "avg_shares": (round(sum(shares) / len(shares))221                                   if shares else None),222                    "avg_saves": (round(sum(saves) / len(saves))223                                  if saves else None),224                    "pinned_videos_count": (sum(1 for v in videos225                                                if v["is_pinned"]) or None),226                    "engagement_rate_pct": engagement,227                    "videos_per_week": videos_per_week,228                    "last_video_at": last_video_at,229                    "top_hashtags": sorted(tag_counts, key=tag_counts.get,230                                           reverse=True)[:8],231                    "top_video": ({"url": top_video["url"],232                                   "views": top_video["views"],233                                   "likes": top_video["likes"],234                                   "cover": top_video.get("cover")}235                                  if top_video else None),236                    "recent_videos": videos,237                })238239        await asyncio.gather(*[one(u) for u in usernames])240        Actor.log.info(f"terminé : {len(usernames)} profils")241