SPB Git

Repository Public

main.py 12.8 KB 326 lines python

1# =============================================================================2#                  SP BOUCHER | SOFTWARE & RESEARCH3#             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4# =============================================================================5#6# PROJECT METADATA7# -----------------------------------------------------------------------------8# Project          : KA Apify Actors9# File             : ka-youtube/src/main.py10# Module           : ka-youtube11# Repository       : https://git.spboucher.ai/ka-apify-actors12# Created          : 2026-08-1813# Last Modified    : 2026-08-1814#15# =============================================================================16# AUTHOR & PROFESSIONAL AFFILIATION17# =============================================================================18#19# Author           : Simon-Pierre Boucher20# Position         : Professor of Finance21# Institution      : Université du Québec en Outaouais (UQO)22# Department       : Département des sciences administratives23# Organization     : SP Boucher24#25# Website          : https://spboucher.ai26# Git              : https://git.spboucher.ai27#28# =============================================================================29# OFFICIAL CONTACT DIRECTORY30# =============================================================================31#32# General          : info@spboucher.ai33# Professional     : contact@spboucher.ai34# Direct           : simon@spboucher.ai35# Research         : research@spboucher.ai36# Development      : dev@spboucher.ai37# Support          : support@spboucher.ai38# Security/Admin   : admin@spboucher.ai39# Notifications    : notifications@spboucher.ai40#41# Preferred Contact: contact@spboucher.ai42#43# =============================================================================44# COPYRIGHT & INTELLECTUAL PROPERTY45# =============================================================================46#47# Copyright (c) 202648# Simon-Pierre Boucher49#50# All rights reserved where applicable.51#52# This file is governed by the applicable project license,53# copyright notices, institutional agreements and relevant law.54#55# Unless expressly authorized by the applicable license or56# rights holder, copying, redistribution, sublicensing,57# publication and commercial exploitation are not permitted.58#59# Existing third-party rights and notices remain unaffected.60#61# =============================================================================62# CONFIDENTIALITY & SECURITY63# =============================================================================64#65# Classification   : Public66# Distribution     : Public repository; use is governed by the project license67# Security Contact : admin@spboucher.ai68#69# Access to this source does not independently grant permission70# to redistribute or disclose confidential materials.71#72# Do not embed credentials, private keys, access tokens,73# certificates or confidential personal information.74#75# =============================================================================76# LICENSE & MAINTENANCE77# =============================================================================78#79# License          : None specified (no license file in this repository)80# Maintainer       : Simon-Pierre Boucher81# Technical Contact: dev@spboucher.ai82#83# =============================================================================84# END OF SP BOUCHER SOURCE HEADER85# =============================================================================8687# ==============================================================================88# Author: Simon-Pierre Boucher <contact@spboucher.ai>89# File:   src/main.py (ka-youtube)90# Desc:   Chaînes YouTube détaillées SANS clé API via le JSON ytInitialData de91#         la page /videos : abonnés, nb de vidéos, description, avatar, liens92#         externes + vidéos récentes (vues, durée, date relative) et agrégats.93#         Accepte @handle, handle nu, ou id de chaîne (UC…).94# ==============================================================================95from __future__ import annotations9697import asyncio98import json99import re100import urllib.parse101102from apify import Actor103104from .net import Fetcher105106_INITIAL_RE = re.compile(r"var ytInitialData\s*=\s*(\{.*?\});</script>", re.S)107_COUNT_RE = re.compile(r"([\d][\d\s  .,]*)\s*([KkMB]?)")108109110def _url_for(handle: str) -> str:111    h = urllib.parse.unquote(handle).strip()112    if h.startswith("UC") and len(h) == 24:113        return f"https://www.youtube.com/channel/{h}/videos"114    if not h.startswith("@"):115        h = "@" + h116    return f"https://www.youtube.com/{urllib.parse.quote(h)}/videos"117118119def parse_text_count(text: str) -> int | None:120    """« 1.23M subscribers », « 12 345 vues », « 4,5 k » → entier."""121    if not text:122        return None123    m = _COUNT_RE.search(str(text))124    if not m:125        return None126    num = re.sub(r"[\s  ]", "", m.group(1))127    unit = (m.group(2) or "").upper()128    if unit:129        num = num.replace(",", ".")130        if num.count(".") > 1:131            num = num.replace(".", "", num.count(".") - 1)132        try:133            val = float(num)134        except ValueError:135            return None136        return int(val * {"K": 1e3, "M": 1e6, "B": 1e9}[unit])137    num = re.sub(r"[.,]", "", num)138    return int(num) if num.isdigit() else None139140141def _walk(node, key: str):142    """Premier objet portant `key` dans l'arbre ytInitialData."""143    if isinstance(node, dict):144        if key in node:145            return node[key]146        for v in node.values():147            hit = _walk(v, key)148            if hit is not None:149                return hit150    elif isinstance(node, list):151        for v in node:152            hit = _walk(v, key)153            if hit is not None:154                return hit155    return None156157158def _texts(node) -> list[str]:159    """Toutes les chaînes `content`/`simpleText`/`text` sous un nœud."""160    out: list[str] = []161162    def rec(n):163        if isinstance(n, dict):164            for k in ("content", "simpleText", "text"):165                if isinstance(n.get(k), str):166                    out.append(n[k])167            for v in n.values():168                rec(v)169        elif isinstance(n, list):170            for v in n:171                rec(v)172    rec(node)173    return out174175176def _lockup_video(lv: dict) -> dict | None:177    """Carte vidéo moderne (lockupViewModel) → dict vidéo standardisé."""178    if lv.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO":179        return None180    vid = lv.get("contentId")181    if not vid:182        return None183    md = lv.get("metadata") or {}184    title = ((md.get("lockupMetadataViewModel") or {}).get("title") or {}) \185        .get("content")186    strings = _texts(md)187    views = next((parse_text_count(s) for s in strings188                  if "view" in s.lower() or "vue" in s.lower()), None)189    published = next((s for s in strings190                      if "ago" in s.lower() or "il y a" in s.lower()), None)191    overlays = _texts(lv.get("contentImage") or {})192    duration = next((s for s in overlays193                     if re.fullmatch(r"\d?\d:\d\d(:\d\d)?", s.strip())), None)194    return {195        "id": vid,196        "url": f"https://www.youtube.com/watch?v={vid}",197        "title": (title or "")[:200],198        "views": views,199        "published": published,200        "duration": duration,201    }202203204def _collect_videos(data: dict, cap: int = 15) -> list[dict]:205    out: list[dict] = []206207    def rec(node):208        if len(out) >= cap:209            return210        if isinstance(node, dict):211            lv = node.get("lockupViewModel")212            if isinstance(lv, dict):213                v = _lockup_video(lv)214                if v:215                    out.append(v)216                    return217            vr = node.get("videoRenderer") or node.get("gridVideoRenderer")218            if vr and vr.get("videoId"):219                out.append({220                    "id": vr["videoId"],221                    "url": f"https://www.youtube.com/watch?v={vr['videoId']}",222                    "title": "".join(223                        r.get("text", "") for r in224                        (vr.get("title") or {}).get("runs") or [])[:200],225                    "views": parse_text_count(226                        (vr.get("viewCountText") or {}).get("simpleText")),227                    "published": ((vr.get("publishedTimeText") or {})228                                  .get("simpleText")),229                    "duration": ((vr.get("lengthText") or {})230                                 .get("simpleText")),231                })232                return233            for v in node.values():234                rec(v)235        elif isinstance(node, list):236            for v in node:237                rec(v)238239    rec(data)240    return out241242243async def main() -> None:244    async with Actor:245        inp = await Actor.get_input() or {}246        usernames = [u.strip() for u in (inp.get("usernames") or [])247                     if u and u.strip()]248        proxy = await Actor.create_proxy_configuration(249            actor_proxy_input=inp.get("proxyConfiguration"))250        fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1))251        sem = asyncio.Semaphore(int(inp.get("concurrency") or 3))252253        async def one(u: str) -> None:254            async with sem:255                miss = {"kind": "profile", "platform": "youtube",256                        "username": u, "found": False}257                try:258                    resp = await fetcher.get(259                        _url_for(u), session_id=u,260                        headers={"Accept-Language": "en-US,en;q=0.9"})261                except Exception as exc:262                    await Actor.push_data({**miss, "error": str(exc)[:200]})263                    return264                m = _INITIAL_RE.search(resp.text or "")265                if resp.status_code != 200 or not m:266                    await Actor.push_data(267                        {**miss, "error": f"http_{resp.status_code}"})268                    return269                try:270                    data = json.loads(m.group(1))271                except Exception:272                    await Actor.push_data({**miss, "error": "bad_json"})273                    return274                meta = _walk(data, "channelMetadataRenderer") or {}275                if not meta.get("externalId"):276                    await Actor.push_data({**miss, "error": "no_channel"})277                    return278                # compteurs : le nouvel en-tête n'a plus subscriberCountText ;279                # on lit les chaînes « N subscribers » / « N videos » du header280                header = _walk(data, "pageHeaderViewModel") or {}281                header_txt = _texts(header) or _texts(282                    _walk(data, "c4TabbedHeaderRenderer") or {})283                sub_text = next((t for t in header_txt284                                 if "subscriber" in t.lower()285                                 or "abonné" in t.lower()), "")286                vid_text = next((t for t in header_txt287                                 if ("video" in t.lower()288                                     or "vidéo" in t.lower())289                                 and any(c.isdigit() for c in t)), "")290                if not sub_text:  # ancien en-tête291                    sub_text = ((_walk(data, "subscriberCountText") or {})292                                .get("simpleText") or "")293                subs = parse_text_count(sub_text)294                videos_count = parse_text_count(vid_text)295                vids = _collect_videos(data)296                views = [v["views"] for v in vids297                         if isinstance(v["views"], int)]298                await Actor.push_data({299                    "kind": "profile",300                    "platform": "youtube",301                    "found": True,302                    "username": u,303                    "channel_id": meta.get("externalId"),304                    "full_name": meta.get("title"),305                    "biography": (meta.get("description") or "")[:1000],306                    "followers": subs,307                    "videos_count": videos_count,308                    "is_family_safe": meta.get("isFamilySafe"),309                    "keywords": (meta.get("keywords") or "")[:300] or None,310                    "country": _walk(data, "country"),311                    "canonical_url": meta.get("vanityChannelUrl"),312                    "avatar": (((meta.get("avatar") or {}).get("thumbnails")313                                or [{}])[-1].get("url")),314                    "avg_views": (round(sum(views) / len(views))315                                  if views else None),316                    "top_video": (max(317                        (v for v in vids if isinstance(v["views"], int)),318                        key=lambda v: v["views"], default=None)),319                    "last_video_published": (vids[0]["published"]320                                             if vids else None),321                    "recent_videos": vids,322                })323324        await asyncio.gather(*[one(u) for u in usernames])325        Actor.log.info(f"terminé : {len(usernames)} chaînes")326
Files main
Menu

Theme