feat(youtube-recherche): profondeur — pagination par continuations (~4 pages/requête, 5x le rendement de découverte)
- API interne de la page publique (INNERTUBE) pour les pages 2-4 de chaque requête - fix: IDs de chaîne UC… — casse préservée + URL /channel/ canonique - fix: parse_count ne lit plus un handle comme un compteur (bug 5 G abonnés) - cartes de comptes : pastille de confiance sur sa propre ligne Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 changed file +77 −28
modified
creaka/connectors/youtube_recherche.py
+77 −28
@@ -27,8 +27,11 @@ from ..schema import Creator | |||||
| 27 | from .base import BaseConnector | 27 | from .base import BaseConnector | ||
| 28 | 28 | ||||
| 29 | SEARCH_URL = "https://www.youtube.com/results" | 29 | SEARCH_URL = "https://www.youtube.com/results" | ||
| 30 | +INNERTUBE_URL = "https://www.youtube.com/youtubei/v1/search" | ||||
| 30 | CHANNEL_FILTER = "EgIQAg%3D%3D" # sp= filtre « chaînes » | 31 | CHANNEL_FILTER = "EgIQAg%3D%3D" # sp= filtre « chaînes » | ||
| 31 | _YTID_RE = re.compile(r"var ytInitialData = (\{.*?\});</script>", re.S) | 32 | _YTID_RE = re.compile(r"var ytInitialData = (\{.*?\});</script>", re.S) | ||
| 33 | +_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"') | ||||
| 34 | +_VER_RE = re.compile(r'"INNERTUBE_CONTEXT_CLIENT_VERSION":"([^"]+)"') | ||||
| 32 | 35 | ||||
| 33 | # marqueurs québécois (comparés SANS accents, en minuscules) | 36 | # marqueurs québécois (comparés SANS accents, en minuscules) | ||
| 34 | _QC_MARKERS = re.compile( | 37 | _QC_MARKERS = re.compile( | ||
@@ -201,6 +204,24 @@ def _walk_channels(node, out: list) -> None: | |||||
| 201 | _walk_channels(item, out) | 204 | _walk_channels(item, out) | ||
| 202 | 205 | ||||
| 203 | 206 | ||||
| 207 | +def _walk_token(node) -> str | None: | ||||
| 208 | + """Jeton de continuation (page suivante) n'importe où dans l'arbre JSON.""" | ||||
| 209 | + if isinstance(node, dict): | ||||
| 210 | + cmd = node.get("continuationCommand") | ||||
| 211 | + if isinstance(cmd, dict) and cmd.get("token"): | ||||
| 212 | + return cmd["token"] | ||||
| 213 | + for v in node.values(): | ||||
| 214 | + tok = _walk_token(v) | ||||
| 215 | + if tok: | ||||
| 216 | + return tok | ||||
| 217 | + elif isinstance(node, list): | ||||
| 218 | + for item in node: | ||||
| 219 | + tok = _walk_token(item) | ||||
| 220 | + if tok: | ||||
| 221 | + return tok | ||||
| 222 | + return None | ||||
| 223 | + | ||||
| 224 | + | ||||
| 204 | def parse_channels(html: str) -> list[dict]: | 225 | def parse_channels(html: str) -> list[dict]: | ||
| 205 | """HTML de résultats → [{handle, channel_id, name, subs, description}].""" | 226 | """HTML de résultats → [{handle, channel_id, name, subs, description}].""" | ||
| 206 | m = _YTID_RE.search(html) | 227 | m = _YTID_RE.search(html) | ||
@@ -210,6 +231,11 @@ def parse_channels(html: str) -> list[dict]: | |||||
| 210 | data = json.loads(m.group(1)) | 231 | data = json.loads(m.group(1)) | ||
| 211 | except Exception: | 232 | except Exception: | ||
| 212 | return [] | 233 | return [] | ||
| 234 | + return channels_from_data(data) | ||||
| 235 | + | ||||
| 236 | + | ||||
| 237 | +def channels_from_data(data) -> list[dict]: | ||||
| 238 | + """Arbre JSON (page 1 ou réponse de continuation) → liste de chaînes.""" | ||||
| 213 | renderers: list[dict] = [] | 239 | renderers: list[dict] = [] | ||
| 214 | _walk_channels(data, renderers) | 240 | _walk_channels(data, renderers) | ||
| 215 | channels = [] | 241 | channels = [] | ||
@@ -237,8 +263,33 @@ def is_quebec(text: str) -> bool: | |||||
| 237 | class YouTubeRechercheConnector(BaseConnector): | 263 | class YouTubeRechercheConnector(BaseConnector): | ||
| 238 | source_id = "youtube-recherche" | 264 | source_id = "youtube-recherche" | ||
| 239 | kind = "discovery" | 265 | kind = "discovery" | ||
| 240 | − request_delay = 1.2 | 266 | + request_delay = 1.0 | ||
| 241 | max_queries = 800 | 267 | max_queries = 800 | ||
| 268 | + max_pages_per_query = 4 # PROFONDEUR : pages de continuation (~20 chaînes/page) | ||||
| 269 | + | ||||
| 270 | + def _collect(self, channels: list[dict], niche: str, seen: set[str], | ||||
| 271 | + creators: list[Creator]) -> None: | ||||
| 272 | + for ch in channels: | ||||
| 273 | + handle = ch["handle"].lstrip("@") | ||||
| 274 | + key = handle or ch["channel_id"] | ||||
| 275 | + if not key or key in seen: | ||||
| 276 | + continue | ||||
| 277 | + # filtre conservateur : marqueur QC requis dans titre+description | ||||
| 278 | + if not is_quebec(f"{ch['name']} {ch['description']}"): | ||||
| 279 | + continue | ||||
| 280 | + seen.add(key) | ||||
| 281 | + acc = account("youtube", handle or ch["channel_id"], "profil_source", | ||||
| 282 | + followers=ch["subs"]) | ||||
| 283 | + creators.append(Creator( | ||||
| 284 | + source=self.source_id, | ||||
| 285 | + external_id=slugify(ch["name"]), | ||||
| 286 | + display_name=ch["name"], | ||||
| 287 | + bio=ch["description"][:1200], | ||||
| 288 | + niches=[niche], | ||||
| 289 | + creator_type="youtubeur", | ||||
| 290 | + platforms=[acc], | ||||
| 291 | + source_ids=[f"{self.source_id}:{key}"], | ||||
| 292 | + )) | ||||
| 242 | 293 | ||||
| 243 | def fetch(self) -> list[Creator]: | 294 | def fetch(self) -> list[Creator]: | ||
| 244 | creators: list[Creator] = [] | 295 | creators: list[Creator] = [] | ||
@@ -251,31 +302,29 @@ class YouTubeRechercheConnector(BaseConnector): | |||||
| 251 | headers={"Accept-Language": "fr-CA,fr;q=0.9"}).text | 302 | headers={"Accept-Language": "fr-CA,fr;q=0.9"}).text | ||
| 252 | except Exception: | 303 | except Exception: | ||
| 253 | continue | 304 | continue | ||
| 254 | − for ch in parse_channels(html): | 305 | + m = _YTID_RE.search(html) | ||
| 255 | − handle = ch["handle"].lstrip("@") | 306 | + if not m: | ||
| 256 | − key = handle or ch["channel_id"] | 307 | + continue | ||
| 257 | − if not key or key in seen: | 308 | + try: | ||
| 258 | − continue | 309 | + data = json.loads(m.group(1)) | ||
| 259 | − # filtre conservateur : marqueur QC requis dans titre+description | 310 | + except Exception: | ||
| 260 | − if not is_quebec(f"{ch['name']} {ch['description']}"): | 311 | + continue | ||
| 261 | − continue | 312 | + key_m, ver_m = _KEY_RE.search(html), _VER_RE.search(html) | ||
| 262 | − seen.add(key) | 313 | + for page in range(self.max_pages_per_query): | ||
| 263 | − if handle: | 314 | + self._collect(channels_from_data(data), niche, seen, creators) | ||
| 264 | − acc = account("youtube", handle, "profil_source", | 315 | + token = _walk_token(data) | ||
| 265 | − followers=ch["subs"]) | 316 | + if not token or not key_m or not ver_m \ | ||
| 266 | − else: | 317 | + or page == self.max_pages_per_query - 1: | ||
| 267 | − acc = account("youtube", ch["channel_id"], "profil_source", | 318 | + break | ||
| 268 | − followers=ch["subs"], | 319 | + try: # pages suivantes via l'API interne de la page publique | ||
| 269 | − url="https://www.youtube.com/channel/" | 320 | + data = self.post( | ||
| 270 | − + ch["channel_id"]) | 321 | + INNERTUBE_URL, params={"key": key_m.group(1)}, | ||
| 271 | − creators.append(Creator( | 322 | + json={"context": {"client": { | ||
| 272 | − source=self.source_id, | 323 | + "clientName": "WEB", | ||
| 273 | − external_id=slugify(ch["name"]), | 324 | + "clientVersion": ver_m.group(1), | ||
| 274 | − display_name=ch["name"], | 325 | + "hl": "fr", "gl": "CA"}}, | ||
| 275 | − bio=ch["description"][:1200], | 326 | + "continuation": token}, | ||
| 276 | − niches=[niche], | 327 | + headers={"Accept-Language": "fr-CA,fr;q=0.9"}).json() | ||
| 277 | − creator_type="youtubeur", | 328 | + except Exception: | ||
| 278 | − platforms=[acc], | 329 | + break | ||
| 279 | − source_ids=[f"{self.source_id}:{key}"], | ||||
| 280 | − )) | ||||
| 281 | return creators | 330 | return creators | ||
| 282 | 331 | ||||