spb/social-runtime-crawler

1import type { Page } from "playwright";2import type { EntityType, ObservedEntity, PageType } from "@src/shared";3import type { ClassifierHints, DomSnapshot } from "@src/observers";4import type { SocialPlatformAdapter, UrlEntityRule } from "./adapter.ts";56/**7 * YouTube adapter — Phase 1 platform. Holds only platform knowledge:8 * URL grammar, page hints, rich-text quirks (`runs` / `simpleText`) and id key names.9 * Nothing here hardcodes a YouTube endpoint: the network layer fingerprints them.10 */11const collapseYouTubeText = (v: unknown): string | undefined => {12  if (!v || typeof v !== "object") return undefined;13  const o = v as Record<string, unknown>;14  if (typeof o.simpleText === "string") return o.simpleText;15  if (Array.isArray(o.runs)) {16    const s = o.runs.map((r) => (r && typeof r === "object" && typeof (r as Record<string, unknown>).text === "string" ? (r as Record<string, string>).text : "")).join("");17    return s || undefined;18  }19  if (typeof o.content === "string") return o.content;20  return undefined;21};2223const classifierHints: ClassifierHints = {24  platform: "youtube",25  textCollapsers: [collapseYouTubeText],26  idKeys: { videoId: "video", channelId: "channel", commentId: "comment", playlistId: "post" },27  urlForId: (type: EntityType, id: string) => {28    if (type === "video") return `https://www.youtube.com/watch?v=${id}`;29    if (type === "channel") return `https://www.youtube.com/channel/${id}`;30    return undefined;31  },32};3334const urlEntityRules: UrlEntityRule[] = [35  { pattern: /^\/watch\?(?:.*&)?v=([A-Za-z0-9_-]{11})/, type: "video", id: (m) => m[1], confidence: 0.95 },36  { pattern: /^\/shorts\/([A-Za-z0-9_-]{11})/, type: "video", id: (m) => m[1], confidence: 0.95 },37  { pattern: /^\/(@[\w.-]{3,60})(?:\/|$|\?)/, type: "channel", id: (m) => m[1], confidence: 0.92 },38  { pattern: /^\/channel\/(UC[\w-]{20,})/, type: "channel", id: (m) => m[1], confidence: 0.95 },39  { pattern: /^\/c\/([\w.-]+)/, type: "channel", id: (m) => `c/${m[1]}`, confidence: 0.85 },40  { pattern: /^\/user\/([\w.-]+)/, type: "channel", id: (m) => `user/${m[1]}`, confidence: 0.85 },41  { pattern: /^\/post\/([\w-]+)/, type: "post", id: (m) => m[1], confidence: 0.9 },42  { pattern: /^\/hashtag\/([\w-]+)/, type: "hashtag", id: (m) => m[1]?.toLowerCase(), confidence: 0.9 },43];4445export const youtubeAdapter: SocialPlatformAdapter = {46  platform: "youtube",47  homeUrl: "https://www.youtube.com/",48  hosts: /(^|\.)youtube\.com$|^youtu\.be$/,49  pageTypeHints: [50    { pattern: /^\/results\?/, page_type: "SEARCH_RESULTS", confidence: 0.95 },51    { pattern: /^\/watch\?/, page_type: "VIDEO_DETAIL", confidence: 0.95 },52    { pattern: /^\/shorts\//, page_type: "VIDEO_DETAIL", confidence: 0.9 },53    { pattern: /^\/(@[\w.-]+|channel\/|c\/|user\/)/, page_type: "CHANNEL", confidence: 0.92 },54    { pattern: /^\/post\//, page_type: "POST_DETAIL", confidence: 0.9 },55    { pattern: /^\/feed\//, page_type: "HOME_FEED", confidence: 0.85 },56    { pattern: /^\/?$/, page_type: "HOME_FEED", confidence: 0.9 },57  ],58  urlEntityRules,59  classifierHints,60  searchUrl: (q) => `https://www.youtube.com/results?search_query=${encodeURIComponent(q)}`,61  isNoiseLink: (href, text) => /^\/(feed\/(library|history|subscriptions)|premium|account|reporthistory|paid_memberships|t\/|about|howyoutubeworks|new|creators|ads)/.test(href.pathname) || /^(Accueil|Home|Shorts|Abonnements|Subscriptions|Vous|You|Historique|History)$/i.test(text),62  entityLabel: (t) => ({ video: "video", channel: "channel", post: "community post", comment: "comment", hashtag: "hashtag" } as Partial<Record<EntityType, string>>)[t] ?? t,63  refineEntities: (entities: ObservedEntity[], snapshot: DomSnapshot) => {64    // On a watch page, the entity for the current video should be first and marked as "current".65    const m = snapshot.url.match(/[?&]v=([A-Za-z0-9_-]{11})/);66    if (m) {67      const cur = entities.find((e) => e.type === "video" && e.platform_id === m[1]);68      if (cur) {69        cur.context = "current video";70        if (snapshot.h1 && (!cur.name || cur.name.length < snapshot.h1.length)) cur.name = snapshot.h1;71        return [cur, ...entities.filter((e) => e !== cur)];72      }73    }74    return entities;75  },76  expectedEntities: (t: PageType) => ({ HOME_FEED: 8, SEARCH_RESULTS: 8, VIDEO_DETAIL: 5, CHANNEL: 4 } as Partial<Record<PageType, number>>)[t] ?? 0,77  dismissOverlays: async (page: Page) => {78    // Consent / "Sign in" nags — clicking a *dismiss* control is read-only UI hygiene, not an interaction with content.79    for (const sel of ['button[aria-label*="Accept" i]', 'button[aria-label*="Accepter" i]', 'tp-yt-paper-dialog #dismiss-button', 'ytd-popup-container button[aria-label*="No thanks" i]', 'ytd-popup-container button[aria-label*="Non merci" i]']) {80      const b = page.locator(sel).first();81      if (await b.isVisible().catch(() => false)) {82        await b.click({ timeout: 2000 }).catch(() => {});83      }84    }85  },86};87