spb/social-runtime-crawler

1import type { Page } from "playwright";2import type { EntityType, ObservedEntity, PageType } from "@src/shared";3import type { ClassifierHints, DomSnapshot } from "@src/observers";4import type { SocialPlatformAdapter, UrlEntityRule } from "./adapter.ts";56/**7 * Facebook adapter — Phase 2 (§52 experimental prototype).8 * Platform knowledge only: URL grammar, page hints, GraphQL `__typename` vocabulary, `{text}` rich-text shape,9 * reserved navigation paths and consent/dialog dismissal. No GraphQL doc_id, no endpoint, no CSS class.10 * Facebook responses arrive as `for (;;);`-prefixed JSON or NDJSON on POST /api/graphql/ with11 * `fb_api_req_friendly_name` — both are handled generically by the network layer.12 */13const RESERVED = /^\/(marketplace|gaming|friends|bookmarks|notifications|messages|messenger|settings|help|privacy|policies|policy|login|logout|recover|reels?|watch\/?$|watch\/(live|saved|shows|latest)|stories|memories|saved|pages\/?$|pages\/(create|discover)|fundraisers|events\/?$|ads|business|about|careers|developers|terms|legal|cookie|donate|places|weather|games|jobs|dating|crisisresponse|community|search\/?$|bookmarks|feeds?\/?$|me\/?$|home\.php|checkpoint|sharer|dialog|plugins|ajax|composer|mediaset|photo\.php\?.*type=3|badges|topics|hashtag\/?$|lite|mobile|profile\.php\?.*sk=|groups\/(feed|discover|joins|create)\/?)/i;1415const classifierHints: ClassifierHints = {16  platform: "facebook",17  textCollapsers: [18    (v) => (v && typeof v === "object" && typeof (v as { text?: unknown }).text === "string" ? (v as { text: string }).text : undefined),19    (v) => {20      if (!v || typeof v !== "object") return undefined;21      const o = v as { ranges?: unknown; text?: unknown; delight_ranges?: unknown };22      return typeof o.text === "string" ? o.text : undefined;23    },24  ],25  idKeys: { post_id: "post", story_id: "post", video_id: "video", videoId: "video", page_id: "page", group_id: "community", comment_id: "comment", legacy_story_hideable_id: "post" },26  typenames: { Story: "post", Video: "video", Photo: "image", User: "profile", Page: "page", Group: "community", Comment: "comment", Event: "event", Hashtag: "hashtag", Reel: "video" },27  urlForId: (type: EntityType, id: string) => {28    if (!/^\d{6,}$/.test(id)) return undefined;29    if (type === "video") return `https://www.facebook.com/watch/?v=${id}`;30    if (type === "page" || type === "profile") return `https://www.facebook.com/profile.php?id=${id}`;31    if (type === "community") return `https://www.facebook.com/groups/${id}`;32    return undefined;33  },34};3536const urlEntityRules: UrlEntityRule[] = [37  { pattern: /^\/watch\/?\?(?:.*&)?v=(\d+)/, type: "video", id: (m) => m[1], confidence: 0.95 },38  { pattern: /^\/reel\/(\d+)/, type: "video", id: (m) => m[1], confidence: 0.95 },39  { pattern: /^\/[^/]+\/videos\/(?:[^/]+\/)?(\d+)/, type: "video", id: (m) => m[1], confidence: 0.93 },40  { pattern: /^\/video\.php\?(?:.*&)?v=(\d+)/, type: "video", id: (m) => m[1], confidence: 0.9 },41  { pattern: /^\/[^/]+\/posts\/(pfbid[\w]+|\d+)/, type: "post", id: (m) => m[1], confidence: 0.95 },42  { pattern: /^\/permalink\.php\?(?:.*&)?story_fbid=(pfbid[\w]+|\d+)/, type: "post", id: (m) => m[1], confidence: 0.93 },43  { pattern: /^\/story\.php\?(?:.*&)?story_fbid=(pfbid[\w]+|\d+)/, type: "post", id: (m) => m[1], confidence: 0.9 },44  { pattern: /^\/groups\/([\w.]+)\/(?:posts|permalink)\/(\d+)/, type: "post", id: (m) => m[2], confidence: 0.93 },45  { pattern: /^\/photo\/?\?(?:.*&)?fbid=(\d+)/, type: "image", id: (m) => m[1], confidence: 0.9 },46  { pattern: /^\/[^/]+\/photos\/(?:[^/]+\/)?(\d+)/, type: "image", id: (m) => m[1], confidence: 0.85 },47  { pattern: /^\/groups\/([\w.]+)\/?(?:\?|$)/, type: "community", id: (m) => m[1]?.toLowerCase(), confidence: 0.92 },48  { pattern: /^\/events\/(\d+)/, type: "event", id: (m) => m[1], confidence: 0.92 },49  { pattern: /^\/hashtag\/([\w]+)/, type: "hashtag", id: (m) => m[1]?.toLowerCase(), confidence: 0.9 },50  { pattern: /^\/profile\.php\?(?:.*&)?id=(\d+)/, type: "profile", id: (m) => m[1], confidence: 0.9 },51  { pattern: /^\/people\/[^/]+\/(pfbid[\w]+|\d+)/, type: "profile", id: (m) => m[1], confidence: 0.9 },52  // Vanity URL: /<slug>/ — a public Page or a Person; the type is refined later by GraphQL __typename evidence.53  { pattern: /^\/([A-Za-z0-9.]{3,60})\/?(?:\?|$)/, type: "page", id: (m) => m[1]?.toLowerCase(), confidence: 0.6 },54];5556export const facebookAdapter: SocialPlatformAdapter = {57  platform: "facebook",58  homeUrl: "https://www.facebook.com/",59  hosts: /(^|\.)facebook\.com$|(^|\.)fb\.com$|^fb\.watch$/,60  pageTypeHints: [61    { pattern: /^\/?(\?|$)|^\/home\.php/, page_type: "HOME_FEED", confidence: 0.9 },62    { pattern: /^\/search\//, page_type: "SEARCH_RESULTS", confidence: 0.95 },63    { pattern: /^\/watch\/?\?v=|^\/reel\/|\/videos\/\d+/, page_type: "VIDEO_DETAIL", confidence: 0.93 },64    { pattern: /\/posts\/|permalink\.php|story\.php|\/photo\/?\?fbid=|\/photos\//, page_type: "POST_DETAIL", confidence: 0.92 },65    { pattern: /^\/groups\/[\w.]+\/?(\?|$)/, page_type: "GROUP", confidence: 0.92 },66    { pattern: /^\/profile\.php\?id=|^\/people\//, page_type: "PROFILE", confidence: 0.9 },67    { pattern: /^\/login|^\/checkpoint|^\/recover/, page_type: "LOGIN", confidence: 0.95 },68    { pattern: /^\/[A-Za-z0-9.]{3,60}\/?(\?|$)/, page_type: "PUBLIC_PAGE", confidence: 0.7 },69  ],70  urlEntityRules,71  classifierHints,72  searchUrl: (q) => `https://www.facebook.com/search/top/?q=${encodeURIComponent(q)}`,73  isNoiseLink: (href, text) => RESERVED.test(href.pathname + href.search) || /^(Accueil|Home|Vidéo|Video|Watch|Marketplace|Groupes|Groups|Gaming|Jeux|Amis|Friends|Menu|Messenger|Notifications|Profil|Profile|Voir plus|See more|J’aime|J'aime|Like|Commenter|Comment|Partager|Share)$/i.test(text.trim()),74  entityLabel: (t) => ({ post: "post", video: "video", image: "photo", page: "page or profile", profile: "profile", community: "group", comment: "comment", event: "event", hashtag: "hashtag" } as Partial<Record<EntityType, string>>)[t] ?? t,75  refineEntities: (entities: ObservedEntity[], snapshot: DomSnapshot) => {76    // Feed cards: a `[role=article]` whose link is a post is the post itself; the same card also exposes its author page.77    // We keep both, but mark posts seen in the feed region so the planner knows their exposure context (§44).78    for (const e of entities) if (e.type === "post" && /feed/.test(e.context ?? "")) e.context = `feed item ${e.context}`;79    // Current post/video on a detail page80    const cur = entities.find((e) => e.url && samePath(e.url, snapshot.url) && (e.type === "post" || e.type === "video" || e.type === "image"));81    if (cur) {82      cur.context = `current ${cur.type}`;83      return [cur, ...entities.filter((e) => e !== cur)];84    }85    return entities;86  },87  expectedEntities: (t: PageType) => ({ HOME_FEED: 5, SEARCH_RESULTS: 5, PUBLIC_PAGE: 4, PROFILE: 3, GROUP: 4, VIDEO_DETAIL: 2, POST_DETAIL: 2 } as Partial<Record<PageType, number>>)[t] ?? 0,88  dismissOverlays: async (page: Page) => {89    // Consent banner and "log in to continue" nag dialogs — dismiss controls only, never a content interaction.90    for (const sel of [91      'button[data-cookiebanner="accept_button"]',92      '[aria-label="Allow all cookies"]',93      '[aria-label="Autoriser tous les cookies"]',94      '[aria-label="Autoriser les cookies essentiels et optionnels"]',95      'div[role=dialog] [aria-label="Close"]',96      'div[role=dialog] [aria-label="Fermer"]',97    ]) {98      const b = page.locator(sel).first();99      if (await b.isVisible().catch(() => false)) await b.click({ timeout: 2000 }).catch(() => {});100    }101  },102};103104function samePath(a: string, b: string): boolean {105  try {106    const ua = new URL(a);107    const ub = new URL(b);108    if (ua.pathname !== ub.pathname) return false;109    for (const k of ["v", "story_fbid", "fbid", "id"]) if (ua.searchParams.get(k) !== ub.searchParams.get(k)) return false;110    return true;111  } catch {112    return false;113  }114}115