spb/social-runtime-crawler

1import { describe, expect, it } from "vitest";2import { facebookAdapter } from "./facebook.ts";3import { entitiesFromDom } from "./adapter.ts";4import { classifyPage, classifyResponse, type CapturedResponse, type DomSnapshot } from "@src/observers";56const snap = (url: string, links: { href: string; text: string; region?: string }[]): DomSnapshot => ({7  url, title: "fb", landmarks: ["main", "feed"], video_elements: [], text_excerpt: "", scroll: { y: 0, height: 4000, viewport: 900 }, has_login_form: false, dialog_open: false,8  candidates: links.map((l, i) => ({ kind: "link", href: l.href, text: l.text, region: l.region ?? "feed", index: i, visible: true, top: i * 100, locator_hint: `a[href="${l.href}"]` })),9});1011describe("facebook adapter", () => {12  it("classifies pages from url grammar", () => {13    expect(classifyPage(snap("https://www.facebook.com/", []), facebookAdapter.pageTypeHints).page_type).toBe("HOME_FEED");14    expect(classifyPage(snap("https://www.facebook.com/search/top/?q=ia", []), facebookAdapter.pageTypeHints).page_type).toBe("SEARCH_RESULTS");15    expect(classifyPage(snap("https://www.facebook.com/Mila.Quebec/", []), facebookAdapter.pageTypeHints).page_type).toBe("PUBLIC_PAGE");16    expect(classifyPage(snap("https://www.facebook.com/Mila.Quebec/posts/pfbid0abc", []), facebookAdapter.pageTypeHints).page_type).toBe("POST_DETAIL");17    expect(classifyPage(snap("https://www.facebook.com/watch/?v=123456789", []), facebookAdapter.pageTypeHints).page_type).toBe("VIDEO_DETAIL");18    expect(classifyPage(snap("https://www.facebook.com/groups/quebecai/", []), facebookAdapter.pageTypeHints).page_type).toBe("GROUP");19    expect(classifyPage(snap("https://www.facebook.com/login/?next=x", []), facebookAdapter.pageTypeHints).page_type).toBe("LOGIN");20  });2122  it("extracts posts, videos, pages and groups from links and drops navigation noise", () => {23    const s = snap("https://www.facebook.com/", [24      { href: "https://www.facebook.com/Mila.Quebec/posts/pfbid02AbC?__cft__[0]=xyz&__tn__=%2CO", text: "Mila lance un nouveau programme" },25      { href: "https://www.facebook.com/watch/?v=987654321012", text: "Conférence IA 12:34" },26      { href: "https://www.facebook.com/Mila.Quebec/", text: "Mila - Institut québécois d'IA" },27      { href: "https://www.facebook.com/groups/iaquebec/", text: "IA Québec" },28      { href: "https://www.facebook.com/marketplace/", text: "Marketplace", region: "navigation" },29      { href: "https://www.facebook.com/friends/", text: "Amis", region: "navigation" },30    ]);31    const ents = entitiesFromDom(s, facebookAdapter);32    const types = ents.map((e) => e.type).sort();33    expect(types).toEqual(["community", "page", "post", "video"]);34    const post = ents.find((e) => e.type === "post")!;35    expect(post.platform_id).toBe("pfbid02AbC");36    expect(post.url).not.toContain("__cft__");37    expect(ents.find((e) => e.type === "video")!.platform_id).toBe("987654321012");38  });3940  it("mines GraphQL payloads with __typename and {text} shapes (for(;;); prefixed)", () => {41    const body = "for (;;);" + JSON.stringify({42      data: { node: { __typename: "Story", id: "UzpfSTEwMDA6MTIz", post_id: "1234567890123456", message: { text: "Nouvelle bourse en IA au Québec" }, actors: [{ __typename: "Page", id: "100064", name: "Mila - Institut québécois d'IA", url: "https://www.facebook.com/Mila.Quebec/" }], attachments: [{ media: { __typename: "Video", id: "987654321012", videoId: "987654321012", title: { text: "Table ronde" }, playable_duration_in_ms: 754000 } }] } },43    });44    const r: CapturedResponse = { request_id: "r", url: "https://www.facebook.com/api/graphql/", method: "POST", status: 200, content_type: "application/x-javascript; charset=utf-8", resource_type: "xhr", body_size: body.length, body, post_data: "fb_api_req_friendly_name=CometNewsFeedPaginationQuery&doc_id=1", captured_at: new Date().toISOString() };45    const c = classifyResponse(r, facebookAdapter.classifierHints);46    expect(c.kind).toBe("graphql");47    expect(c.fingerprint.graphql_operation).toBe("CometNewsFeedPaginationQuery");48    const types = new Set(c.entities.map((e) => e.type));49    expect(types.has("post")).toBe(true);50    expect(types.has("page")).toBe(true);51    expect(types.has("video")).toBe(true);52    const post = c.entities.find((e) => e.type === "post")!;53    expect(post.text).toBe("Nouvelle bourse en IA au Québec");54    expect(post.author).toBe("Mila - Institut québécois d'IA");55    const page = c.entities.find((e) => e.type === "page")!;56    expect(page.url).toBe("https://facebook.com/Mila.Quebec");57  });58});59