SPB Git forge
spboucher.ai ↗

spb/social-runtime-crawler

Public
7commits 1branches 0releases
229.0 KBsize
maindefault branch
12 days agolast push
TypeScript 91.8% HTML 3.2% JavaScript 3% SQL 1.4% CSS 0.7%

Facebook adapter (Phase 2): url grammar, page hints, GraphQL __typename vocabulary, {text} collapsers, noise filter, consent dismissal

- observers: __typename → entity type, nested author names, anti-hijack prefixes recognised as JSON/GraphQL
- browser: interactive login detects the session cookie (no TTY needed), 15-min window
- api/console: platforms with an adapter are discovered from the connectors registry; Facebook presets
- tests: facebook.test.ts (24 tests total)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 12 days ago (Sep 12, 2026) parent 94bf3a0

12 changed files +276 −26

modified apps/api/package.json +2 −1
@@ -6,7 +6,8 @@
6 6 "dependencies": {
7 7 "@src/shared": "workspace:*",
8 8 "@src/storage": "workspace:*",
9 − "@src/platform-model": "workspace:*"
9 + "@src/platform-model": "workspace:*",
10 + "@src/connectors": "workspace:*"
10 11 },
11 12 "scripts": {
12 13 "start": "tsx src/server.ts"
modified apps/api/src/jobs.ts +2 −1
@@ -4,6 +4,7 @@ import path from "node:path";
4 4 import { fileURLToPath } from "node:url";
5 5 import { createLogger, isPlatform, newId, type AgentMode, type AppConfig } from "@src/shared";
6 6 import type { PostgresStore } from "@src/storage";
7 +import { hasAdapter, SUPPORTED_PLATFORMS } from "@src/connectors";
7 8
8 9 const log = createLogger("jobs");
9 10 const here = path.dirname(fileURLToPath(import.meta.url));
@@ -40,7 +41,7 @@ export class JobRunner {
40 41
41 42 async start(req: JobRequest): Promise<{ job_id: string }> {
42 43 if (!isPlatform(req.platform)) throw new Error("unknown platform");
43 − if (!["youtube", "reddit"].includes(req.platform)) throw new Error(`no adapter for ${req.platform} yet (Phase 1 = youtube, reddit)`);
44 + if (!hasAdapter(req.platform)) throw new Error(`no adapter for ${req.platform} yet (available: ${SUPPORTED_PLATFORMS.join(", ")})`);
44 45 if (this.running.size >= this.maxConcurrent) throw new Error(`max ${this.maxConcurrent} concurrent jobs`);
45 46 const job_id = newId("job");
46 47 const mode = req.mode ?? "research";
modified apps/api/src/queries.ts +3 −2
@@ -3,6 +3,7 @@ import path from "node:path";
3 3 import type { PostgresStore } from "@src/storage";
4 4 import { PLATFORMS, type AppConfig, type Platform } from "@src/shared";
5 5 import { PlatformModel } from "@src/platform-model";
6 +import { hasAdapter } from "@src/connectors";
6 7
7 8 /** Read-side queries for the research console. All read PostgreSQL; files (models, manifests, session dirs) complete them. */
8 9 export class Queries {
@@ -60,9 +61,9 @@ export class Queries {
60 61 return PLATFORMS.map((p) => {
61 62 const file = path.join(this.cfg.platformModelDir, p, "platform_model.json");
62 63 const manifest = path.join(process.cwd(), "connectors", p, "manifest.json");
63 − if (!fs.existsSync(file)) return { platform: p, learned: false, adapter: p === "youtube" || p === "reddit", confidence: 0, page_types: 0, entity_types: 0, navigation_actions: 0, network_schemas: 0, media_patterns: 0, sessions: 0, has_manifest: fs.existsSync(manifest), profiles: this.profilesFor(p) };
64 + if (!fs.existsSync(file)) return { platform: p, learned: false, adapter: hasAdapter(p), confidence: 0, page_types: 0, entity_types: 0, navigation_actions: 0, network_schemas: 0, media_patterns: 0, sessions: 0, has_manifest: fs.existsSync(manifest), profiles: this.profilesFor(p) };
64 65 const m = new PlatformModel(p, this.cfg.platformModelDir);
65 − return { learned: true, adapter: p === "youtube" || p === "reddit", has_manifest: fs.existsSync(manifest), profiles: this.profilesFor(p), ...m.summary() };
66 + return { learned: true, adapter: hasAdapter(p), has_manifest: fs.existsSync(manifest), profiles: this.profilesFor(p), ...m.summary() };
66 67 });
67 68 }
68 69
modified apps/dashboard/src/app/jobs/page.tsx +3 −0
@@ -27,6 +27,8 @@ const PRESETS = [
27 27 { label: "Learn · YouTube runtime", platform: "youtube", mode: "learn", query: "", goal: "", minutes: 15, actions: 80, media: 1 },
28 28 { label: "Observe · YouTube home feed", platform: "youtube", mode: "observe", query: "", goal: "Observe the recommendation feed without navigating", minutes: 5, actions: 30, media: 0 },
29 29 { label: "Research · Reddit r/Quebec AI", platform: "reddit", mode: "research", query: "intelligence artificielle Québec", goal: "Discover public Reddit posts and users discussing AI in Quebec", minutes: 10, actions: 60, media: 1 },
30 + { label: "Facebook · public page prototype (§52)", platform: "facebook", mode: "research", query: "", goal: "Observe one public Facebook page: its posts, videos, comments and the public pages/profiles they link to", minutes: 8, actions: 40, media: 1 },
31 + { label: "Observe · Facebook feed", platform: "facebook", mode: "observe", query: "", goal: "Observe the Facebook home feed and its recommendation exposure without navigating", minutes: 5, actions: 30, media: 0 },
30 32 ];
31 33
32 34 export default function Missions() {
@@ -78,6 +80,7 @@ export default function Missions() {
78 80 <select value={form.platform} onChange={(e) => setForm({ ...form, platform: e.target.value })} className={input}>
79 81 <option value="youtube">youtube</option>
80 82 <option value="reddit">reddit</option>
83 + <option value="facebook">facebook</option>
81 84 </select>
82 85 </label>
83 86 <label className="text-xs text-dim">
modified apps/dashboard/src/app/platforms/page.tsx +1 −1
@@ -19,7 +19,7 @@ interface P {
19 19 profiles: { alias: string; has_state: boolean }[];
20 20 }
21 21
22 −const PHASE: Record<string, string> = { youtube: "Phase 1", reddit: "Phase 1", facebook: "Phase 2", instagram: "Phase 2", tiktok: "Phase 3", x: "Phase 3", linkedin: "Phase 4", threads: "Phase 4" };
22 +const PHASE: Record<string, string> = { youtube: "Phase 1", reddit: "Phase 1", facebook: "Phase 2 · adapter", instagram: "Phase 2", tiktok: "Phase 3", x: "Phase 3", linkedin: "Phase 4", threads: "Phase 4" };
23 23
24 24 export default function Platforms() {
25 25 const { data, loading } = useApi<P[]>("/platforms", { refreshMs: 15_000 });
modified packages/browser/src/login.ts +55 −13
@@ -1,5 +1,5 @@
1 1 import readline from "node:readline";
2 −import { createLogger, type Platform } from "@src/shared";
2 +import { createLogger, sleep, type Platform } from "@src/shared";
3 3 import { SocialBrowserSession } from "./session.ts";
4 4
5 5 const log = createLogger("login");
@@ -15,23 +15,65 @@ export const LOGIN_URLS: Record<Platform, string> = {
15 15 threads: "https://www.threads.net/login",
16 16 };
17 17
18 +/** Session cookies whose presence means "the human is logged in" (read-only check; values are never logged). */
19 +export const AUTH_COOKIES: Record<Platform, RegExp> = {
20 + youtube: /^(SAPISID|LOGIN_INFO|__Secure-3PAPISID)$/,
21 + reddit: /^(reddit_session|token_v2)$/,
22 + facebook: /^c_user$/,
23 + instagram: /^(sessionid|ds_user_id)$/,
24 + tiktok: /^(sessionid|sid_tt)$/,
25 + x: /^(auth_token)$/,
26 + linkedin: /^(li_at)$/,
27 + threads: /^(sessionid|ds_user_id)$/,
28 +};
29 +
30 +export async function isAuthenticated(session: SocialBrowserSession, platform: Platform): Promise<boolean> {
31 + const cookies = await session.getContext().cookies().catch(() => []);
32 + return cookies.some((c) => AUTH_COOKIES[platform].test(c.name) && c.value.length > 0);
33 +}
34 +
18 35 /**
19 − * Human-in-the-loop authentication (§7): open a headed browser on the platform,
20 − * let the operator log in, then persist the profile. No credentials are read, typed or stored by the crawler.
36 + * Human-in-the-loop authentication (§7): open a headed browser on the platform and wait until the operator
37 + * has logged in — detected by the platform's session cookie, or by <Enter> when a TTY is attached.
38 + * No credentials are read, typed or stored by the crawler; the persistent profile is the only state kept.
21 39 */
22 −export async function interactiveLogin(opts: { platform: Platform; accountAlias: string; profilesDir: string; channel?: string }): Promise<void> {
40 +export async function interactiveLogin(opts: { platform: Platform; accountAlias: string; profilesDir: string; channel?: string; timeoutMin?: number }): Promise<boolean> {
23 41 const session = new SocialBrowserSession({ ...opts, headless: false });
24 42 await session.start();
25 43 await session.navigate(LOGIN_URLS[opts.platform]);
26 − log.info("Browser is open. Log in manually, dismiss consent dialogs, then come back here.");
27 − await waitForEnter("Press <Enter> once you are logged in (the profile will be saved)… ");
44 + if (await isAuthenticated(session, opts.platform)) {
45 + log.info("profile already authenticated", { platform: opts.platform, alias: opts.accountAlias });
46 + await sleep(1500);
47 + await session.stop();
48 + return true;
49 + }
50 + log.info(`Browser is open on ${opts.platform}. Log in manually (and pass any 2FA yourself). I will detect the session automatically.`);
51 + const deadline = Date.now() + (opts.timeoutMin ?? 15) * 60_000;
52 + let enterPressed = false;
53 + const rl = process.stdin.isTTY ? readline.createInterface({ input: process.stdin, output: process.stdout }) : undefined;
54 + rl?.question("…or press <Enter> once you are logged in. ", () => (enterPressed = true));
55 + let ok = false;
56 + while (Date.now() < deadline) {
57 + if (session.getInfo().health === "crashed") break;
58 + if (await isAuthenticated(session, opts.platform)) {
59 + ok = true;
60 + break;
61 + }
62 + if (enterPressed) {
63 + ok = await isAuthenticated(session, opts.platform);
64 + break;
65 + }
66 + await sleep(2000);
67 + }
68 + rl?.close();
69 + if (ok) {
70 + // Let the app finish writing its storage, then land on the home page so the profile starts in a clean state.
71 + await sleep(4000);
72 + await session.navigate(LOGIN_URLS[opts.platform].replace(/\/login.*$/, "/")).catch(() => {});
73 + await sleep(2500);
74 + }
28 75 const page = session.getPage();
29 − const cookies = await session.getContext().cookies();
30 − log.info("profile saved", { platform: opts.platform, alias: opts.accountAlias, url: page.url(), cookies: cookies.length, profile: session.profilePath });
76 + log.info(ok ? "profile saved — authenticated" : "no authenticated session detected (timeout)", { platform: opts.platform, alias: opts.accountAlias, url: page.url(), profile: session.profilePath });
31 77 await session.stop();
32 −}
33 −
34 −function waitForEnter(prompt: string): Promise<void> {
35 − const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
36 − return new Promise((res) => rl.question(prompt, () => (rl.close(), res())));
78 + return ok;
37 79 }
added packages/connectors/src/facebook.test.ts +58 −0
@@ -0,0 +1,58 @@
1 +import { describe, expect, it } from "vitest";
2 +import { facebookAdapter } from "./facebook.ts";
3 +import { entitiesFromDom } from "./adapter.ts";
4 +import { classifyPage, classifyResponse, type CapturedResponse, type DomSnapshot } from "@src/observers";
5 +
6 +const snap = (url: string, links: { href: string; text: string; region?: string }[]): DomSnapshot => ({
7 + url, title: "fb", landmarks: ["main", "feed"], video_elements: [], text_excerpt: "", scroll: { y: 0, height: 4000, viewport: 900 }, has_login_form: false, dialog_open: false,
8 + candidates: links.map((l, i) => ({ kind: "link", href: l.href, text: l.text, region: l.region ?? "feed", index: i, visible: true, top: i * 100, locator_hint: `a[href="${l.href}"]` })),
9 +});
10 +
11 +describe("facebook adapter", () => {
12 + it("classifies pages from url grammar", () => {
13 + expect(classifyPage(snap("https://www.facebook.com/", []), facebookAdapter.pageTypeHints).page_type).toBe("HOME_FEED");
14 + expect(classifyPage(snap("https://www.facebook.com/search/top/?q=ia", []), facebookAdapter.pageTypeHints).page_type).toBe("SEARCH_RESULTS");
15 + expect(classifyPage(snap("https://www.facebook.com/Mila.Quebec/", []), facebookAdapter.pageTypeHints).page_type).toBe("PUBLIC_PAGE");
16 + expect(classifyPage(snap("https://www.facebook.com/Mila.Quebec/posts/pfbid0abc", []), facebookAdapter.pageTypeHints).page_type).toBe("POST_DETAIL");
17 + expect(classifyPage(snap("https://www.facebook.com/watch/?v=123456789", []), facebookAdapter.pageTypeHints).page_type).toBe("VIDEO_DETAIL");
18 + expect(classifyPage(snap("https://www.facebook.com/groups/quebecai/", []), facebookAdapter.pageTypeHints).page_type).toBe("GROUP");
19 + expect(classifyPage(snap("https://www.facebook.com/login/?next=x", []), facebookAdapter.pageTypeHints).page_type).toBe("LOGIN");
20 + });
21 +
22 + it("extracts posts, videos, pages and groups from links and drops navigation noise", () => {
23 + const s = snap("https://www.facebook.com/", [
24 + { href: "https://www.facebook.com/Mila.Quebec/posts/pfbid02AbC?__cft__[0]=xyz&__tn__=%2CO", text: "Mila lance un nouveau programme" },
25 + { href: "https://www.facebook.com/watch/?v=987654321012", text: "Conférence IA 12:34" },
26 + { href: "https://www.facebook.com/Mila.Quebec/", text: "Mila - Institut québécois d'IA" },
27 + { href: "https://www.facebook.com/groups/iaquebec/", text: "IA Québec" },
28 + { href: "https://www.facebook.com/marketplace/", text: "Marketplace", region: "navigation" },
29 + { href: "https://www.facebook.com/friends/", text: "Amis", region: "navigation" },
30 + ]);
31 + const ents = entitiesFromDom(s, facebookAdapter);
32 + const types = ents.map((e) => e.type).sort();
33 + expect(types).toEqual(["community", "page", "post", "video"]);
34 + const post = ents.find((e) => e.type === "post")!;
35 + expect(post.platform_id).toBe("pfbid02AbC");
36 + expect(post.url).not.toContain("__cft__");
37 + expect(ents.find((e) => e.type === "video")!.platform_id).toBe("987654321012");
38 + });
39 +
40 + it("mines GraphQL payloads with __typename and {text} shapes (for(;;); prefixed)", () => {
41 + const body = "for (;;);" + JSON.stringify({
42 + data: { node: { __typename: "Story", id: "UzpfSTEwMDA6MTIz", post_id: "1234567890123456", message: { text: "Nouvelle bourse en IA au Québec" }, actors: [{ __typename: "Page", id: "100064", name: "Mila - Institut québécois d'IA", url: "https://www.facebook.com/Mila.Quebec/" }], attachments: [{ media: { __typename: "Video", id: "987654321012", videoId: "987654321012", title: { text: "Table ronde" }, playable_duration_in_ms: 754000 } }] } },
43 + });
44 + const r: CapturedResponse = { request_id: "r", url: "https://www.facebook.com/api/graphql/", method: "POST", status: 200, content_type: "application/x-javascript; charset=utf-8", resource_type: "xhr", body_size: body.length, body, post_data: "fb_api_req_friendly_name=CometNewsFeedPaginationQuery&doc_id=1", captured_at: new Date().toISOString() };
45 + const c = classifyResponse(r, facebookAdapter.classifierHints);
46 + expect(c.kind).toBe("graphql");
47 + expect(c.fingerprint.graphql_operation).toBe("CometNewsFeedPaginationQuery");
48 + const types = new Set(c.entities.map((e) => e.type));
49 + expect(types.has("post")).toBe(true);
50 + expect(types.has("page")).toBe(true);
51 + expect(types.has("video")).toBe(true);
52 + const post = c.entities.find((e) => e.type === "post")!;
53 + expect(post.text).toBe("Nouvelle bourse en IA au Québec");
54 + expect(post.author).toBe("Mila - Institut québécois d'IA");
55 + const page = c.entities.find((e) => e.type === "page")!;
56 + expect(page.url).toBe("https://facebook.com/Mila.Quebec");
57 + });
58 +});
added packages/connectors/src/facebook.ts +114 −0
@@ -0,0 +1,114 @@
1 +import type { Page } from "playwright";
2 +import type { EntityType, ObservedEntity, PageType } from "@src/shared";
3 +import type { ClassifierHints, DomSnapshot } from "@src/observers";
4 +import type { SocialPlatformAdapter, UrlEntityRule } from "./adapter.ts";
5 +
6 +/**
7 + * Facebook adapter — Phase 2 (§52 experimental prototype).
8 + * Platform knowledge only: URL grammar, page hints, GraphQL `__typename` vocabulary, `{text}` rich-text shape,
9 + * reserved navigation paths and consent/dialog dismissal. No GraphQL doc_id, no endpoint, no CSS class.
10 + * Facebook responses arrive as `for (;;);`-prefixed JSON or NDJSON on POST /api/graphql/ with
11 + * `fb_api_req_friendly_name` — both are handled generically by the network layer.
12 + */
13 +const RESERVED = /^\/(marketplace|gaming|friends|bookmarks|notifications|messages|messenger|settings|help|privacy|policies|policy|login|logout|recover|reels?|watch\/?$|watch\/(live|saved|shows|latest)|stories|memories|saved|pages\/?$|pages\/(create|discover)|fundraisers|events\/?$|ads|business|about|careers|developers|terms|legal|cookie|donate|places|weather|games|jobs|dating|crisisresponse|community|search\/?$|bookmarks|feeds?\/?$|me\/?$|home\.php|checkpoint|sharer|dialog|plugins|ajax|composer|mediaset|photo\.php\?.*type=3|badges|topics|hashtag\/?$|lite|mobile|profile\.php\?.*sk=|groups\/(feed|discover|joins|create)\/?)/i;
14 +
15 +const classifierHints: ClassifierHints = {
16 + platform: "facebook",
17 + textCollapsers: [
18 + (v) => (v && typeof v === "object" && typeof (v as { text?: unknown }).text === "string" ? (v as { text: string }).text : undefined),
19 + (v) => {
20 + if (!v || typeof v !== "object") return undefined;
21 + const o = v as { ranges?: unknown; text?: unknown; delight_ranges?: unknown };
22 + return typeof o.text === "string" ? o.text : undefined;
23 + },
24 + ],
25 + idKeys: { post_id: "post", story_id: "post", video_id: "video", videoId: "video", page_id: "page", group_id: "community", comment_id: "comment", legacy_story_hideable_id: "post" },
26 + typenames: { Story: "post", Video: "video", Photo: "image", User: "profile", Page: "page", Group: "community", Comment: "comment", Event: "event", Hashtag: "hashtag", Reel: "video" },
27 + urlForId: (type: EntityType, id: string) => {
28 + if (!/^\d{6,}$/.test(id)) return undefined;
29 + if (type === "video") return `https://www.facebook.com/watch/?v=${id}`;
30 + if (type === "page" || type === "profile") return `https://www.facebook.com/profile.php?id=${id}`;
31 + if (type === "community") return `https://www.facebook.com/groups/${id}`;
32 + return undefined;
33 + },
34 +};
35 +
36 +const urlEntityRules: UrlEntityRule[] = [
37 + { pattern: /^\/watch\/?\?(?:.*&)?v=(\d+)/, type: "video", id: (m) => m[1], confidence: 0.95 },
38 + { pattern: /^\/reel\/(\d+)/, type: "video", id: (m) => m[1], confidence: 0.95 },
39 + { pattern: /^\/[^/]+\/videos\/(?:[^/]+\/)?(\d+)/, type: "video", id: (m) => m[1], confidence: 0.93 },
40 + { pattern: /^\/video\.php\?(?:.*&)?v=(\d+)/, type: "video", id: (m) => m[1], confidence: 0.9 },
41 + { pattern: /^\/[^/]+\/posts\/(pfbid[\w]+|\d+)/, type: "post", id: (m) => m[1], confidence: 0.95 },
42 + { pattern: /^\/permalink\.php\?(?:.*&)?story_fbid=(pfbid[\w]+|\d+)/, type: "post", id: (m) => m[1], confidence: 0.93 },
43 + { pattern: /^\/story\.php\?(?:.*&)?story_fbid=(pfbid[\w]+|\d+)/, type: "post", id: (m) => m[1], confidence: 0.9 },
44 + { pattern: /^\/groups\/([\w.]+)\/(?:posts|permalink)\/(\d+)/, type: "post", id: (m) => m[2], confidence: 0.93 },
45 + { pattern: /^\/photo\/?\?(?:.*&)?fbid=(\d+)/, type: "image", id: (m) => m[1], confidence: 0.9 },
46 + { pattern: /^\/[^/]+\/photos\/(?:[^/]+\/)?(\d+)/, type: "image", id: (m) => m[1], confidence: 0.85 },
47 + { pattern: /^\/groups\/([\w.]+)\/?(?:\?|$)/, type: "community", id: (m) => m[1]?.toLowerCase(), confidence: 0.92 },
48 + { pattern: /^\/events\/(\d+)/, type: "event", id: (m) => m[1], confidence: 0.92 },
49 + { pattern: /^\/hashtag\/([\w]+)/, type: "hashtag", id: (m) => m[1]?.toLowerCase(), confidence: 0.9 },
50 + { pattern: /^\/profile\.php\?(?:.*&)?id=(\d+)/, type: "profile", id: (m) => m[1], confidence: 0.9 },
51 + { pattern: /^\/people\/[^/]+\/(pfbid[\w]+|\d+)/, type: "profile", id: (m) => m[1], confidence: 0.9 },
52 + // Vanity URL: /<slug>/ — a public Page or a Person; the type is refined later by GraphQL __typename evidence.
53 + { pattern: /^\/([A-Za-z0-9.]{3,60})\/?(?:\?|$)/, type: "page", id: (m) => m[1]?.toLowerCase(), confidence: 0.6 },
54 +];
55 +
56 +export const facebookAdapter: SocialPlatformAdapter = {
57 + platform: "facebook",
58 + homeUrl: "https://www.facebook.com/",
59 + hosts: /(^|\.)facebook\.com$|(^|\.)fb\.com$|^fb\.watch$/,
60 + pageTypeHints: [
61 + { pattern: /^\/?(\?|$)|^\/home\.php/, page_type: "HOME_FEED", confidence: 0.9 },
62 + { pattern: /^\/search\//, page_type: "SEARCH_RESULTS", confidence: 0.95 },
63 + { pattern: /^\/watch\/?\?v=|^\/reel\/|\/videos\/\d+/, page_type: "VIDEO_DETAIL", confidence: 0.93 },
64 + { pattern: /\/posts\/|permalink\.php|story\.php|\/photo\/?\?fbid=|\/photos\//, page_type: "POST_DETAIL", confidence: 0.92 },
65 + { pattern: /^\/groups\/[\w.]+\/?(\?|$)/, page_type: "GROUP", confidence: 0.92 },
66 + { pattern: /^\/profile\.php\?id=|^\/people\//, page_type: "PROFILE", confidence: 0.9 },
67 + { pattern: /^\/login|^\/checkpoint|^\/recover/, page_type: "LOGIN", confidence: 0.95 },
68 + { pattern: /^\/[A-Za-z0-9.]{3,60}\/?(\?|$)/, page_type: "PUBLIC_PAGE", confidence: 0.7 },
69 + ],
70 + urlEntityRules,
71 + classifierHints,
72 + searchUrl: (q) => `https://www.facebook.com/search/top/?q=${encodeURIComponent(q)}`,
73 + isNoiseLink: (href, text) => RESERVED.test(href.pathname + href.search) || /^(Accueil|Home|Vidéo|Video|Watch|Marketplace|Groupes|Groups|Gaming|Jeux|Amis|Friends|Menu|Messenger|Notifications|Profil|Profile|Voir plus|See more|J’aime|J'aime|Like|Commenter|Comment|Partager|Share)$/i.test(text.trim()),
74 + entityLabel: (t) => ({ post: "post", video: "video", image: "photo", page: "page or profile", profile: "profile", community: "group", comment: "comment", event: "event", hashtag: "hashtag" } as Partial<Record<EntityType, string>>)[t] ?? t,
75 + refineEntities: (entities: ObservedEntity[], snapshot: DomSnapshot) => {
76 + // Feed cards: a `[role=article]` whose link is a post is the post itself; the same card also exposes its author page.
77 + // We keep both, but mark posts seen in the feed region so the planner knows their exposure context (§44).
78 + for (const e of entities) if (e.type === "post" && /feed/.test(e.context ?? "")) e.context = `feed item ${e.context}`;
79 + // Current post/video on a detail page
80 + const cur = entities.find((e) => e.url && samePath(e.url, snapshot.url) && (e.type === "post" || e.type === "video" || e.type === "image"));
81 + if (cur) {
82 + cur.context = `current ${cur.type}`;
83 + return [cur, ...entities.filter((e) => e !== cur)];
84 + }
85 + return entities;
86 + },
87 + expectedEntities: (t: PageType) => ({ HOME_FEED: 5, SEARCH_RESULTS: 5, PUBLIC_PAGE: 4, PROFILE: 3, GROUP: 4, VIDEO_DETAIL: 2, POST_DETAIL: 2 } as Partial<Record<PageType, number>>)[t] ?? 0,
88 + dismissOverlays: async (page: Page) => {
89 + // Consent banner and "log in to continue" nag dialogs — dismiss controls only, never a content interaction.
90 + for (const sel of [
91 + 'button[data-cookiebanner="accept_button"]',
92 + '[aria-label="Allow all cookies"]',
93 + '[aria-label="Autoriser tous les cookies"]',
94 + '[aria-label="Autoriser les cookies essentiels et optionnels"]',
95 + 'div[role=dialog] [aria-label="Close"]',
96 + 'div[role=dialog] [aria-label="Fermer"]',
97 + ]) {
98 + const b = page.locator(sel).first();
99 + if (await b.isVisible().catch(() => false)) await b.click({ timeout: 2000 }).catch(() => {});
100 + }
101 + },
102 +};
103 +
104 +function samePath(a: string, b: string): boolean {
105 + try {
106 + const ua = new URL(a);
107 + const ub = new URL(b);
108 + if (ua.pathname !== ub.pathname) return false;
109 + for (const k of ["v", "story_fbid", "fbid", "id"]) if (ua.searchParams.get(k) !== ub.searchParams.get(k)) return false;
110 + return true;
111 + } catch {
112 + return false;
113 + }
114 +}
modified packages/connectors/src/index.ts +6 −2
@@ -2,18 +2,22 @@ import type { Platform } from "@src/shared";
2 2 import type { SocialPlatformAdapter } from "./adapter.ts";
3 3 import { youtubeAdapter } from "./youtube.ts";
4 4 import { redditAdapter } from "./reddit.ts";
5 +import { facebookAdapter } from "./facebook.ts";
5 6
6 7 export * from "./adapter.ts";
7 −export { youtubeAdapter, redditAdapter };
8 +export { youtubeAdapter, redditAdapter, facebookAdapter };
8 9
9 10 const ADAPTERS: Partial<Record<Platform, SocialPlatformAdapter>> = {
10 11 youtube: youtubeAdapter,
11 12 reddit: redditAdapter,
13 + facebook: facebookAdapter,
12 14 };
13 15
16 +export const SUPPORTED_PLATFORMS = Object.keys(ADAPTERS) as Platform[];
17 +
14 18 export function getAdapter(platform: Platform): SocialPlatformAdapter {
15 19 const a = ADAPTERS[platform];
16 − if (!a) throw new Error(`No adapter for platform "${platform}" yet (Phase 1 = youtube, reddit). Use PLATFORM LEARNING mode to bootstrap one.`);
20 + if (!a) throw new Error(`No adapter for platform "${platform}" yet (available: ${SUPPORTED_PLATFORMS.join(", ")}). Use PLATFORM LEARNING mode to bootstrap one.`);
17 21 return a;
18 22 }
19 23
modified packages/observers/src/network/ResponseClassifier.ts +28 −5
@@ -47,6 +47,8 @@ export interface ClassifierHints {
47 47 textCollapsers?: TextCollapser[];
48 48 /** Key names (case-insensitive substrings) that strongly denote an entity id on this platform. */
49 49 idKeys?: Record<string, EntityType>;
50 + /** GraphQL `__typename` vocabulary → entity type (generic GraphQL convention, values are platform knowledge). */
51 + typenames?: Record<string, EntityType>;
50 52 /** Build a canonical page URL from a platform id. */
51 53 urlForId?: (type: EntityType, id: string) => string | undefined;
52 54 }
@@ -70,7 +72,9 @@ export function classifyKind(r: CapturedResponse): ClassifiedResponse["kind"] {
70 72 if (/\.m3u8|\.mpd|mpegurl|dash\+xml|manifest/.test(u + " " + ct)) return "media_manifest";
71 73 if (/videoplayback|\.ts(\?|$)|\.m4s|\.mp4|video\/|audio\//.test(u + " " + ct)) return "media_segment";
72 74 if (ct.startsWith("image/")) return "image";
73 − if (ct.includes("json") || (r.body && /^\s*[[{]/.test(r.body))) {
75 + // Anti-hijack prefixes (`for (;;);`, `)]}'`) hide JSON behind a javascript content-type.
76 + const bodyStart = r.body?.replace(/^\)\]\}'\s*/, "").replace(/^for \(;;\);/, "").trimStart().slice(0, 1);
77 + if (ct.includes("json") || bodyStart === "{" || bodyStart === "[" || /\/graphql\/?(\?|$)/i.test(u)) {
74 78 if (/graphql/i.test(u) || (r.post_data && /"query"\s*:|operationName|doc_id|fb_api_req_friendly_name/.test(r.post_data))) return "graphql";
75 79 return "json";
76 80 }
@@ -128,10 +132,14 @@ export function mineEntities(root: unknown, hints: ClassifierHints, step?: numbe
128 132 if (!v || typeof v !== "object" || Array.isArray(v)) return;
129 133 const obj = v as Record<string, unknown>;
130 134 const keys = Object.keys(obj);
135 + // 0) GraphQL __typename is the strongest type evidence when the adapter knows the vocabulary
136 + const typename = typeof obj.__typename === "string" ? hints.typenames?.[obj.__typename] : undefined;
131 137 // 1) find an id
132 138 let idKey: string | undefined;
133 − let idType: EntityType | undefined;
139 + let idType: EntityType | undefined = typename;
140 + if (typename && typeof obj.id === "string" && obj.id.length >= 4) idKey = "id";
134 141 for (const k of keys) {
142 + if (idKey) break;
135 143 const kl = k.toLowerCase();
136 144 const hint = idKeyEntries.find(([hk]) => kl === hk);
137 145 if (hint && (typeof obj[k] === "string" || typeof obj[k] === "number")) {
@@ -160,8 +168,8 @@ export function mineEntities(root: unknown, hints: ClassifierHints, step?: numbe
160 168 return undefined;
161 169 };
162 170 const title = pick(/^(title|headline|name|displayName|fullName|display_name)$/i);
163 − const text = pick(/^(text|body|caption|description|selftext|descriptionSnippet|content|message)$/i);
164 − const author = pick(/^(author|ownerText|shortBylineText|longBylineText|author_name|username|channelName|user_name|screen_name|handle)$/i);
171 + const text = pick(/^(text|body|caption|description|selftext|descriptionSnippet|content|message|savable_description|body_text)$/i);
172 + const author = pick(/^(author|ownerText|shortBylineText|longBylineText|author_name|username|channelName|user_name|screen_name|handle|owner|actors?)$/i) ?? nestedName(obj, /^(owner|author|actors?|creator|profile|user|page|video_owner|feedback_owner)$/i, collapse);
165 173 const thumb = (() => {
166 174 let out: string | undefined;
167 175 walkJson(obj, (x, p) => {
@@ -184,7 +192,7 @@ export function mineEntities(root: unknown, hints: ClassifierHints, step?: numbe
184 192 else if (/comment|reply/i.test(idKey) || /parent_id|replyCount|depth/i.test(keys.join(" "))) type = "comment";
185 193 }
186 194
187 − const url = hints.urlForId?.(type, id) ?? (pick(/^(url|permalink|canonicalUrl|href|link)$/i) ?? undefined);
195 + const url = pick(/^(url|permalink|canonicalUrl|href|link|permalink_url|wwwURL|profile_url|share_url)$/i) ?? hints.urlForId?.(type, id) ?? undefined;
188 196 const fingerprint = `${hints.platform}:${type}:${id}`;
189 197 if (found.has(fingerprint)) return;
190 198
@@ -227,6 +235,21 @@ export function mineEntities(root: unknown, hints: ClassifierHints, step?: numbe
227 235 return [...found.values()];
228 236 }
229 237
238 +/** Author objects are often nested: `{ owner: { name, __typename: "Page" } }` → "name". */
239 +function nestedName(obj: Record<string, unknown>, keyRe: RegExp, collapse: (v: unknown) => string | undefined): string | undefined {
240 + for (const [k, v] of Object.entries(obj)) {
241 + if (!keyRe.test(k)) continue;
242 + const candidates = Array.isArray(v) ? v : [v];
243 + for (const c of candidates) {
244 + if (!c || typeof c !== "object") continue;
245 + const o = c as Record<string, unknown>;
246 + const s = collapse(o.name) ?? collapse(o.title) ?? collapse(o.display_name) ?? collapse(o.username);
247 + if (s) return s;
248 + }
249 + }
250 + return undefined;
251 +}
252 +
230 253 export function classifyResponse(r: CapturedResponse, hints: ClassifierHints): ClassifiedResponse {
231 254 const kind = classifyKind(r);
232 255 const { hostname, path_pattern } = pathPattern(r.url);
modified packages/shared/src/util.ts +1 −1
@@ -86,7 +86,7 @@ export function canonicalUrl(url: string): string {
86 86 u.hash = "";
87 87 // strip common tracking params
88 88 for (const k of [...u.searchParams.keys()]) {
89 − if (/^(utm_|fbclid|gclid|igshid|si|feature|pp|ref_src|ref_url|t)$/i.test(k)) u.searchParams.delete(k);
89 + if (/^(utm_|fbclid|gclid|igshid|si|feature|pp|ref_src|ref_url|t|ref|refid|rdid|mibextid|locale|__cft__|__tn__|__xts__|eid|hc_ref|notif_id|notif_t|comment_tracking|source|sfnsn|extid)/i.test(k)) u.searchParams.delete(k);
90 90 }
91 91 u.hostname = u.hostname.toLowerCase().replace(/^(www|m|mobile)\./, "");
92 92 let s = u.toString();
modified pnpm-lock.yaml +3 −0
@@ -23,6 +23,9 @@ importers:
23 23
24 24 apps/api:
25 25 dependencies:
26 + '@src/connectors':
27 + specifier: workspace:*
28 + version: link:../../packages/connectors
26 29 '@src/platform-model':
27 30 specifier: workspace:*
28 31 version: link:../../packages/platform-model
29 32