spb/social-runtime-crawler

1import { describe, expect, it } from "vitest";2import { profileJson, guessFieldSemantics, shapeSignature } from "./SchemaProfiler.ts";3import { mineEntities, classifyResponse, type CapturedResponse } from "./ResponseClassifier.ts";4import { youtubeAdapter } from "@src/connectors";56// Fixture shaped like an innertube search response (property names are *not* assumed by the profiler).7const youtubeLike = {8  contents: {9    sectionList: {10      items: [11        { videoRenderer: { videoId: "dQw4w9WgXcQ", title: { runs: [{ text: "IA au Québec — table ronde" }] }, ownerText: { runs: [{ text: "Chaîne Exemple" }] }, lengthText: { simpleText: "12:34" }, viewCountText: { simpleText: "1,2 M de vues" }, thumbnail: { thumbnails: [{ url: "https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg", width: 480, height: 360 }] } } },12        { videoRenderer: { videoId: "abcdefghijk", title: { runs: [{ text: "Montréal AI meetup" }] }, ownerText: { runs: [{ text: "Autre Chaîne" }] }, lengthText: { simpleText: "1:02:03" }, viewCountText: { simpleText: "48 k vues" }, thumbnail: { thumbnails: [{ url: "https://i.ytimg.com/vi/abcdefghijk/hqdefault.jpg", width: 480, height: 360 }] } } },13        { channelRenderer: { channelId: "UC1234567890abcdefghij", title: { simpleText: "Québec IA" }, subscriberCountText: { simpleText: "12 k abonnés" } } },14      ],15    },16    continuation: { token: "EpMBEgVoZWxsbxqJAVNCU0NBUXFsY1ZvVFVIZG9kMlV5UmtsbGRuZG1kMkkyYjBOaGRIZG9UbGxSVUZGVmxWMFVrMWFTV2xhVjFwWGRGUlY" },17  },18};1920describe("SchemaProfiler", () => {21  it("infers field semantics from value shapes and key names", () => {22    expect(guessFieldSemantics("videoId", ["dQw4w9WgXcQ", "abcdefghijk"])[0]!.kind).toBe("identifier");23    expect(guessFieldSemantics("url", ["https://i.ytimg.com/vi/x/hq.jpg"])[0]!.kind).toBe("thumbnail_url");24    expect(guessFieldSemantics("simpleText", ["12:34", "1:02:03"])[0]!.kind).toBe("duration");25    expect(guessFieldSemantics("handle", ["@simon", "@bob"])[0]!.kind).toBe("username");26    expect(guessFieldSemantics("created_utc", [1700000000, 1700000500])[0]!.kind).toBe("timestamp");27  });2829  it("produces a stable shape hash independent of values", () => {30    const a = shapeSignature({ id: "1", name: "a", tags: ["x"] });31    const b = shapeSignature({ name: "zzz", id: "999", tags: ["y", "z"] });32    expect(a).toBe(b);33  });3435  it("finds repeated objects and candidate entity lists", () => {36    const p = profileJson(youtubeLike);37    expect(p.repeated_object_paths).toContain("contents.sectionList.items");38    expect(p.fields.some((f) => f.path.endsWith("videoId") && f.semantic[0]!.kind === "identifier")).toBe(true);39    expect(p.candidate_entity_types.length).toBeGreaterThan(0);40  });41});4243describe("mineEntities (generic JSON entity miner)", () => {44  it("extracts videos and channels with collapsed rich text and provenance", () => {45    const ents = mineEntities(youtubeLike, youtubeAdapter.classifierHints);46    const videos = ents.filter((e) => e.type === "video");47    const channels = ents.filter((e) => e.type === "channel");48    expect(videos).toHaveLength(2);49    expect(channels).toHaveLength(1);50    const v = videos.find((e) => e.platform_id === "dQw4w9WgXcQ")!;51    expect(v.name).toBe("IA au Québec — table ronde");52    expect(v.author).toBe("Chaîne Exemple");53    expect(v.media?.duration_s).toBe(12 * 60 + 34);54    expect(v.metrics?.views).toBe(1_200_000);55    expect(v.url).toBe("https://youtube.com/watch?v=dQw4w9WgXcQ");56    expect(v.provenance[0]!.surface).toBe("network");57    expect(Object.keys(v.fields)).toContain("platform_id");58  });5960  it("classifies a JSON response end-to-end", () => {61    const r: CapturedResponse = { request_id: "req_1", url: "https://www.youtube.com/youtubei/v1/search?prettyPrint=false", method: "POST", status: 200, content_type: "application/json; charset=UTF-8", resource_type: "xhr", body_size: 100, body: JSON.stringify(youtubeLike), captured_at: new Date().toISOString() };62    const c = classifyResponse(r, youtubeAdapter.classifierHints);63    expect(c.kind).toBe("json");64    expect(c.entities.length).toBe(3);65    expect(c.fingerprint.observed_entity_types).toContain("video");66    expect(c.fingerprint.path_pattern).toBe("/youtubei/v1/search");67    expect(c.confidence).toBeGreaterThan(0.8);68  });69});70