import { describe, expect, it } from "vitest"; import { profileJson, guessFieldSemantics, shapeSignature } from "./SchemaProfiler.ts"; import { mineEntities, classifyResponse, type CapturedResponse } from "./ResponseClassifier.ts"; import { youtubeAdapter } from "@src/connectors"; // Fixture shaped like an innertube search response (property names are *not* assumed by the profiler). const youtubeLike = { contents: { sectionList: { items: [ { videoRenderer: { videoId: "dQw4w9WgXcQ", title: { runs: [{ text: "IA au Québec — table ronde" }] }, ownerText: { runs: [{ text: "Chaîne Exemple" }] }, lengthText: { simpleText: "12:34" }, viewCountText: { simpleText: "1,2 M de vues" }, thumbnail: { thumbnails: [{ url: "https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg", width: 480, height: 360 }] } } }, { videoRenderer: { videoId: "abcdefghijk", title: { runs: [{ text: "Montréal AI meetup" }] }, ownerText: { runs: [{ text: "Autre Chaîne" }] }, lengthText: { simpleText: "1:02:03" }, viewCountText: { simpleText: "48 k vues" }, thumbnail: { thumbnails: [{ url: "https://i.ytimg.com/vi/abcdefghijk/hqdefault.jpg", width: 480, height: 360 }] } } }, { channelRenderer: { channelId: "UC1234567890abcdefghij", title: { simpleText: "Québec IA" }, subscriberCountText: { simpleText: "12 k abonnés" } } }, ], }, continuation: { token: "EpMBEgVoZWxsbxqJAVNCU0NBUXFsY1ZvVFVIZG9kMlV5UmtsbGRuZG1kMkkyYjBOaGRIZG9UbGxSVUZGVmxWMFVrMWFTV2xhVjFwWGRGUlY" }, }, }; describe("SchemaProfiler", () => { it("infers field semantics from value shapes and key names", () => { expect(guessFieldSemantics("videoId", ["dQw4w9WgXcQ", "abcdefghijk"])[0]!.kind).toBe("identifier"); expect(guessFieldSemantics("url", ["https://i.ytimg.com/vi/x/hq.jpg"])[0]!.kind).toBe("thumbnail_url"); expect(guessFieldSemantics("simpleText", ["12:34", "1:02:03"])[0]!.kind).toBe("duration"); expect(guessFieldSemantics("handle", ["@simon", "@bob"])[0]!.kind).toBe("username"); expect(guessFieldSemantics("created_utc", [1700000000, 1700000500])[0]!.kind).toBe("timestamp"); }); it("produces a stable shape hash independent of values", () => { const a = shapeSignature({ id: "1", name: "a", tags: ["x"] }); const b = shapeSignature({ name: "zzz", id: "999", tags: ["y", "z"] }); expect(a).toBe(b); }); it("finds repeated objects and candidate entity lists", () => { const p = profileJson(youtubeLike); expect(p.repeated_object_paths).toContain("contents.sectionList.items"); expect(p.fields.some((f) => f.path.endsWith("videoId") && f.semantic[0]!.kind === "identifier")).toBe(true); expect(p.candidate_entity_types.length).toBeGreaterThan(0); }); }); describe("mineEntities (generic JSON entity miner)", () => { it("extracts videos and channels with collapsed rich text and provenance", () => { const ents = mineEntities(youtubeLike, youtubeAdapter.classifierHints); const videos = ents.filter((e) => e.type === "video"); const channels = ents.filter((e) => e.type === "channel"); expect(videos).toHaveLength(2); expect(channels).toHaveLength(1); const v = videos.find((e) => e.platform_id === "dQw4w9WgXcQ")!; expect(v.name).toBe("IA au Québec — table ronde"); expect(v.author).toBe("Chaîne Exemple"); expect(v.media?.duration_s).toBe(12 * 60 + 34); expect(v.metrics?.views).toBe(1_200_000); expect(v.url).toBe("https://youtube.com/watch?v=dQw4w9WgXcQ"); expect(v.provenance[0]!.surface).toBe("network"); expect(Object.keys(v.fields)).toContain("platform_id"); }); it("classifies a JSON response end-to-end", () => { const r: CapturedResponse = { request_id: "req_1", url: "https://www.youtube.com/youtubei/v1/search?prettyPrint=false", method: "POST", status: 200, content_type: "application/json; charset=UTF-8", resource_type: "xhr", body_size: 100, body: JSON.stringify(youtubeLike), captured_at: new Date().toISOString() }; const c = classifyResponse(r, youtubeAdapter.classifierHints); expect(c.kind).toBe("json"); expect(c.entities.length).toBe(3); expect(c.fingerprint.observed_entity_types).toContain("video"); expect(c.fingerprint.path_pattern).toBe("/youtubei/v1/search"); expect(c.confidence).toBeGreaterThan(0.8); }); });