Repository Public
1import { describe, expect, it } from "vitest";2import katex from "katex";3import { LOCALES, allBlocks, parseSheet, type Block, type LeafBlock, type Locale, type Sheet } from "@/schema";4import { AI_TEMPLATES } from "@/templates/ai";5import { validateSimulation } from "@/sim/registry";67const EXPECTED_IDS = ["transformer-architecture", "mixture-of-experts", "rag-pipeline", "finetuning-vs-rag-vs-prompting", "kv-cache", "neural-network-backprop"];89function build(id: string, locale: Locale): Sheet {10 const def = AI_TEMPLATES.find((d) => d.id === id);11 if (!def) throw new Error(`template ${id} is not exported`);12 const r = parseSheet(def.build(locale));13 if (!r.ok) throw new Error(`${id} [${locale}] is not a valid sheet:\n` + r.issues.map((i) => ` ${i.path}: ${i.message}`).join("\n"));14 return r.sheet;15}1617/** Every string of a document that is shown to the reader, with a path for error messages. Code is excluded. */18function texts(doc: Sheet): { path: string; text: string }[] {19 const out: { path: string; text: string }[] = [];20 const walk = (v: unknown, path: string) => {21 if (typeof v === "string") out.push({ path, text: v });22 else if (Array.isArray(v)) v.forEach((x, i) => walk(x, `${path}[${i}]`));23 else if (v && typeof v === "object") {24 for (const [k, x] of Object.entries(v)) {25 if (["id", "type", "latex", "symbol", "code", "url", "doi", "checkedAt", "from", "to", "group", "blockId"].includes(k)) continue;26 walk(x, `${path}.${k}`);27 }28 }29 };30 walk(doc.metadata, "metadata");31 allBlocks(doc).forEach((b) => walk(b, `block ${b.id}`));32 doc.claims.forEach((c) => walk(c, `claim ${c.id}`));33 return out;34}3536const blocksOf = <T extends Block["type"]>(doc: Sheet, type: T) => allBlocks(doc).filter((b): b is Extract<Block | LeafBlock, { type: T }> => b.type === type);37const renders = (latex: string, displayMode: boolean) => katex.renderToString(latex, { throwOnError: true, displayMode, strict: "error" });3839describe("AI templates — catalogue", () => {40 it("exports the six templates with stable ids, bilingual titles and descriptions", () => {41 expect(AI_TEMPLATES.map((d) => d.id)).toEqual(EXPECTED_IDS);42 for (const def of AI_TEMPLATES) {43 expect(def.category).toBe("ai");44 for (const locale of LOCALES) {45 expect(def.title[locale].length).toBeGreaterThan(5);46 expect(def.description[locale].length).toBeGreaterThan(30);47 }48 }49 });5051 it("builds deterministically", () => {52 for (const def of AI_TEMPLATES) for (const locale of LOCALES) expect(JSON.stringify(def.build(locale))).toBe(JSON.stringify(def.build(locale)));53 });54});5556describe.each(EXPECTED_IDS)("AI template %s", (id) => {57 describe.each(LOCALES)("locale %s", (locale) => {58 const doc = build(id, locale);59 const blocks = allBlocks(doc);6061 it("is a valid sheet marked as an example, in the requested language", () => {62 expect(doc.metadata.locale).toBe(locale);63 expect(doc.metadata.example).toBe(true);64 expect(doc.metadata.objectives.length).toBeGreaterThanOrEqual(2);65 expect(doc.metadata.objectives.length).toBeLessThanOrEqual(4);66 expect(doc.metadata.summary.length).toBeGreaterThan(40);67 expect(doc.metadata.topics.length).toBeGreaterThan(0);68 });6970 it("has at least 14 blocks with unique ids, starting with an h1 and a lead and ending with the references", () => {71 expect(blocks.length).toBeGreaterThanOrEqual(14);72 expect(new Set(blocks.map((b) => b.id)).size).toBe(blocks.length);73 expect(blocks[0]).toMatchObject({ type: "heading", level: 1 });74 expect((blocks[0] as { eyebrow?: string }).eyebrow).toBeTruthy();75 expect(blocks[1]).toMatchObject({ type: "paragraph", lead: true });76 expect(blocks.at(-1)!.type).toBe("references");77 expect(blocksOf(doc, "quiz").length).toBe(1);78 expect(blocksOf(doc, "accordion").length).toBe(1);79 expect(blocksOf(doc, "equation").length).toBeGreaterThanOrEqual(2);80 });8182 it("lays blocks out in rows that add up to 12 columns", () => {83 let row = 0;84 for (const b of doc.blocks) {85 row += b.span;86 expect(row, `row overflows at block ${b.id}`).toBeLessThanOrEqual(12);87 if (row === 12) row = 0;88 }89 expect(row).toBe(0);90 });9192 it("only cites sources that exist, and uses every source", () => {93 const sources = new Set(doc.sources.map((s) => s.id));94 const cited = new Set<string>();95 for (const { path, text } of texts(doc)) for (const m of text.matchAll(/\[\^([^\]]+)\]/g)) {96 expect(sources.has(m[1]), `${path}: unknown citation [^${m[1]}]`).toBe(true);97 cited.add(m[1]);98 }99 for (const b of blocks) for (const s of b.sourceIds ?? []) cited.add(s);100 for (const c of doc.claims) for (const s of c.sourceIds) cited.add(s);101 expect([...sources].filter((s) => !cited.has(s))).toEqual([]);102 });103104 it("has 3–6 verified sources and 4–8 claims that are sourced or derived", () => {105 expect(doc.sources.length).toBeGreaterThanOrEqual(3);106 expect(doc.sources.length).toBeLessThanOrEqual(6);107 for (const s of doc.sources) {108 expect(s.status, s.id).toBe("resolved");109 expect(s.checkedAt, s.id).toBeTruthy();110 expect(s.url ?? s.doi, s.id).toBeTruthy();111 expect(s.authors.length, s.id).toBeGreaterThan(0);112 expect(s.year, s.id).toBeGreaterThan(1900);113 }114 expect(doc.claims.length).toBeGreaterThanOrEqual(4);115 expect(doc.claims.length).toBeLessThanOrEqual(8);116 for (const c of doc.claims) {117 expect(["sourced", "derived"], c.id).toContain(c.status);118 if (c.status === "sourced") expect(c.sourceIds.length, c.id).toBeGreaterThan(0);119 }120 });121122 it("has a rich diagram whose edges, groups and animation refer to existing nodes", () => {123 const diagrams = blocksOf(doc, "diagram");124 expect(diagrams.length).toBeGreaterThanOrEqual(1);125 expect(Math.max(...diagrams.map((d) => d.nodes.length))).toBeGreaterThanOrEqual(8);126 for (const d of diagrams) {127 const nodes = new Set(d.nodes.map((n) => n.id));128 const edges = new Set(d.edges.map((e) => e.id));129 const groups = new Set(d.groups.map((g) => g.id));130 expect(nodes.size).toBe(d.nodes.length);131 expect(edges.size).toBe(d.edges.length);132 for (const e of d.edges) {133 expect(nodes.has(e.from), `${d.id}/${e.id}: from ${e.from}`).toBe(true);134 expect(nodes.has(e.to), `${d.id}/${e.id}: to ${e.to}`).toBe(true);135 }136 for (const n of d.nodes) {137 if (n.group) expect(groups.has(n.group), `${d.id}/${n.id}: group`).toBe(true);138 expect(d.edges.some((e) => e.from === n.id || e.to === n.id), `${d.id}/${n.id} is isolated`).toBe(true);139 }140 expect(d.legend.length, `${d.id}: legend`).toBeGreaterThanOrEqual(3);141 for (const s of d.animation?.steps ?? []) {142 for (const n of s.nodes) expect(nodes.has(n), `${d.id}: animation node ${n}`).toBe(true);143 for (const e of s.edges) expect(edges.has(e), `${d.id}: animation edge ${e}`).toBe(true);144 expect(s.nodes.length + s.edges.length).toBeGreaterThan(0);145 }146 if (d.animation) {147 expect(d.animation.steps.length).toBeGreaterThanOrEqual(4);148 expect(d.animation.steps.length).toBeLessThanOrEqual(7);149 }150 // Diagram labels are drawn as SVG text: no inline LaTeX there.151 for (const label of [d.title, ...d.nodes.flatMap((n) => [n.label, n.sublabel]), ...d.edges.map((e) => e.label), ...d.groups.map((g) => g.label)]) {152 expect(label ?? "", `${d.id}: label`).not.toContain("$");153 }154 }155 });156157 it("gives every visual block a real text alternative", () => {158 for (const b of blocks) if (["diagram", "chart", "matrix", "simulation"].includes(b.type)) expect((b.alt ?? "").length, `${b.id}: alt`).toBeGreaterThan(60);159 });160161 it("never presents hand-made numbers as measurements", () => {162 for (const b of [...blocksOf(doc, "chart"), ...blocksOf(doc, "matrix")]) {163 expect(["illustrative", "synthetic"], b.id).toContain(b.dataKind);164 expect((b.caption ?? "").length, `${b.id}: the caption must say what the numbers are`).toBeGreaterThan(40);165 }166 });167168 it("has equations that KaTeX can render, with every variable defined", () => {169 for (const e of blocksOf(doc, "equation")) {170 expect(() => renders(e.latex, true), `${e.id}: ${e.latex}`).not.toThrow();171 expect((e.variables ?? []).length, `${e.id}: variables`).toBeGreaterThan(0);172 for (const v of e.variables ?? []) expect(() => renders(v.symbol, false), `${e.id}: symbol ${v.symbol}`).not.toThrow();173 }174 });175176 it("has inline LaTeX that KaTeX can render, and balanced inline markup", () => {177 for (const { path, text } of texts(doc)) {178 const withoutCode = text.replace(/`[^`]*`/g, "");179 expect((withoutCode.match(/\$/g) ?? []).length % 2, `${path}: unbalanced $ in "${text}"`).toBe(0);180 for (const m of withoutCode.matchAll(/\$([^$]+)\$/g)) expect(() => renders(m[1], false), `${path}: $${m[1]}$`).not.toThrow();181 const prose = withoutCode.replace(/\$[^$]+\$/g, "");182 expect((prose.match(/\*\*/g) ?? []).length % 2, `${path}: unbalanced ** in "${text}"`).toBe(0);183 expect(prose, `${path}: raw HTML`).not.toMatch(/<\/?[a-z][^>]*>/i);184 }185 });186187 it("has a quiz with exactly one correct, explained answer", () => {188 const [quiz] = blocksOf(doc, "quiz");189 expect(quiz.choices.filter((c) => c.correct).length).toBe(1);190 for (const c of quiz.choices) expect(c.explanation, quiz.id).toBeTruthy();191 });192 });193194 it("has the same structure in French and in English", () => {195 const fr = build(id, "fr"), en = build(id, "en");196 expect(allBlocks(fr).map((b) => `${b.id}:${b.type}:${b.span}`)).toEqual(allBlocks(en).map((b) => `${b.id}:${b.type}:${b.span}`));197 expect(fr.sources).toEqual(en.sources);198 expect(fr.claims.map((c) => [c.id, c.status, c.sourceIds, c.blockId])).toEqual(en.claims.map((c) => [c.id, c.status, c.sourceIds, c.blockId]));199 const shape = (d: Sheet) => allBlocks(d).map((b) => {200 switch (b.type) {201 case "diagram": return [b.nodes.map((n) => n.id), b.edges.map((e) => e.id), b.animation?.steps.length, b.legend.length];202 case "equation": return [b.latex, (b.variables ?? []).map((v) => v.symbol)];203 case "chart": return b.series.map((s) => s.points.map((p) => p.y));204 case "matrix": return b.values;205 case "simulation": return [b.kind, b.params, Object.keys(b.config), b.seed];206 case "cards": case "steps": case "accordion": case "list": return b.items.length;207 case "table": case "comparison": return [b.columns.length, b.rows.length];208 case "quiz": return b.choices.map((c) => c.correct);209 default: return null;210 }211 });212 expect(shape(fr)).toEqual(shape(en));213 });214215 it("is actually written in each language", () => {216 const fr = build(id, "fr"), en = build(id, "en");217 const lead = (d: Sheet) => (d.blocks[1] as Extract<Block, { type: "paragraph" }>).text;218 const words = (s: string) => new Set(s.toLowerCase().split(/[^a-zàâçéèêëîïôûùüÿœ]+/).filter(Boolean));219 const frWords = words(lead(fr)), enWords = words(lead(en));220 const FR = ["le", "la", "les", "un", "une", "des", "est", "et", "qui", "que", "pour", "dans"];221 const EN = ["the", "is", "and", "of", "that", "which", "with", "for", "its", "it"];222 expect(FR.filter((w) => frWords.has(w)).length).toBeGreaterThanOrEqual(4);223 expect(EN.filter((w) => frWords.has(w)).length).toBeLessThanOrEqual(1);224 expect(EN.filter((w) => enWords.has(w)).length).toBeGreaterThanOrEqual(3);225 expect(lead(fr)).not.toBe(lead(en));226 // Every prose block differs between the two languages (nothing left untranslated).227 const proseTypes = ["paragraph", "callout", "accordion", "quiz", "steps", "cards", "comparison"];228 allBlocks(fr).forEach((b, i) => { if (proseTypes.includes(b.type)) expect(JSON.stringify(b), `block ${b.id} is identical in both languages`).not.toBe(JSON.stringify(allBlocks(en)[i])); });229 // French typography: guillemets, never straight double quotes around words; no ordinary space before : ; ? !230 for (const { path, text } of texts(fr)) {231 const prose = text.replace(/`[^`]*`/g, "").replace(/\$[^$]+\$/g, "");232 expect(prose, `${path}: straight or English quotes in French`).not.toMatch(/["“”]/);233 expect(prose, `${path}: ordinary space before punctuation`).not.toMatch(/[a-zàâçéèêëîïôûùü»)] [:;?!](\s|$)/i);234 }235 });236237 it("uses the simulation that fits the subject", () => {238 const expected: Record<string, string | undefined> = { "transformer-architecture": "attention", "mixture-of-experts": "moe-routing", "kv-cache": "kv-cache", "neural-network-backprop": "neural-forward" };239 const sims = blocksOf(build(id, "en"), "simulation").map((s) => s.kind);240 if (expected[id]) expect(sims).toEqual([expected[id]]); else expect(sims).toEqual([]);241 // Initial parameters and options are accepted by the simulation library, in both languages.242 for (const locale of LOCALES) for (const sim of blocksOf(build(id, locale), "simulation")) {243 expect(Object.keys(sim.params).length, `${sim.id}: initial parameters`).toBeGreaterThan(0);244 expect(validateSimulation(sim), `${sim.id} [${locale}]`).toEqual([]);245 }246 });247});248