SPB Git

Repository Public

templates.ai.test.ts 13.2 KB 248 lines typescript

1import { describe, expect, it } from "vitest";2import katex from "katex";3import { LOCALES, allBlocks, parseSheet, type Block, type LeafBlock, type Locale, type Sheet } from "@/schema";4import { AI_TEMPLATES } from "@/templates/ai";5import { validateSimulation } from "@/sim/registry";67const EXPECTED_IDS = ["transformer-architecture", "mixture-of-experts", "rag-pipeline", "finetuning-vs-rag-vs-prompting", "kv-cache", "neural-network-backprop"];89function build(id: string, locale: Locale): Sheet {10  const def = AI_TEMPLATES.find((d) => d.id === id);11  if (!def) throw new Error(`template ${id} is not exported`);12  const r = parseSheet(def.build(locale));13  if (!r.ok) throw new Error(`${id} [${locale}] is not a valid sheet:\n` + r.issues.map((i) => `  ${i.path}: ${i.message}`).join("\n"));14  return r.sheet;15}1617/** Every string of a document that is shown to the reader, with a path for error messages. Code is excluded. */18function texts(doc: Sheet): { path: string; text: string }[] {19  const out: { path: string; text: string }[] = [];20  const walk = (v: unknown, path: string) => {21    if (typeof v === "string") out.push({ path, text: v });22    else if (Array.isArray(v)) v.forEach((x, i) => walk(x, `${path}[${i}]`));23    else if (v && typeof v === "object") {24      for (const [k, x] of Object.entries(v)) {25        if (["id", "type", "latex", "symbol", "code", "url", "doi", "checkedAt", "from", "to", "group", "blockId"].includes(k)) continue;26        walk(x, `${path}.${k}`);27      }28    }29  };30  walk(doc.metadata, "metadata");31  allBlocks(doc).forEach((b) => walk(b, `block ${b.id}`));32  doc.claims.forEach((c) => walk(c, `claim ${c.id}`));33  return out;34}3536const blocksOf = <T extends Block["type"]>(doc: Sheet, type: T) => allBlocks(doc).filter((b): b is Extract<Block | LeafBlock, { type: T }> => b.type === type);37const renders = (latex: string, displayMode: boolean) => katex.renderToString(latex, { throwOnError: true, displayMode, strict: "error" });3839describe("AI templates — catalogue", () => {40  it("exports the six templates with stable ids, bilingual titles and descriptions", () => {41    expect(AI_TEMPLATES.map((d) => d.id)).toEqual(EXPECTED_IDS);42    for (const def of AI_TEMPLATES) {43      expect(def.category).toBe("ai");44      for (const locale of LOCALES) {45        expect(def.title[locale].length).toBeGreaterThan(5);46        expect(def.description[locale].length).toBeGreaterThan(30);47      }48    }49  });5051  it("builds deterministically", () => {52    for (const def of AI_TEMPLATES) for (const locale of LOCALES) expect(JSON.stringify(def.build(locale))).toBe(JSON.stringify(def.build(locale)));53  });54});5556describe.each(EXPECTED_IDS)("AI template %s", (id) => {57  describe.each(LOCALES)("locale %s", (locale) => {58    const doc = build(id, locale);59    const blocks = allBlocks(doc);6061    it("is a valid sheet marked as an example, in the requested language", () => {62      expect(doc.metadata.locale).toBe(locale);63      expect(doc.metadata.example).toBe(true);64      expect(doc.metadata.objectives.length).toBeGreaterThanOrEqual(2);65      expect(doc.metadata.objectives.length).toBeLessThanOrEqual(4);66      expect(doc.metadata.summary.length).toBeGreaterThan(40);67      expect(doc.metadata.topics.length).toBeGreaterThan(0);68    });6970    it("has at least 14 blocks with unique ids, starting with an h1 and a lead and ending with the references", () => {71      expect(blocks.length).toBeGreaterThanOrEqual(14);72      expect(new Set(blocks.map((b) => b.id)).size).toBe(blocks.length);73      expect(blocks[0]).toMatchObject({ type: "heading", level: 1 });74      expect((blocks[0] as { eyebrow?: string }).eyebrow).toBeTruthy();75      expect(blocks[1]).toMatchObject({ type: "paragraph", lead: true });76      expect(blocks.at(-1)!.type).toBe("references");77      expect(blocksOf(doc, "quiz").length).toBe(1);78      expect(blocksOf(doc, "accordion").length).toBe(1);79      expect(blocksOf(doc, "equation").length).toBeGreaterThanOrEqual(2);80    });8182    it("lays blocks out in rows that add up to 12 columns", () => {83      let row = 0;84      for (const b of doc.blocks) {85        row += b.span;86        expect(row, `row overflows at block ${b.id}`).toBeLessThanOrEqual(12);87        if (row === 12) row = 0;88      }89      expect(row).toBe(0);90    });9192    it("only cites sources that exist, and uses every source", () => {93      const sources = new Set(doc.sources.map((s) => s.id));94      const cited = new Set<string>();95      for (const { path, text } of texts(doc)) for (const m of text.matchAll(/\[\^([^\]]+)\]/g)) {96        expect(sources.has(m[1]), `${path}: unknown citation [^${m[1]}]`).toBe(true);97        cited.add(m[1]);98      }99      for (const b of blocks) for (const s of b.sourceIds ?? []) cited.add(s);100      for (const c of doc.claims) for (const s of c.sourceIds) cited.add(s);101      expect([...sources].filter((s) => !cited.has(s))).toEqual([]);102    });103104    it("has 3–6 verified sources and 4–8 claims that are sourced or derived", () => {105      expect(doc.sources.length).toBeGreaterThanOrEqual(3);106      expect(doc.sources.length).toBeLessThanOrEqual(6);107      for (const s of doc.sources) {108        expect(s.status, s.id).toBe("resolved");109        expect(s.checkedAt, s.id).toBeTruthy();110        expect(s.url ?? s.doi, s.id).toBeTruthy();111        expect(s.authors.length, s.id).toBeGreaterThan(0);112        expect(s.year, s.id).toBeGreaterThan(1900);113      }114      expect(doc.claims.length).toBeGreaterThanOrEqual(4);115      expect(doc.claims.length).toBeLessThanOrEqual(8);116      for (const c of doc.claims) {117        expect(["sourced", "derived"], c.id).toContain(c.status);118        if (c.status === "sourced") expect(c.sourceIds.length, c.id).toBeGreaterThan(0);119      }120    });121122    it("has a rich diagram whose edges, groups and animation refer to existing nodes", () => {123      const diagrams = blocksOf(doc, "diagram");124      expect(diagrams.length).toBeGreaterThanOrEqual(1);125      expect(Math.max(...diagrams.map((d) => d.nodes.length))).toBeGreaterThanOrEqual(8);126      for (const d of diagrams) {127        const nodes = new Set(d.nodes.map((n) => n.id));128        const edges = new Set(d.edges.map((e) => e.id));129        const groups = new Set(d.groups.map((g) => g.id));130        expect(nodes.size).toBe(d.nodes.length);131        expect(edges.size).toBe(d.edges.length);132        for (const e of d.edges) {133          expect(nodes.has(e.from), `${d.id}/${e.id}: from ${e.from}`).toBe(true);134          expect(nodes.has(e.to), `${d.id}/${e.id}: to ${e.to}`).toBe(true);135        }136        for (const n of d.nodes) {137          if (n.group) expect(groups.has(n.group), `${d.id}/${n.id}: group`).toBe(true);138          expect(d.edges.some((e) => e.from === n.id || e.to === n.id), `${d.id}/${n.id} is isolated`).toBe(true);139        }140        expect(d.legend.length, `${d.id}: legend`).toBeGreaterThanOrEqual(3);141        for (const s of d.animation?.steps ?? []) {142          for (const n of s.nodes) expect(nodes.has(n), `${d.id}: animation node ${n}`).toBe(true);143          for (const e of s.edges) expect(edges.has(e), `${d.id}: animation edge ${e}`).toBe(true);144          expect(s.nodes.length + s.edges.length).toBeGreaterThan(0);145        }146        if (d.animation) {147          expect(d.animation.steps.length).toBeGreaterThanOrEqual(4);148          expect(d.animation.steps.length).toBeLessThanOrEqual(7);149        }150        // Diagram labels are drawn as SVG text: no inline LaTeX there.151        for (const label of [d.title, ...d.nodes.flatMap((n) => [n.label, n.sublabel]), ...d.edges.map((e) => e.label), ...d.groups.map((g) => g.label)]) {152          expect(label ?? "", `${d.id}: label`).not.toContain("$");153        }154      }155    });156157    it("gives every visual block a real text alternative", () => {158      for (const b of blocks) if (["diagram", "chart", "matrix", "simulation"].includes(b.type)) expect((b.alt ?? "").length, `${b.id}: alt`).toBeGreaterThan(60);159    });160161    it("never presents hand-made numbers as measurements", () => {162      for (const b of [...blocksOf(doc, "chart"), ...blocksOf(doc, "matrix")]) {163        expect(["illustrative", "synthetic"], b.id).toContain(b.dataKind);164        expect((b.caption ?? "").length, `${b.id}: the caption must say what the numbers are`).toBeGreaterThan(40);165      }166    });167168    it("has equations that KaTeX can render, with every variable defined", () => {169      for (const e of blocksOf(doc, "equation")) {170        expect(() => renders(e.latex, true), `${e.id}: ${e.latex}`).not.toThrow();171        expect((e.variables ?? []).length, `${e.id}: variables`).toBeGreaterThan(0);172        for (const v of e.variables ?? []) expect(() => renders(v.symbol, false), `${e.id}: symbol ${v.symbol}`).not.toThrow();173      }174    });175176    it("has inline LaTeX that KaTeX can render, and balanced inline markup", () => {177      for (const { path, text } of texts(doc)) {178        const withoutCode = text.replace(/`[^`]*`/g, "");179        expect((withoutCode.match(/\$/g) ?? []).length % 2, `${path}: unbalanced $ in "${text}"`).toBe(0);180        for (const m of withoutCode.matchAll(/\$([^$]+)\$/g)) expect(() => renders(m[1], false), `${path}: $${m[1]}$`).not.toThrow();181        const prose = withoutCode.replace(/\$[^$]+\$/g, "");182        expect((prose.match(/\*\*/g) ?? []).length % 2, `${path}: unbalanced ** in "${text}"`).toBe(0);183        expect(prose, `${path}: raw HTML`).not.toMatch(/<\/?[a-z][^>]*>/i);184      }185    });186187    it("has a quiz with exactly one correct, explained answer", () => {188      const [quiz] = blocksOf(doc, "quiz");189      expect(quiz.choices.filter((c) => c.correct).length).toBe(1);190      for (const c of quiz.choices) expect(c.explanation, quiz.id).toBeTruthy();191    });192  });193194  it("has the same structure in French and in English", () => {195    const fr = build(id, "fr"), en = build(id, "en");196    expect(allBlocks(fr).map((b) => `${b.id}:${b.type}:${b.span}`)).toEqual(allBlocks(en).map((b) => `${b.id}:${b.type}:${b.span}`));197    expect(fr.sources).toEqual(en.sources);198    expect(fr.claims.map((c) => [c.id, c.status, c.sourceIds, c.blockId])).toEqual(en.claims.map((c) => [c.id, c.status, c.sourceIds, c.blockId]));199    const shape = (d: Sheet) => allBlocks(d).map((b) => {200      switch (b.type) {201        case "diagram": return [b.nodes.map((n) => n.id), b.edges.map((e) => e.id), b.animation?.steps.length, b.legend.length];202        case "equation": return [b.latex, (b.variables ?? []).map((v) => v.symbol)];203        case "chart": return b.series.map((s) => s.points.map((p) => p.y));204        case "matrix": return b.values;205        case "simulation": return [b.kind, b.params, Object.keys(b.config), b.seed];206        case "cards": case "steps": case "accordion": case "list": return b.items.length;207        case "table": case "comparison": return [b.columns.length, b.rows.length];208        case "quiz": return b.choices.map((c) => c.correct);209        default: return null;210      }211    });212    expect(shape(fr)).toEqual(shape(en));213  });214215  it("is actually written in each language", () => {216    const fr = build(id, "fr"), en = build(id, "en");217    const lead = (d: Sheet) => (d.blocks[1] as Extract<Block, { type: "paragraph" }>).text;218    const words = (s: string) => new Set(s.toLowerCase().split(/[^a-zàâçéèêëîïôûùüÿœ]+/).filter(Boolean));219    const frWords = words(lead(fr)), enWords = words(lead(en));220    const FR = ["le", "la", "les", "un", "une", "des", "est", "et", "qui", "que", "pour", "dans"];221    const EN = ["the", "is", "and", "of", "that", "which", "with", "for", "its", "it"];222    expect(FR.filter((w) => frWords.has(w)).length).toBeGreaterThanOrEqual(4);223    expect(EN.filter((w) => frWords.has(w)).length).toBeLessThanOrEqual(1);224    expect(EN.filter((w) => enWords.has(w)).length).toBeGreaterThanOrEqual(3);225    expect(lead(fr)).not.toBe(lead(en));226    // Every prose block differs between the two languages (nothing left untranslated).227    const proseTypes = ["paragraph", "callout", "accordion", "quiz", "steps", "cards", "comparison"];228    allBlocks(fr).forEach((b, i) => { if (proseTypes.includes(b.type)) expect(JSON.stringify(b), `block ${b.id} is identical in both languages`).not.toBe(JSON.stringify(allBlocks(en)[i])); });229    // French typography: guillemets, never straight double quotes around words; no ordinary space before : ; ? !230    for (const { path, text } of texts(fr)) {231      const prose = text.replace(/`[^`]*`/g, "").replace(/\$[^$]+\$/g, "");232      expect(prose, `${path}: straight or English quotes in French`).not.toMatch(/["“”]/);233      expect(prose, `${path}: ordinary space before punctuation`).not.toMatch(/[a-zàâçéèêëîïôûùü»)] [:;?!](\s|$)/i);234    }235  });236237  it("uses the simulation that fits the subject", () => {238    const expected: Record<string, string | undefined> = { "transformer-architecture": "attention", "mixture-of-experts": "moe-routing", "kv-cache": "kv-cache", "neural-network-backprop": "neural-forward" };239    const sims = blocksOf(build(id, "en"), "simulation").map((s) => s.kind);240    if (expected[id]) expect(sims).toEqual([expected[id]]); else expect(sims).toEqual([]);241    // Initial parameters and options are accepted by the simulation library, in both languages.242    for (const locale of LOCALES) for (const sim of blocksOf(build(id, locale), "simulation")) {243      expect(Object.keys(sim.params).length, `${sim.id}: initial parameters`).toBeGreaterThan(0);244      expect(validateSimulation(sim), `${sim.id} [${locale}]`).toEqual([]);245    }246  });247});248
Files main
Menu

Theme