spb/llmindex Public

1/**2 * llmindex.io — item bank unit tests3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import { describe, expect, it } from 'vitest';8import {9  TEMPLATES,10  createRng,11  extractAnswer,12  generateBatch,13  gradeAnswer,14  normalizeAnswer,15} from './index';16import { numToFrench, numToSpanish } from './templates/multilingual';17import { IRT_HYPERPARAMS } from '@llmindex/scoring';1819describe('rng', () => {20  it('is deterministic for the same seed', () => {21    const a = createRng('seed-1');22    const b = createRng('seed-1');23    expect([a.next(), a.next()]).toEqual([b.next(), b.next()]);24  });25  it('int stays within bounds', () => {26    const rng = createRng('bounds');27    for (let i = 0; i < 500; i++) {28      const v = rng.int(3, 7);29      expect(v).toBeGreaterThanOrEqual(3);30      expect(v).toBeLessThanOrEqual(7);31    }32  });33});3435describe('answer extraction and grading', () => {36  it('extracts the last ANSWER/CONFIDENCE lines', () => {37    const { answer, confidence } = extractAnswer(38      'Reasoning...\nANSWER: draft\nActually:\nANSWER: 42\nCONFIDENCE: 85',39    );40    expect(answer).toBe('42');41    expect(confidence).toBeCloseTo(0.85);42  });43  it('returns nulls when the format is missing', () => {44    expect(extractAnswer('no structured output')).toEqual({ answer: null, confidence: null });45  });46  it('grades numerically with tolerance and formatting noise', () => {47    expect(gradeAnswer('1,234', '1234', 'numeric')).toBe(true);48    expect(gradeAnswer('$1234.00', '1234', 'numeric')).toBe(true);49    expect(gradeAnswer('1235', '1234', 'numeric')).toBe(false);50  });51  it('grades exact answers after normalization (markdown, latex, hyphens)', () => {52    expect(gradeAnswer('  Canberra.', 'canberra', 'exact')).toBe(true);53    expect(normalizeAnswer('  VingT-Deux ! ')).toBe('vingt deux');54    expect(gradeAnswer('vingt-deux', 'vingt deux', 'exact')).toBe(true);55    expect(gradeAnswer('**42**', '42', 'numeric')).toBe(true);56    expect(gradeAnswer('\\boxed{721}', '721', 'numeric')).toBe(true);57  });58  it('extracts markdown-wrapped and boxed answers', () => {59    expect(extractAnswer('**ANSWER:** 42\nCONFIDENCE: 90').answer).toBe('42');60    expect(extractAnswer('thus $x=7$\n\\boxed{7}\nno tag here').answer).toBe('7');61    expect(extractAnswer('FINAL ANSWER: quatre-vingt-un\nCONFIDENCE: 55').answer).toBe(62      'quatre-vingt-un',63    );64  });65});6667describe('templates', () => {68  it('every template renders a valid, self-consistent item', () => {69    for (const t of TEMPLATES) {70      for (let i = 0; i < 25; i++) {71        const seed = `test:${t.id}:${i}`;72        const item = t.render(createRng(seed), seed);73        expect(item.templateId).toBe(t.id);74        expect(item.prompt.length).toBeGreaterThan(20);75        expect(item.answerKey.length).toBeGreaterThan(0);76        expect(item.prompt).toContain('CONFIDENCE');77        if (item.grading === 'constraints') {78          // key is a machine-checkable spec, not a literal answer79          expect(() => JSON.parse(item.answerKey)).not.toThrow();80        } else {81          // a perfect oracle must grade correct against its own key82          expect(gradeAnswer(item.answerKey, item.answerKey, item.grading)).toBe(true);83        }84      }85    }86  });87  it('constraint-stack specs accept a valid witness and reject violations', () => {88    const spec = {89      wordCount: 5,90      startsWithWord: 'nova',91      endsWithWord: 'ember',92      includeWordExactly: [{ word: 'drift', count: 2 }],93      forbiddenLetter: 'j',94      allLowercase: true,95    };96    const key = JSON.stringify(spec);97    expect(gradeAnswer('nova drift and drift ember', key, 'constraints')).toBe(true);98    expect(gradeAnswer('Nova drift and drift ember', key, 'constraints')).toBe(false); // casing99    expect(gradeAnswer('nova drift drift drift ember', key, 'constraints')).toBe(false); // count100  });101  it('renders are deterministic given a seed and differ across seeds', () => {102    const t = TEMPLATES[0]!;103    const one = t.render(createRng('s1'), 's1');104    const two = t.render(createRng('s1'), 's1');105    const three = t.render(createRng('s2'), 's2');106    expect(one.prompt).toBe(two.prompt);107    expect(one.prompt).not.toBe(three.prompt);108  });109});110111describe('generateBatch', () => {112  it('caps anchors at the methodology maximum', () => {113    const batch = generateBatch({ domain: 'math', n: 100, seed: 'run-x', anchorFraction: 0.9 });114    const anchors = batch.filter((i) => i.isAnchor).length;115    expect(anchors).toBeLessThanOrEqual(100 * IRT_HYPERPARAMS.maxAnchorFraction);116  });117  it('anchor items are stable across different batch seeds', () => {118    const a = generateBatch({ domain: 'math', n: 40, seed: 'run-1' });119    const b = generateBatch({ domain: 'math', n: 40, seed: 'run-2' });120    const anchorsA = a.filter((i) => i.isAnchor).map((i) => i.prompt);121    const anchorsB = b.filter((i) => i.isAnchor).map((i) => i.prompt);122    expect(anchorsA).toEqual(anchorsB);123    const freshA = a.filter((i) => !i.isAnchor).map((i) => i.prompt);124    const freshB = b.filter((i) => !i.isAnchor).map((i) => i.prompt);125    expect(freshA).not.toEqual(freshB);126  });127  it('throws for duel-only domains', () => {128    expect(() => generateBatch({ domain: 'writing', n: 5, seed: 's' })).toThrow();129  });130});131132describe('number words', () => {133  it('French including the irregular 70-99 zone and hundreds', () => {134    expect(numToFrench(21)).toBe('vingt et un');135    expect(numToFrench(37)).toBe('trente-sept');136    expect(numToFrench(71)).toBe('soixante et onze');137    expect(numToFrench(77)).toBe('soixante-dix-sept');138    expect(numToFrench(80)).toBe('quatre-vingts');139    expect(numToFrench(91)).toBe('quatre-vingt-onze');140    expect(numToFrench(100)).toBe('cent');141    expect(numToFrench(200)).toBe('deux cents');142    expect(numToFrench(281)).toBe('deux cent quatre-vingt-un');143  });144  it('Spanish including hundreds', () => {145    expect(numToSpanish(22)).toBe('veintidós');146    expect(numToSpanish(41)).toBe('cuarenta y uno');147    expect(numToSpanish(100)).toBe('cien');148    expect(numToSpanish(500)).toBe('quinientos');149    expect(numToSpanish(731)).toBe('setecientos treinta y uno');150  });151});152