spb/llmindex Public

1/**2 * llmindex.io — scoring package unit tests3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import { describe, expect, it } from 'vitest';8import {9  DOMAINS,10  DOMAIN_WEIGHTS,11  SUBMETRIC_WEIGHTS,12  abilityToUnit,13  contaminationResistance,14  domainComposite,15  domainScore,16  globalIndex,17  paretoFrontier,18  thetaToIndex,19} from './index';2021describe('weights invariants', () => {22  it('domain weights sum to 1', () => {23    const sum = Object.values(DOMAIN_WEIGHTS).reduce((a, b) => a + b, 0);24    expect(sum).toBeCloseTo(1, 10);25  });26  it('sub-metric weights sum to 1 and exclude latency/cost', () => {27    const sum = Object.values(SUBMETRIC_WEIGHTS).reduce((a, b) => a + b, 0);28    expect(sum).toBeCloseTo(1, 10);29    expect(Object.keys(SUBMETRIC_WEIGHTS)).not.toContain('latency_p50');30    expect(Object.keys(SUBMETRIC_WEIGHTS)).not.toContain('cost_per_1k_items');31  });32  it('has the 12 v0.2 domains', () => {33    expect(DOMAINS).toHaveLength(12);34    expect(DOMAINS).toContain('agentic');35    expect(DOMAINS).toContain('terminal');36    expect(DOMAINS).toContain('svg_design');37    expect(DOMAINS).toContain('vision_ocr');38  });39});4041describe('thetaToIndex', () => {42  it('maps θ=0 to center with symmetric CI', () => {43    const s = thetaToIndex(0, 0.2);44    expect(s.score).toBe(500);45    expect(s.score - s.scoreLow).toBe(s.scoreHigh - s.score);46  });47  it('is monotonic in θ and clamps to [0,1000]', () => {48    expect(thetaToIndex(1, 0.1).score).toBeGreaterThan(thetaToIndex(-1, 0.1).score);49    expect(thetaToIndex(10, 0.1).score).toBe(1000);50    expect(thetaToIndex(-10, 0.1).score).toBe(0);51  });52});5354describe('domain composite', () => {55  it('renormalizes over missing metrics (accuracy-only equals ability)', () => {56    const c = domainComposite({ theta: 0.8, thetaSe: 0.1 });57    expect(c).toBeCloseTo(abilityToUnit(0.8), 10);58  });59  it('penalizes contamination', () => {60    const clean = domainComposite({ theta: 0.5, thetaSe: 0.1, contaminationDelta: 0 });61    const dirty = domainComposite({ theta: 0.5, thetaSe: 0.1, contaminationDelta: 0.2 });62    expect(clean).toBeGreaterThan(dirty);63  });64  it('contamination resistance floors at 0 for gaps ≥ 20 points', () => {65    expect(contaminationResistance(0.25)).toBe(0);66    expect(contaminationResistance(0)).toBe(1);67  });68  it('domainScore CI widens with θ SE', () => {69    const narrow = domainScore({ theta: 0, thetaSe: 0.05 });70    const wide = domainScore({ theta: 0, thetaSe: 0.5 });71    expect(wide.scoreHigh - wide.scoreLow).toBeGreaterThan(narrow.scoreHigh - narrow.scoreLow);72  });73});7475describe('globalIndex', () => {76  it('returns null with no domains', () => {77    expect(globalIndex({})).toBeNull();78  });79  it('averages with renormalized weights', () => {80    const g = globalIndex({81      math: { score: 600, scoreLow: 580, scoreHigh: 620 },82      code: { score: 400, scoreLow: 380, scoreHigh: 420 },83    });84    expect(g?.score).toBe(500);85  });86});8788describe('paretoFrontier', () => {89  it('keeps only non-dominated points', () => {90    const frontier = paretoFrontier([91      { slug: 'cheap-weak', score: 400, costPer1kItems: 1 },92      { slug: 'dominated', score: 390, costPer1kItems: 2 },93      { slug: 'mid', score: 600, costPer1kItems: 5 },94      { slug: 'best-expensive', score: 800, costPer1kItems: 30 },95    ]);96    expect(frontier.map((p) => p.slug)).toEqual(['cheap-weak', 'mid', 'best-expensive']);97  });98});99