spb/llmindex Public

1/**2 * llmindex.io — worker unit tests (cost gate, judge protocol invariants)3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import { describe, expect, it } from 'vitest';8import { DEGRADED_FAILURE_RATE, EST_TOKENS, estimateBatchCostUsd } from './eval-runner';9import {10  JudgeConfigError,11  eligibleJudges,12  loadJudgeConfig,13  parseVerdict,14} from './judges/config';1516describe('cost guardrails', () => {17  it('estimates batch cost from pricing and k samples', () => {18    const cost = estimateBatchCostUsd(200, 1, { promptPerM: 3, completionPerM: 15 });19    expect(cost).toBeCloseTo(20      (200 * (EST_TOKENS.prompt * 3 + EST_TOKENS.completion * 15)) / 1_000_000,21      8,22    );23    const withK = estimateBatchCostUsd(200, 5, { promptPerM: 3, completionPerM: 15 });24    expect(withK).toBeCloseTo(cost * 5, 8);25  });26  it('degraded threshold matches §6 (2%)', () => {27    expect(DEGRADED_FAILURE_RATE).toBe(0.02);28  });29});3031describe('judge config', () => {32  it('requires ≥2 judges across ≥2 providers', () => {33    expect(() => loadJudgeConfig({ JUDGE_MODELS: 'a/x' } as NodeJS.ProcessEnv)).toThrow(34      JudgeConfigError,35    );36    expect(() => loadJudgeConfig({ JUDGE_MODELS: 'a/x,a/y' } as NodeJS.ProcessEnv)).toThrow(37      JudgeConfigError,38    );39    const ok = loadJudgeConfig({ JUDGE_MODELS: 'a/x, b/y' } as NodeJS.ProcessEnv);40    expect(ok.slugs).toEqual(['a/x', 'b/y']);41  });42  it('a model never judges its own duels', () => {43    const config = { slugs: ['a/x', 'b/y', 'c/z'] };44    expect(eligibleJudges(config, 'a/x', 'q/r')).toEqual(['b/y', 'c/z']);45    expect(eligibleJudges(config, 'a/x', 'b/y')).toEqual(['c/z']);46  });47  it('parses verdicts robustly', () => {48    expect(parseVerdict('VERDICT: 1')).toBe('1');49    expect(parseVerdict('after thought…\nverdict: tie')).toBe('tie');50    expect(parseVerdict('no verdict here')).toBeNull();51  });52});53